diff --git a/pyproject.toml b/pyproject.toml index d5441a62..4d5b3689 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ dependencies = [ "colorama", "confetti>=2.5.3", "dessert>=1.6.3,<2", - "emport~=1.3.1", "gossip>=2.3.0", "Jinja2", "Logbook>=1.2.0", diff --git a/slash/core/local_config.py b/slash/core/local_config.py index 13ffa6e1..dd2ae010 100644 --- a/slash/core/local_config.py +++ b/slash/core/local_config.py @@ -1,6 +1,6 @@ import os import dessert -from emport import import_file +from ..utils.import_utils import import_file from ..utils.python import check_duplicate_functions class LocalConfig(object): diff --git a/slash/loader.py b/slash/loader.py index c22e50a1..a323b3a8 100644 --- a/slash/loader.py +++ b/slash/loader.py @@ -6,7 +6,6 @@ from contextlib import contextmanager import dessert -from emport import import_file from logbook import Logger from sentinels import NOTHING @@ -22,6 +21,7 @@ from .exception_handling import handling_exceptions, mark_exception_handled, get_exception_frame_correction from .exceptions import CannotLoadTests, SlashInternalError from .core.runnable_test_factory import RunnableTestFactory +from .utils.import_utils import import_file from .utils.pattern_matching import Matcher from .utils.python import check_duplicate_functions from .resuming import ResumedTestData diff --git a/slash/plugins/plugin_manager.py b/slash/plugins/plugin_manager.py index 6d5a97e8..6b6e64ab 100644 --- a/slash/plugins/plugin_manager.py +++ b/slash/plugins/plugin_manager.py @@ -11,7 +11,7 @@ from ..utils.python import reraise from ..conf import config from contextlib import contextmanager -from emport import import_file +from ..utils.import_utils import import_file from sentinels import NOTHING from vintage import warn_deprecation import logbook diff --git a/slash/utils/import_utils.py b/slash/utils/import_utils.py new file mode 100644 index 00000000..ba24eb74 --- /dev/null +++ b/slash/utils/import_utils.py @@ -0,0 +1,86 @@ +"""Importing Python files by path, as modules of synthetic top-level packages. + +Adapted from emport (https://github.com/vmalloc/emport, BSD license, by Rotem Yaari). +""" +import importlib.util +import itertools +import os +import sys +from importlib.machinery import ModuleSpec, SourceFileLoader + +import logbook + +_logger = logbook.Logger(__name__) + + +class NoInitFileFound(Exception): + pass + + +def import_file(filename): + """Given a path to a file, imports it as a Python module + """ + module_name = _create_new_module_name(filename) + return importlib.import_module(module_name) + + +_package_name_generator = ('_{}'.format(x) for x in itertools.count()) +_cached_package_names = {} + + +def _generate_package_name(): + for suggested in _package_name_generator: + if suggested not in sys.modules: + return suggested + + +def _create_new_module_name(filename): + _logger.trace('Creating new package for {}', filename) + nonpackage_dir, remainder = _split_nonpackage_dir(filename) + _logger.trace('After split: {}, {}', nonpackage_dir, remainder) + package_name = _cached_package_names.get(nonpackage_dir, None) + if package_name is None: + package_name = _generate_package_name() + _create_package_module(package_name, nonpackage_dir) + _cached_package_names[nonpackage_dir] = package_name + returned = '{}.{}'.format(package_name, remainder) + if returned.endswith('.__init__'): + returned = returned.rsplit('.', 1)[0] + return returned + + +def _split_nonpackage_dir(path): + if not os.path.isdir(path): + nonpackage_dir, module = os.path.split(_normalize_path(path)) + module = _make_module_name(module).split('.') + else: + nonpackage_dir = path + module = [] + while os.path.isfile(os.path.join(nonpackage_dir, '__init__.py')): + if '.' in os.path.split(nonpackage_dir)[-1]: + # we cannot import from such packages, stop traversing upwards... + break + nonpackage_dir, current_component = os.path.split(nonpackage_dir) + module.insert(0, current_component) + _logger.trace('Now at {}, {}', nonpackage_dir, module) + if not module: + raise NoInitFileFound('Could not find __init__.py file in {}'.format(path)) + return nonpackage_dir, '.'.join(module) + + +def _normalize_path(path): + return os.path.normpath(os.path.abspath(str(path))) + + +def _make_module_name(filename): + assert filename.endswith('.py') or filename.endswith('.pyc') + return filename.rsplit('.', 1)[0].replace(os.path.sep, '.') + + +def _create_package_module(name, path): + spec = ModuleSpec(origin=path, name=name, loader=SourceFileLoader(name, path), is_package=True) + spec.submodule_search_locations.append(path) + returned = importlib.util.module_from_spec(spec) + returned.__path__ = [path] + sys.modules[name] = returned + return returned diff --git a/tests/test_error_object.py b/tests/test_error_object.py index df2d50bf..93079644 100644 --- a/tests/test_error_object.py +++ b/tests/test_error_object.py @@ -3,12 +3,12 @@ import os import types -import emport import dessert import pytest from slash.core.error import Error from slash.exception_handling import mark_exception_frame_correction +from slash.utils.import_utils import import_file from .utils import without_pyc @@ -218,7 +218,7 @@ def func(): assert f(g(1)) == g(f(2))""") with dessert.rewrite_assertions_context(): - module = emport.import_file(str(filename)) + module = import_file(str(filename)) try: module.func() diff --git a/tests/test_import_utils.py b/tests/test_import_utils.py new file mode 100644 index 00000000..1535fada --- /dev/null +++ b/tests/test_import_utils.py @@ -0,0 +1,153 @@ +# pylint: disable=redefined-outer-name +import os +import sys +import warnings +from uuid import uuid4 + +import pytest +from slash.utils.import_utils import NoInitFileFound, import_file + + +@pytest.fixture +def module_file_factory(): + def create_module_file(directory, filename): + directory = str(directory) + os.makedirs(directory, exist_ok=True) + value = str(uuid4()) + full_filename = os.path.join(directory, filename) + with open(full_filename, 'w') as f: + f.write('value = {!r}'.format(value)) + return full_filename, value + return create_module_file + + +def test_import_from_non_package(tmpdir, module_file_factory): + for i in range(5): + _import_and_verify(*module_file_factory(tmpdir, 'module{}.py'.format(i))) + + +def test_multiple_imports_from_same_non_package(tmpdir, module_file_factory): + module1 = _import_and_verify(*module_file_factory(tmpdir, 'module1.py')) + module2 = _import_and_verify(*module_file_factory(tmpdir, 'module2.py')) + assert _package_name(module1) == _package_name(module2) + assert module1 is not module2 + + +def test_importing_sub_package(tmpdir, module_file_factory): + subpackage_dir = tmpdir.join('subpackage') + subpackage_dir.join('__init__.py').ensure(file=True) + filename, _ = module_file_factory(subpackage_dir, 'module.py') + assert import_file(filename).__name__.endswith('.subpackage.module') + + +def test_importing_sub_package_and_subdir(tmpdir, module_file_factory): + module = _import_and_verify(*module_file_factory(tmpdir, 'module.py')) + subdir = tmpdir.join('sub') + subdir.join('__init__.py').ensure(file=True) + sub_module = _import_and_verify(*module_file_factory(subdir, 'module.py')) + assert _package_name(module) + '.sub' == _package_name(sub_module) + + +def test_importing_different_directories_same_escaping(tmpdir, module_file_factory): + module1 = _import_and_verify(*module_file_factory(tmpdir.join('pkg+'), 'module.py')) + module2 = _import_and_verify(*module_file_factory(tmpdir.join('pkg-'), 'module.py')) + assert _package_name(module1) != _package_name(module2) + + +def test_importing_dotted_name(tmpdir, module_file_factory): + path = tmpdir + for component in ('a', 'b.c', 'd'): + path = path.join(component) + path.join('__init__.py').ensure(file=True) + _import_and_verify(*module_file_factory(path, 'module.py')) + + +def test_importing_directory_no_init_file(tmpdir): + with pytest.raises(NoInitFileFound): + import_file(str(tmpdir)) + + +@pytest.mark.parametrize('init_py_directly', [True, False]) +def test_importing_directory(init_py_directly, tmpdir, module_file_factory): + directory = str(tmpdir.join('pkg')) + init_filename, value = module_file_factory(directory, '__init__.py') + module = import_file(init_filename if init_py_directly else directory) + assert module.__name__.endswith('.pkg') + assert module.value == value + + +def test_importing_subdirectory_init_file(tmpdir, module_file_factory): + directory = tmpdir.join('pkg') + module_file_factory(directory, '__init__.py') + filename, value = module_file_factory(directory.join('subpkg'), '__init__.py') + assert import_file(filename).value == value + + +def test_relative_imports(tmpdir): + directory = tmpdir.join('dir') + directory.join('__init__.py').ensure(file=True) + directory.join('file_1.py').write('value = "file_1"') + directory.join('proxy.py').write('from .file_1 import value as file_1_value') + assert import_file(str(directory.join('proxy.py'))).file_1_value == 'file_1' + + +def test_module_specs(tmpdir): + package_dir = tmpdir.join('package') + subpackage_dir = package_dir.join('sub') + for p in (package_dir, subpackage_dir): + p.join('__init__.py').ensure(file=True) + subpackage_dir.join('utils.py').ensure(file=True) + filename = subpackage_dir.join('module.py') + filename.write('from . import utils') + + module = import_file(str(filename)) + + for m in (module, module.utils): + metapackage_name, remainder = m.__spec__.name.split('.', 1) + assert remainder == 'package.sub.{}'.format(m.__name__.split('.')[-1]) + assert m.__spec__.parent == '{}.package.sub'.format(metapackage_name) + + metapackage = sys.modules[metapackage_name] + assert metapackage.__spec__.origin == str(package_dir.dirname) + assert metapackage.__spec__.name == metapackage_name + assert str(package_dir.dirname) in metapackage.__spec__.submodule_search_locations + assert metapackage.__package__ == metapackage_name + + +@pytest.mark.parametrize('add_init_py', [True, False]) +def test_importing_doesnt_emit_warnings(tmpdir, add_init_py): + value = str(uuid4()) + directory = tmpdir.join('files') + directory.join('utils.py').write('value = {!r}'.format(value), ensure=True) + filename = directory.join('testfile.py') + filename.write('from .utils import value as new_value') + if add_init_py: + directory.join('__init__.py').ensure(file=True) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + module = import_file(str(filename)) + + assert module.new_value == value + assert not caught + + +def test_failed_package_import_is_not_left_in_sys_modules(tmpdir): + directory = tmpdir.join('broken_pkg') + directory.join('__init__.py').write('raise RuntimeError("broken")', ensure=True) + directory.join('module.py').ensure(file=True) + + with pytest.raises(RuntimeError, match='broken'): + import_file(str(directory.join('module.py'))) + + assert not [name for name in sys.modules if name.endswith('.broken_pkg')] + + +def _import_and_verify(filename, value): + module = import_file(filename) + assert module.value == value + return module + + +def _package_name(module): + return module.__name__.rsplit('.', 1)[0] diff --git a/tests/test_traceback_utils.py b/tests/test_traceback_utils.py index 7fed0dd4..3dd0fd49 100644 --- a/tests/test_traceback_utils.py +++ b/tests/test_traceback_utils.py @@ -1,10 +1,10 @@ import sys -import emport import vintage from slash.core.error import Error from slash.utils import traceback_utils +from slash.utils.import_utils import import_file from slash.utils.traceback_utils import _MAX_VARIABLE_VALUE_LENGTH @@ -30,7 +30,7 @@ def context(): yield ''') - mod = emport.import_file(str(filename)) + mod = import_file(str(filename)) try: mod.f() except ZeroDivisionError: