Change the way we invalidate the module cache to support reloading code from subpackages

This commit is contained in:
kkthxbye-code 2023-02-16 12:33:08 +01:00 committed by Jeremy Stretch
parent 3150c1f8b3
commit c78022a74c

View File

@ -524,27 +524,39 @@ def get_scripts(use_names=False):
defined name in place of the actual module name. defined name in place of the actual module name.
""" """
scripts = {} scripts = {}
# Iterate through all modules within the scripts path. These are the user-created files in which reports are
# Get all modules within the scripts path. These are the user-created files in which scripts are
# defined. # defined.
for importer, module_name, _ in pkgutil.iter_modules([settings.SCRIPTS_ROOT]): modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT]))
# Use a lock as removing and loading modules is not thread safe modules_bases = set([name.split(".")[0] for _, name, _ in modules])
# Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is
# removed from sys.modules while another thread is importing
with lock: with lock:
# Remove cached module to ensure consistency with filesystem for module_name in list(sys.modules.keys()):
if module_name in sys.modules: # Everything sharing a base module path with a module in the script folder is removed.
# We also remove all modules with a base module called "scripts". This allows modifying imported
# non-script modules without having to reload the RQ worker.
module_base = module_name.split(".")[0]
if module_base == "scripts" or module_base in modules_bases:
del sys.modules[module_name] del sys.modules[module_name]
for importer, module_name, _ in modules:
module = importer.find_module(module_name).load_module(module_name) module = importer.find_module(module_name).load_module(module_name)
if use_names and hasattr(module, 'name'): if use_names and hasattr(module, 'name'):
module_name = module.name module_name = module.name
module_scripts = {} module_scripts = {}
script_order = getattr(module, "script_order", ()) script_order = getattr(module, "script_order", ())
ordered_scripts = [cls for cls in script_order if is_script(cls)] ordered_scripts = [cls for cls in script_order if is_script(cls)]
unordered_scripts = [cls for _, cls in inspect.getmembers(module, is_script) if cls not in script_order] unordered_scripts = [cls for _, cls in inspect.getmembers(module, is_script) if cls not in script_order]
for cls in [*ordered_scripts, *unordered_scripts]: for cls in [*ordered_scripts, *unordered_scripts]:
# For scripts in submodules use the full import path w/o the root module as the name # For scripts in submodules use the full import path w/o the root module as the name
script_name = cls.full_name.split(".", maxsplit=1)[1] script_name = cls.full_name.split(".", maxsplit=1)[1]
module_scripts[script_name] = cls module_scripts[script_name] = cls
if module_scripts: if module_scripts:
scripts[module_name] = module_scripts scripts[module_name] = module_scripts