"maximum recursion depth reached" when submitting ~200 workchains

[ltalirz]

@danieleongari reports the following error when submitting ~200 workchains with aiida-core 1.6.1

2021-04-27 16:41:13 [26387 | ERROR]: Traceback (most recent call last):
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/manage/external/rmq.py", line 208, in _continue
    result = await super()._continue(communicator, pid, nowait, tag)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/plumpy/process_comms.py", line 607, in _continue
    proc = cast('Process', saved_state.unbundle(self._load_context))
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/plumpy/persistence.py", line 60, in unbundle
    return Savable.load(self, load_context)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/plumpy/persistence.py", line 452, in load
    return load_cls.recreate_from(saved_state, load_context)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/plumpy/processes.py", line 238, in recreate_from
    process = cast(Process, super().recreate_from(saved_state, load_context))
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/plumpy/persistence.py", line 477, in recreate_from
    call_with_super_check(obj.load_instance_state, saved_state, load_context)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/plumpy/base/utils.py", line 29, in call_with_super_check
    wrapped(*args, **kwargs)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/engine/processes/workchains/workchain.py", line 105, in load_instance_state
    super().load_instance_state(saved_state, load_context)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/engine/processes/process.py", line 284, in load_instance_state
    super().load_instance_state(saved_state, load_context)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/plumpy/processes.py", line 620, in load_instance_state
    decoded = self.decode_input_args(saved_state[BundleKeys.INPUTS_RAW])
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/engine/processes/process.py", line 607, in decode_input_args
    return serialize.deserialize(encoded)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/utils/serialize.py", line 230, in deserialize
    return yaml.load(serialized, Loader=AiiDALoader)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/__init__.py", line 114, in load
    return loader.get_single_data()
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 43, in get_single_data
    return self.construct_document(node)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 47, in construct_document
    data = self.construct_object(node)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 92, in construct_object
    data = constructor(self, node)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/utils/serialize.py", line 131, in mapping_constructor
    yaml_node = loader.construct_mapping(mapping, deep=True)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 210, in construct_mapping
    return super().construct_mapping(node, deep=deep)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 135, in construct_mapping
    value = self.construct_object(value_node, deep=deep)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 99, in construct_object
    for dummy in generator:
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 404, in construct_yaml_map
    value = self.construct_mapping(node)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 210, in construct_mapping
    return super().construct_mapping(node, deep=deep)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 135, in construct_mapping
    value = self.construct_object(value_node, deep=deep)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/yaml/constructor.py", line 92, in construct_object
    data = constructor(self, node)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/utils/serialize.py", line 56, in node_constructor
    return orm.load_node(uuid=yaml_node)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/utils/__init__.py", line 197, in load_node
    return load_entity(
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/utils/__init__.py", line 77, in load_entity
    return entity_loader.load_entity(
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/utils/loaders.py", line 213, in load_entity
    entity = builder.one()[0]
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/querybuilder.py", line 2179, in one
    res = self.all()
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/querybuilder.py", line 2252, in all
    matches = list(self.iterall(batch_size=batch_size))
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/querybuilder.py", line 2209, in iterall
    query = self.get_query()
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/querybuilder.py", line 2088, in get_query
    query = self._build()
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/querybuilder.py", line 1938, in _build
    self._query = self._query.filter(self._build_filters(alias, filter_specs))
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/querybuilder.py", line 1373, in _build_filters
    self._impl.get_filter_expr(
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/implementation/django/querybuilder.py", line 212, in get_filter_expr
    self.get_filter_expr(
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/implementation/django/querybuilder.py", line 237, in get_filter_expr
    expr = self.get_filter_expr_from_column(operator, value, column)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/aiida/orm/implementation/querybuilder.py", line 217, in get_filter_expr_from_column
    expr = database_entity.cast(String).like(value)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 236, in __getattr__
    return getattr(self.comparator, key)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 987, in __getattr__
    return self._fallback_getattr(key)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/orm/properties.py", line 364, in _fallback_getattr
    return getattr(self.__clause_element__(), key)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 974, in oneshot
    result = fn(*args, **kw)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/orm/properties.py", line 316, in _memoized_method___clause_element__
    return self.adapter(self.prop.columns[0])
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/orm/util.py", line 680, in _adapt_element
    return self._adapter.traverse(elem)._annotate(
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/util.py", line 936, in traverse
    return self.columns[obj]
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/util/_collections.py", line 745, in __missing__
    self[key] = val = self.creator(self.weakself(), key)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/util.py", line 943, in _locate_col
    c = ClauseAdapter.traverse(self, col)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py", line 240, in traverse
    return replacement_traverse(obj, self.__traverse_options__, replace)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py", line 484, in replacement_traverse
    obj = clone(obj, **opts)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py", line 473, in clone
    newelem = replace(elem)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py", line 236, in replace
    e = v.replace(elem)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/util.py", line 848, in replace
    return self._corresponding_column(col, True)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/util.py", line 820, in _corresponding_column
    newcol = self.selectable.corresponding_column(
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/selectable.py", line 560, in corresponding_column
    if self.c.contains_column(column):
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 893, in __get__
    obj.__dict__[self.__name__] = result = self.fget(obj)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/selectable.py", line 647, in columns
    self._populate_column_collection()
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/selectable.py", line 1393, in _populate_column_collection
    col._make_proxy(self)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 1802, in _make_proxy
    c = self._constructor(
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 1568, in __init__
    self._init_items(*args)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 121, in _init_items
    spwd(self)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/base.py", line 461, in _set_parent_with_dispatch
    self._set_parent(parent)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 2282, in _set_parent
    self.parent._on_table_attach(self._set_table)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 1722, in _on_table_attach
    event.listen(self, "after_parent_attach", fn)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/event/api.py", line 102, in listen
    _event_key(target, identifier, fn).listen(*args, **kw)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/event/api.py", line 25, in _event_key
    tgt = evt_cls._accept_with(target)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/event/base.py", line 232, in _accept_with
    if hasattr(target, "dispatch"):
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/event/base.py", line 298, in __get__
    obj.__dict__["dispatch"] = disp = self.dispatch._for_instance(obj)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/event/base.py", line 121, in _for_instance
    return self._for_class(instance_cls)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/event/base.py", line 117, in _for_class
    return self.__class__(self, instance_cls)
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/site-packages/sqlalchemy/event/base.py", line 83, in __init__
    self._empty_listeners = self._empty_listener_reg[instance_cls]
  File "/home/daniele/anaconda3/envs/aiida_py38/lib/python3.8/weakref.py", line 383, in __getitem__
    return self.data[ref(key)]
RecursionError: maximum recursion depth exceeded while calling a Python object

the error goes away when adding a time.sleep(2) in between submissions

Your environment

• Operating system [e.g. Linux]: ubuntu
• Python version [e.g. 3.7.1]: 3.8
• aiida-core version [e.g. 1.2.1]: 1.6.1

[sphuber]

Does this come from the submission script or the daemon? Looks to be on the daemon side, correct?

[ltalirz]

Sorry, forgot to mention: it comes from the submission script, not from the daemon logs

[chrisjsewell]

It may also be helpful to provide the output of pip freeze, I.e to check the version of sqlalchemy

[sphuber]

Sorry, forgot to mention: it comes from the submission script

Are you sure? Is he running the workchains in that script or sending them to the daemon? Reason that I am asking is that the stack trace shows that the problem originates from the ProcessLauncher._continue_ call which should not be called during submission. This is the hook that is called on a daemon runner when it receives a task to continue a process from RabbitMQ.

[danieleongari]

@sphuber The error comes from inspecting the report (verdi process report). The workchain is submitted from a Jupyter notebook using the builder, and then submitting it. Of the ca. 200 calculations, the first 10ish run ok, and then the most of them start to be excepted with the error that Leo reported: it looks like there is a flooding of processes that causes problems.

@chrisjsewell is this enough?

~ pip freeze | egrep sql
SQLAlchemy @ file:///home/conda/feedstock_root/build_artifacts/sqlalchemy_1612225077951/work
SQLAlchemy-Utils @ file:///home/conda/feedstock_root/build_artifacts/sqlalchemy-utils_1614043858099/work
sqlparse @ file:///home/conda/feedstock_root/build_artifacts/sqlparse_1602142927465/work

Thank you for the help!

[sphuber]

Thanks for the additional info @danieleongari . It would be very useful if we actually could get the version of sqlalchemy. Maybe you can do the following in a shell or notebook:

In [1]: import sqlalchemy

In [2]: sqlalchemy.__version__
Out[2]: '1.3.23'

So as expected, the problem is on the daemon worker side and not the submit script. What is happening is that the daemon worker receives a task from the process queue from RabbitMQ, it loads the corresponding node from the database and then uses the YAML dump in the checkpoint attribute, to reconstruct the Process instance in memory from that serialized version. This includes the entire set of inputs of the process that were also serialized and so each of those are reloaded from the database, which happens in the line:

return orm.load_node(uuid=yaml_node)

from the aiida.orm.utils.serialize module. The load_node call will use the QueryBuilder underneath which will then go into sqlalchemy which has a big part in the stack trace. This is where I lose the thread, because apparently somewhere in the sqlalchemy code that is invoked when we call load_node leads to this infinite recursion. I have no idea why or why this should be related to many processes being run.

The only hint there is is that the final method in the stacktrace comes from the weakref built in module. At this point I have to start speculating but it could be that the daemon worker is trying to load a node that is already loaded in its memory (as part of the inputs of another process that it is working on, they could share the same Code as input for example) and there might be some caching mechanism in place that when the same node gets loaded, it is coupled to the existing ref instead. I am really spitballing here and have no idea how to further debug this or try and reproduce.

[ltalirz]

In [1]: import sqlalchemy

In [2]: sqlalchemy.__version__
Out[2]: '1.3.23'

sorry for the hickkup with the submission script vs daemon - there was a miscommunication on our side

[mbercx]

I'm having a somewhat similar issue, so I'll add my case to this one. When submitting a 100 PwBaseWorkChains in a loop (without any pause in between 😬 ), half of them got stuck in the "Running" state:

$ verdi process list -S running
   PK  Created    Process label    Process State    Process status
-----  ---------  ---------------  ---------------  ----------------
36774  1h ago     PwBaseWorkChain  ⏵ Running
36783  1h ago     PwBaseWorkChain  ⏵ Running
...
37166  1h ago     PwBaseWorkChain  ⏵ Running

Total results: 49

(Note that I've already deleted and restarted one manually). The logs for these are completely empty:

$ verdi process report 37156
No log messages recorded for this entry

But a bit of digging through the daemon logs gives me the following trace:

05/03/2021 08:52:17 PM <16823> kiwipy.rmq.tasks: [ERROR] Exception occurred while processing task.
Traceback (most recent call last):
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/plumpy/utils.py", line 128, in __getattr__
    return self[attr]
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/plumpy/utils.py", line 85, in __getitem__
    return self._dict[key]
KeyError: 'kpoints'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/mbercx/envs/aiida-sirius/code/aiida-quantumespresso/aiida_quantumespresso/workflows/pw/base.py", line 247, in validate_kpoints
    kpoints = self.inputs.kpoints
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/plumpy/utils.py", line 131, in __getattr__
    raise AttributeError(errmsg)
AttributeError: 'AttributesFrozendict' object has no attribute 'kpoints'

These two are basically repeated until a RecursionError is raised:

Traceback (most recent call last):
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/kiwipy/rmq/tasks.py", line 166, in _on_task
    result = await result
  File "/usr/lib/python3.8/asyncio/futures.py", line 257, in __await__
    yield self  # This tells Task to wait for completion.
  File "/usr/lib/python3.8/asyncio/tasks.py", line 349, in __wakeup
    future.result()
  File "/usr/lib/python3.8/asyncio/futures.py", line 175, in result
    raise self._exception
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/kiwipy/rmq/threadcomms.py", line 253, in done
    result = kiwi_future.result()
  File "/usr/lib/python3.8/concurrent/futures/_base.py", line 432, in result
    return self.__get_result()
  File "/usr/lib/python3.8/concurrent/futures/_base.py", line 388, in __get_result
    raise self._exception
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/kiwipy/futures.py", line 54, in capture_exceptions
    yield
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/plumpy/communications.py", line 48, in on_done
    result = plum_future.result()
  File "/usr/lib/python3.8/asyncio/futures.py", line 175, in result
    raise self._exception
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/kiwipy/futures.py", line 54, in capture_exceptions
    yield
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/plumpy/futures.py", line 73, in run_task
    res = await coro()
  File "/home/mbercx/.virtualenvs/aiida-sirius/lib/python3.8/site-packages/plumpy/process_comms.py", line 539, in __call__
    return await self._continue(communicator, **task.get(TASK_ARGS, {}))
  File "/home/mbercx/envs/aiida-sirius/code/aiida-core/aiida/manage/external/rmq.py", line 219, in _continue
    self.handle_continue_exception(node, exception, message)
  File "/home/mbercx/envs/aiida-sirius/code/aiida-core/aiida/manage/external/rmq.py", line 158, in handle_continue_exception
    node.logger.exception(message)
  File "/usr/lib/python3.8/logging/__init__.py", line 1814, in exception
    self.log(ERROR, msg, *args, exc_info=exc_info, **kwargs)
  File "/usr/lib/python3.8/logging/__init__.py", line 1829, in log
    self.logger.log(level, msg, *args, **kwargs)
  File "/usr/lib/python3.8/logging/__init__.py", line 1500, in log
    self._log(level, msg, args, **kwargs)
  File "/usr/lib/python3.8/logging/__init__.py", line 1577, in _log
    self.handle(record)
  File "/usr/lib/python3.8/logging/__init__.py", line 1587, in handle
    self.callHandlers(record)
  File "/usr/lib/python3.8/logging/__init__.py", line 1649, in callHandlers
    hdlr.handle(record)
  File "/usr/lib/python3.8/logging/__init__.py", line 950, in handle
    self.emit(record)
  File "/usr/lib/python3.8/logging/__init__.py", line 1081, in emit
    msg = self.format(record)
  File "/usr/lib/python3.8/logging/__init__.py", line 925, in format
    return fmt.format(record)
  File "/usr/lib/python3.8/logging/__init__.py", line 672, in format
    record.exc_text = self.formatException(record.exc_info)
  File "/usr/lib/python3.8/logging/__init__.py", line 622, in formatException
    traceback.print_exception(ei[0], ei[1], tb, None, sio)
  File "/usr/lib/python3.8/traceback.py", line 103, in print_exception
    for line in TracebackException(
  File "/usr/lib/python3.8/traceback.py", line 493, in __init__
    context = TracebackException(
  File "/usr/lib/python3.8/traceback.py", line 493, in __init__
    context = TracebackException(
  File "/usr/lib/python3.8/traceback.py", line 493, in __init__
    context = TracebackException(
  [Previous line repeated 34 more times]
  File "/usr/lib/python3.8/traceback.py", line 476, in __init__
    _seen.add(id(exc_value))
RecursionError: maximum recursion depth exceeded while calling a Python object

After deleting one and restarting it, it ran just fine. I've also started ~35 work chains like this just fine. I then tried deleting the remaining 49, and restarting them all at once, and then I wind up with 7 PwBaseWorkChains that are stuck in Running, with the same error trace as above, after which is a whole range of reports of launching the PwCalculations:

RecursionError: maximum recursion depth exceeded while calling a Python object
05/03/2021 10:28:13 PM <17394> aiida.orm.nodes.process.workflow.workchain.WorkChainNode: [REPORT] [37947|PwBaseWorkChain|run_process]: launching PwCalculation<37961> iteration #1
05/03/2021 10:28:14 PM <17394> aiida.orm.nodes.process.workflow.workchain.WorkChainNode: [REPORT] [37849|PwBaseWorkChain|run_process]: launching PwCalculation<37964> iteration #1
...

Afterwards, I deleted all 100 work chains, and then restarted them with a 5 second pause in between. Now all work chains were able to create the PwCalculations without issue.

I'm running:

OS: Ubuntu 18.04.5 LTS
Python: 3.8
aiida-core: 1.6.3 (but I think I've also spotted this issue when I was running v1.6.1, just couldn't pin it down at the time)
plumpy: 0.19.0
kiwipy: 0.7.4
sqlalchemy: 1.3.23

[dev-zero]

Seeing the same thing, also when submitting a large amount of workchains (whether one-by-one in a script or by a parent workchain launching them does not matter, but when launching them via a parent workchain the issue is triggered sooner).

The backtrace is a bit different, though:

2021-05-14 10:55:09 [1739 | REPORT]: [6047|Cp2kEosWorkChain|on_except]: Traceback (most recent call last):
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/process_states.py", line 230, in execute
    result = self.run_fn(*self.args, **self.kwargs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/workchains/workchain.py", line 214, in _do_step
    finished, stepper_result = self._stepper.step()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/workchains.py", line 299, in step
    finished, result = self._child_stepper.step()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/workchains.py", line 250, in step
    return True, self._fn(self._workchain)
  File "/scratch/tiziano/work/aiida/aiida-cp2k/aiida_cp2k/workchains/eos.py", line 170, in run_calculations
    builder.cp2k.structure = get_rescaled_structure(self.inputs.structure, Float(scale))
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/functions.py", line 179, in decorated_function
    result, _ = run_get_node(*args, **kwargs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/functions.py", line 131, in run_get_node
    process = process_class(inputs=inputs, runner=runner)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 193, in __call__
    inst.transition_to(inst.create_initial_state())
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 335, in transition_to
    self.transition_failed(initial_state_label, label, *sys.exc_info()[1:])
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 351, in transition_failed
    raise exception.with_traceback(trace)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 320, in transition_to
    self._enter_next_state(new_state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 382, in _enter_next_state
    self._fire_state_event(StateEventHook.ENTERING_STATE, next_state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 299, in _fire_state_event
    callback(self, hook, state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/processes.py", line 324, in <lambda>
    lambda _s, _h, state: self.on_entering(cast(process_states.State, state)),
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 380, in on_entering
    super().on_entering(state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/processes.py", line 669, in on_entering
    call_with_super_check(self.on_create)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/utils.py", line 29, in call_with_super_check
    wrapped(*args, **kwargs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 376, in on_create
    self._pid = self._create_and_setup_db_record()  # pylint: disable=attribute-defined-outside-init
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 563, in _create_and_setup_db_record
    self._setup_db_record()
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/functions.py", line 362, in _setup_db_record
    super()._setup_db_record()
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 672, in _setup_db_record
    self._setup_inputs()
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 709, in _setup_inputs
    self.node.add_incoming(node, LinkType.INPUT_CALC, name)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/nodes/node.py", line 417, in add_incoming
    self.validate_incoming(source, link_type, link_label)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/nodes/process/process.py", line 472, in validate_incoming
    super().validate_incoming(source, link_type, link_label)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/utils/mixins.py", line 148, in validate_incoming
    super().validate_incoming(source, link_type=link_type, link_label=link_label)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/nodes/node.py", line 449, in validate_incoming
    if builder.count() > 0:
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/querybuilder.py", line 2164, in count
    return self._impl.count(query)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/implementation/querybuilder.py", line 288, in count
    return query.count()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3803, in count
    return self.from_self(col).scalar()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3523, in scalar
    ret = self.one()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3490, in one
    ret = self.one_or_none()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3459, in one_or_none
    ret = list(self)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__
    return self._execute_and_instances(context)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
    result = conn.execute(querycontext.statement, self._params)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
    return meth(self, multiparams, params)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1115, in _execute_clauseelement
    compiled_sql = elem.compile(
  File "<string>", line 1, in <lambda>
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 481, in compile
    return self._compiler(dialect, bind=bind, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 487, in _compiler
    return dialect.statement_compiler(dialect, self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 592, in __init__
    Compiled.__init__(self, dialect, statement, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 322, in __init__
    self.string = self.process(self.statement, **compile_kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 352, in process
    return obj._compiler_dispatch(self, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2201, in visit_select
    text = self._compose_select_body(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2292, in _compose_select_body
    [
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2293, in <listcomp>
    f._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1801, in visit_alias
    ret = alias.original._compiler_dispatch(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2201, in visit_select
    text = self._compose_select_body(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2292, in _compose_select_body
    [
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2293, in <listcomp>
    f._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2435, in visit_join
    join.left._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2437, in visit_join
    + join.right._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1727, in visit_cte
    self.visit_cte(pre_alias_cte, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1758, in visit_cte
    cte.original._compiler_dispatch(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1216, in visit_compound_select
    text = (" " + keyword + " ").join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1218, in <genexpr>
    c._compiler_dispatch(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2201, in visit_select
    text = self._compose_select_body(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2301, in _compose_select_body
    t = select._whereclause._compiler_dispatch(self, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in visit_clauselist
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in <genexpr>
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1043, in <genexpr>
    c._compiler_dispatch(self, **kw) for c in clauselist.clauses
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in visit_clauselist
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in <genexpr>
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1043, in <genexpr>
    c._compiler_dispatch(self, **kw) for c in clauselist.clauses
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1354, in visit_binary
    return disp(binary, operator_, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1471, in visit_like_op_binary
    binary.left._compiler_dispatch(self, **kw),
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1075, in visit_cast
    cast.clause._compiler_dispatch(self, **kwargs),
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/annotation.py", line 79, in _compiler_dispatch
    return self.__element.__class__._compiler_dispatch(self, visitor, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 912, in visit_column
    if not is_literal and isinstance(name, elements._truncated_label):
RecursionError: maximum recursion depth exceeded while calling a Python object

aiida-core: 403f7e7
plumpy: 0.19.0
kiwipy: 0.7.4
sqlalchemy: 1.3.24
python: 3.9.5

[zhubonan]

Just want to say that I did a similar thing like submitting the work chains (~100 VaspRelaxWorkChain) in a loop without delay previously (aiida-core < 1.6.1), and found no issue. In my case, the submission process itself is rather slow though, takes about 1.5 seconds for each call to submit.

[dev-zero]

And another one, again at a different place (but this one comes from the sub-workchain called by the primary one):

$ verdi process report 15370
2021-05-17 10:26:29 [7359 | REPORT]: [15370|Cp2kBaseWorkChain|on_except]: Traceback (most recent call last):
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/process_states.py", line 230, in execute
    result = self.run_fn(*self.args, **self.kwargs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/workchains/workchain.py", line 214, in _do_step
    finished, stepper_result = self._stepper.step()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/workchains.py", line 299, in step
    finished, result = self._child_stepper.step()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/workchains.py", line 532, in step
    finished, result = self._child_stepper.step()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/workchains.py", line 299, in step
    finished, result = self._child_stepper.step()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/workchains.py", line 250, in step
    return True, self._fn(self._workchain)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/workchains/restart.py", line 183, in run_process
    node = self.submit(self.process_class, **inputs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 498, in submit
    return self.runner.submit(process, *args, **kwargs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/runners.py", line 184, in submit
    process_inited = self.instantiate_process(process, *args, **inputs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/runners.py", line 170, in instantiate_process
    return instantiate_process(self, process, *args, **inputs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/utils.py", line 65, in instantiate_process
    process = process_class(runner=runner, inputs=inputs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 193, in __call__
    inst.transition_to(inst.create_initial_state())
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 335, in transition_to
    self.transition_failed(initial_state_label, label, *sys.exc_info()[1:])
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 351, in transition_failed
    raise exception.with_traceback(trace)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 320, in transition_to
    self._enter_next_state(new_state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 382, in _enter_next_state
    self._fire_state_event(StateEventHook.ENTERING_STATE, next_state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/state_machine.py", line 299, in _fire_state_event
    callback(self, hook, state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/processes.py", line 324, in <lambda>
    lambda _s, _h, state: self.on_entering(cast(process_states.State, state)),
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 380, in on_entering
    super().on_entering(state)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/processes.py", line 669, in on_entering
    call_with_super_check(self.on_create)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/plumpy/base/utils.py", line 29, in call_with_super_check
    wrapped(*args, **kwargs)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 376, in on_create
    self._pid = self._create_and_setup_db_record()  # pylint: disable=attribute-defined-outside-init
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 563, in _create_and_setup_db_record
    self._setup_db_record()
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 672, in _setup_db_record
    self._setup_inputs()
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/engine/processes/process.py", line 709, in _setup_inputs
    self.node.add_incoming(node, LinkType.INPUT_CALC, name)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/nodes/node.py", line 417, in add_incoming
    self.validate_incoming(source, link_type, link_label)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/nodes/process/process.py", line 472, in validate_incoming
    super().validate_incoming(source, link_type, link_label)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/utils/mixins.py", line 148, in validate_incoming
    super().validate_incoming(source, link_type=link_type, link_label=link_label)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/nodes/node.py", line 449, in validate_incoming
    if builder.count() > 0:
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/querybuilder.py", line 2164, in count
    return self._impl.count(query)
  File "/scratch/tiziano/work/aiida/aiida-core/aiida/orm/implementation/querybuilder.py", line 288, in count
    return query.count()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3803, in count
    return self.from_self(col).scalar()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3523, in scalar
    ret = self.one()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3490, in one
    ret = self.one_or_none()
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3459, in one_or_none
    ret = list(self)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__
    return self._execute_and_instances(context)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
    result = conn.execute(querycontext.statement, self._params)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
    return meth(self, multiparams, params)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1115, in _execute_clauseelement
    compiled_sql = elem.compile(
  File "<string>", line 1, in <lambda>
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 481, in compile
    return self._compiler(dialect, bind=bind, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 487, in _compiler
    return dialect.statement_compiler(dialect, self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 592, in __init__
    Compiled.__init__(self, dialect, statement, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 322, in __init__
    self.string = self.process(self.statement, **compile_kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 352, in process
    return obj._compiler_dispatch(self, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2201, in visit_select
    text = self._compose_select_body(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2292, in _compose_select_body
    [
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2293, in <listcomp>
    f._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1801, in visit_alias
    ret = alias.original._compiler_dispatch(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2201, in visit_select
    text = self._compose_select_body(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2292, in _compose_select_body
    [
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2293, in <listcomp>
    f._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2435, in visit_join
    join.left._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2437, in visit_join
    + join.right._compiler_dispatch(self, asfrom=True, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1727, in visit_cte
    self.visit_cte(pre_alias_cte, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1758, in visit_cte
    cte.original._compiler_dispatch(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1216, in visit_compound_select
    text = (" " + keyword + " ").join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1218, in <genexpr>
    c._compiler_dispatch(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2201, in visit_select
    text = self._compose_select_body(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 2301, in _compose_select_body
    t = select._whereclause._compiler_dispatch(self, **kwargs)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in visit_clauselist
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in <genexpr>
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1043, in <genexpr>
    c._compiler_dispatch(self, **kw) for c in clauselist.clauses
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in visit_clauselist
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1040, in <genexpr>
    text = sep.join(
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1043, in <genexpr>
    c._compiler_dispatch(self, **kw) for c in clauselist.clauses
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1354, in visit_binary
    return disp(binary, operator_, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1471, in visit_like_op_binary
    binary.left._compiler_dispatch(self, **kw),
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/visitors.py", line 96, in _compiler_dispatch
    return meth(self, **kw)
  File "/scratch/tiziano/virtualenvs/aiida/lib/python3.9/site-packages/sqlalchemy/sql/compiler.py", line 1075, in visit_cast
    cast.clause._compiler_dispatch(self, **kwargs),
RecursionError: maximum recursion depth exceeded in comparison