Merge remote-tracking branch 'upstream/main' into docs/integrate-tutorials

Author: GeigerJ2

.docker/aiida-core-base/s6-assets/init/aiida-prepare.sh

@@ -18,8 +18,11 @@ verdi config set warnings.development_version False
 # If the environment variable `SETUP_DEFAULT_AIIDA_PROFILE` is not set, set it to `true`.
 if [[ ${SETUP_DEFAULT_AIIDA_PROFILE:-true} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then
 
-    # Create AiiDA profile.
-    verdi presto \
+    # Create AiiDA profile. Set AIIDA_NO_DAEMON_AUTOSTART because the
+    # Docker container manages the daemon lifecycle separately via
+    # s6-overlay (aiida-daemon-start runs after this script and
+    # run-before-daemon-start).
+    AIIDA_NO_DAEMON_AUTOSTART=1 verdi presto \
         --verbosity info \
         --profile-name "${AIIDA_PROFILE_NAME:-default}" \
         --email "${AIIDA_USER_EMAIL:-aiida@localhost}" \

.github/workflows/ci-code.yml

@@ -26,8 +26,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.14']
+        python-version: ['3.10', '3.14']
         database-backend: [psql]
+        broker-backend: [rmq, zmq]
 
     services:
       postgres:
@@ -72,7 +73,7 @@ jobs:
         AIIDA_TEST_PROFILE: test_aiida
         AIIDA_WARN_v3: 1
       run: |
-        pytest -n auto --db-backend ${{ matrix.database-backend }} -m 'not nightly' tests/ ${{ matrix.python-version == '3.14' && '--cov aiida' || '' }}
+        pytest -n auto --db-backend ${{ matrix.database-backend }} --broker-backend ${{ matrix.broker-backend }} -m 'not nightly' tests/ ${{ matrix.python-version == '3.14' && '--cov aiida' || '' }}
 
     - name: Upload coverage report
       if: matrix.python-version == 3.14 && github.repository == 'aiidateam/aiida-core'
@@ -101,16 +102,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9']
+        python-version: ['3.10']
         database-backend: [sqlite]
 
-    services:
-      rabbitmq:
-        image: rabbitmq:3.8.14-management
-        ports:
-        - 5672:5672
-        - 15672:15672
-
     steps:
     - uses: actions/checkout@v6
 
@@ -131,7 +125,7 @@ jobs:
     - name: Run test suite
       env:
         AIIDA_WARN_v3: 0
-      run: pytest --disable-warnings -n auto --db-backend ${{ matrix.database-backend }} -m 'not nightly' tests/
+      run: pytest --disable-warnings -n auto --db-backend ${{ matrix.database-backend }} --broker-backend zmq -m 'not nightly' tests/
 
 
   tests-presto:
@@ -156,7 +150,7 @@ jobs:
     - name: Run test suite
       env:
         AIIDA_WARN_v3: 0
-      run: pytest -n auto -m 'presto' tests/
+      run: pytest -n auto --broker-backend zmq -m 'presto' tests/
 
 
   verdi:
@@ -213,7 +207,7 @@ jobs:
     - name: Install aiida-core
       uses: ./.github/actions/install-aiida-core
       with:
-        python-version: '3.9'
+        python-version: '3.10'
         from-lock: 'true'
         extras: tests
 

.github/workflows/docs-build.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Install aiida-core and docs deps
       uses: ./.github/actions/install-aiida-core
       with:
-        python-version: '3.9'
+        python-version: '3.10'
         extras: docs,tutorials,tests,rest,atomic_tools
         from-lock: 'false'
 

.github/workflows/release.yml

@@ -1,6 +1,6 @@
 name: release
 
-# Automate plubishing to PyPI and TestPyPI. When pushing a release tag vX.Y.Z,
+# Automate publishing to PyPI and TestPyPI. When pushing a release tag vX.Y.Z,
 # the workflow will publish to PyPI. When the 'test-release' label is active on
 # a PR, the workflow will publish to TestPyPI on each update. In both cases it
 # is checked if pre-commit and the tests pass. When a release tag is pushed it
@@ -30,10 +30,10 @@ jobs:
     - name: Checkout repo
       uses: actions/checkout@v6
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v6
       with:
-        python-version: '3.9'
+        python-version: '3.10'
 
     - name: Check tag
       run: python .github/workflows/check_release_tag.py $GITHUB_REF
@@ -63,21 +63,14 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
 
-    services:
-      rabbitmq:
-        image: rabbitmq:3.8.14-management
-        ports:
-        - 5672:5672
-        - 15672:15672
-
     steps:
     - uses: actions/checkout@v6
 
     - name: Install aiida-core
       uses: ./.github/actions/install-aiida-core
 
     - name: Run sub-set of test suite
-      run: pytest -s -m requires_rmq --db-backend=sqlite tests/
+      run: pytest -s -m requires_broker --broker-backend zmq --db-backend=sqlite tests/
 
   publish-pypi:
     if: startsWith(github.ref, 'refs/tags/')
@@ -95,10 +88,10 @@ jobs:
     steps:
     - name: Checkout source
       uses: actions/checkout@v6
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v6
       with:
-        python-version: '3.9'
+        python-version: '3.10'
     - name: Install flit
       run: pip install flit~=3.4
     - name: Build
@@ -123,10 +116,10 @@ jobs:
     steps:
     - name: Checkout source
       uses: actions/checkout@v6
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v6
       with:
-        python-version: '3.9'
+        python-version: '3.10'
     - name: Install flit
       run: pip install flit~=3.4
     - name: Build

.github/workflows/test-install.yml

@@ -1,6 +1,7 @@
 name: test-install
 
 on:
+  workflow_dispatch:
   pull_request:
     paths:
     - environment.yml
@@ -84,10 +85,10 @@ jobs:
     steps:
     - uses: actions/checkout@v6
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v6
       with:
-        python-version: '3.9'
+        python-version: '3.10'
 
     - name: Pip install
       id: pip_install
@@ -111,7 +112,7 @@ jobs:
       fail-fast: false
       matrix:
 
-        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
         # Not being able to install with conda on a specific Python version is
         # not sufficient to fail the run, but something we want to be aware of.
@@ -165,7 +166,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
     services:
       postgres:

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## Unreleased
+
+### Behavior changes
+
+**`verdi presto`: auto-starts the daemon** ([#7351](https://github.com/aiidateam/aiida-core/pull/7351))
+
+`verdi presto` now starts the daemon automatically after creating the profile, when a broker is configured.
+Scripts that previously ran `verdi daemon start` after `verdi presto` continue to work; the daemon-start invocation is idempotent.
+
 ## v2.8.0 - 2026-03-16
 
 This release brings important improvements to the `BaseRestartWorkChain`, the engine, stashing, typing coverage, and dependency updates.

docs/source/howto/include/scripts/performance_benchmark_base.py

@@ -50,7 +50,7 @@ def main(code, number, daemon):
     computer_created = False
     code_created = False
 
-    echo.echo(f'Running on profile {get_manager().get_profile().name}')
+    echo.echo(f'Running on profile `{get_manager().get_profile().name}`')
 
     if not code:
         label = f'benchmark-{uuid.uuid4().hex[:8]}'

docs/source/howto/index.rst

@@ -22,6 +22,7 @@ How-To Guides
    archive_profile
    visualising_graphs
    installation
+   tune_performance
    plugins_develop
    cookbook
    tricks_real_world_runs

docs/source/howto/installation.rst

@@ -158,7 +158,7 @@ To view all configuration options set for the current profile:
     logging.paramiko_loglevel              default   WARNING
     logging.plumpy_loglevel                default   WARNING
     logging.sqlalchemy_loglevel            default   WARNING
-    rmq.task_timeout                       default   10
+    broker.task_timeout                    default   10
     runner.poll.interval                   profile   50
     transport.task_maximum_attempts        global    6
     transport.task_retry_initial_interval  default   20
@@ -367,141 +367,6 @@ How to do this, is operating system specific.
 For Ubuntu, here is `a template for the service file <https://github.com/marvel-nccr/ansible-role-aiida/blob/c709088dff74d1e1ae4d8379e740aba35fb2ef97/templates/aiida-daemon%40.service>`_ and `ansible instructions to install the service <https://github.com/marvel-nccr/ansible-role-aiida/blob/c709088dff74d1e1ae4d8379e740aba35fb2ef97/tasks/aiida-daemon.yml>`_.
 
 
-.. _how-to:installation:performance:
-
-Tuning performance
-==================
-
-AiiDA supports running hundreds of thousands of calculations and graphs with millions of nodes.
-However, optimal performance at that scale can require tweaking the AiiDA configuration to balance the CPU and disk load.
-
-Below, we share a few practical tips for assessing and tuning AiiDA performance.
-Further in-depth information is available in the dedicated :ref:`topic on performance<topics:performance>`.
-
-.. dropdown:: Benchmark workflow engine performance
-
-    Download the :download:`benchmark script <include/scripts/performance_benchmark_base.py>` :fa:`download`, and run it in your AiiDA environment.
-
-    .. code:: console
-
-        sph@citadel:~/$ python performance_benchmark_base.py -n 100
-            Success: Created and configured temporary `Computer` benchmark-e73b8647 for localhost.
-            Success: Created temporary `Code` bash for localhost.
-            Running 100 calculations.  [####################################]  100%
-            Success: All calculations finished successfully.
-            Elapsed time: 24.90 seconds.
-            Cleaning up...
-            12/19/2022 10:57:43 AM <12625> aiida.delete: [REPORT] 400 Node(s) marked for deletion
-            12/19/2022 10:57:43 AM <12625> aiida.delete: [REPORT] Starting node deletion...
-            12/19/2022 10:57:43 AM <12625> aiida.delete: [REPORT] Deletion of nodes completed.
-            Success: Deleted all calculations.
-            Success: Deleted the created code bash@benchmark-e73b8647.
-            Success: Deleted the created computer benchmark-e73b8647.
-            Performance: 0.25 s / process
-
-    The output above was generated on an AMD Ryzen 5 3600 6-Core processor (3.6 GHz, 4.2 GHz turbo boost) using AiiDA v2.2.0, and RabbitMQ and PostgreSQL running on the same machine.
-    Here, 100 ``ArithmeticAddCalculation`` processes completed in ~25s, corresponding to 0.25 seconds per process.
-
-    If you observe a significantly higher runtime, you may want to check whether any relevant component (CPU, disk, postgresql, rabbitmq) is congested.
-
-.. dropdown:: Increase the number of daemon workers
-
-    By default, the AiiDA daemon only uses a single worker, i.e. a single operating system process.
-    If ``verdi daemon status`` shows the daemon worker constantly at high CPU usage, you can use ``verdi daemon incr X`` to add ``X`` parallel daemon workers.
-
-    Keep in mind that other processes need to run on your computer (e.g. rabbitmq, the PostgreSQL database, ...), i.e. it's a good idea to stop increasing the number of workers before you reach the number of cores of your CPU.
-
-    To make the change permanent, set
-    ::
-
-        verdi config set daemon.default_workers 4
-
-.. dropdown:: Increase the number of daemon worker slots
-
-    Each daemon worker accepts only a limited number of tasks at a time.
-    If ``verdi daemon status`` constantly warns about a high percentage of the available daemon worker slots being used, you can increase the number of tasks handled by each daemon worker (thus increasing the workload per worker).
-    Increasing it to 1000 should typically work.
-
-    Set the corresponding config variable and restart the daemon
-    ::
-
-        verdi config set daemon.worker_process_slots 1000
-
-.. dropdown:: Prevent your operating system from indexing the file repository.
-
-    Many Linux distributions include the ``locate`` command to quickly find files and folders, and run a daily cron job ``updatedb.mlocate`` to create the corresponding index.
-    A large file repository can take a long time to index, up to the point where the hard drive is constantly indexing.
-
-    In order to exclude the repository folder from indexing, add its path to the ``PRUNEPATH`` variable in the ``/etc/updatedb.conf`` configuration file (use ``sudo``).
-
-.. dropdown:: Move the Postgresql database to a fast disk (SSD), ideally on a large partition.
-
-    1. Stop the AiiDA daemon and :ref:`back up your database <how-to:installation:backup>`.
-
-    2. Find the data directory of your postgres installation (something like ``/var/lib/postgresql/9.6/main``, ``/scratch/postgres/9.6/main``, ...).
-
-        The best way is to become the postgres UNIX user and enter the postgres shell::
-
-            psql
-            SHOW data_directory;
-            \q
-
-        If you are unable to enter the postgres shell, try looking for the ``data_directory`` variable in a file ``/etc/postgresql/9.6/main/postgresql.conf`` or similar.
-
-    3. Stop the postgres database service::
-
-        service postgresql stop
-
-    4. Copy all files and folders from the postgres ``data_directory`` to the new location::
-
-        cp -a SOURCE_DIRECTORY DESTINATION_DIRECTORY
-
-        .. note:: Flag ``-a`` will create a directory within ``DESTINATION_DIRECTORY``, e.g.::
-
-        cp -a OLD_DIR/main/ NEW_DIR/
-
-        creates ``NEW_DIR/main``.
-        It will also keep the file permissions (necessary).
-
-        The file permissions of the new and old directory need to be identical (including subdirectories).
-        In particular, the owner and group should be both ``postgres`` (except for symbolic links in ``server.crt`` and ``server.key`` that may or may not be present).
-
-        .. note::
-
-            If the permissions of these links need to be changed, use the ``-h`` option of ``chown`` to avoid changing the permissions of the destination of the links.
-            In case you have changed the permission of the links destination by mistake, they should typically be (beware that this might depend on your actual distribution!)::
-
-            -rw-r--r-- 1 root root 989 Mar  1  2012 /etc/ssl/certs/ssl-cert-snakeoil.pem
-            -rw-r----- 1 root ssl-cert 1704 Mar  1  2012 /etc/ssl/private/ssl-cert-snakeoil.key
-
-    5. Point the ``data_directory`` variable in your postgres configuration file (e.g. ``/etc/postgresql/9.6/main/postgresql.conf``) to the new directory.
-
-    6. Restart the database daemon::
-
-        service postgresql start
-
-    Finally, check that the data directory has indeed changed::
-
-        psql
-        SHOW data_directory;
-        \q
-
-    and try a simple AiiDA query with the new database.
-    If everything went fine, you can delete the old database location.
-
-If you're still encountering performance issues, the following tips can help with pinpointing performance bottlenecks.
-
-.. dropdown:: Analyze the RabbitMQ message rate
-
-    If you're observing slow performance of the AiiDA engine, the `RabbitMQ management plugin <https://www.rabbitmq.com/management.html>`_ provides an intuitive dashboard that lets you monitor the message rate and check on what the AiiDA engine is up to.
-
-    Enable the management plugin via something like::
-
-        sudo rabbitmq-plugins enable rabbitmq_management
-
-    Then, navigate to http://localhost:15672/ and log in with ``guest``/``guest``.
-
-
 .. _how-to:installation:update:
 
 Updating your installation