Set up infrastructure for qiskit-tutorials migration (Qiskit#10443) (Qiskit#10625)

mergify[bot] · Eric-Arellano · web-flow · commit 3d22ba143731 · 2023-08-14T21:07:32.000Z
* Add infrastructure for building tutorials This first commit is a rebase of Eric's initial PR as of db1ce62 onto `main`, fixing up some changes caused by the CI infrastructure changing a bit since the PR was first opened. Co-authored-by: Jake Lishman <jake.lishman@ibm.com> * Harden tutorials Azure job This moves much of the fetch- and process-related code into separate scripts that assert far more about the directory structure, and fail if they do not match the assumptions. We don't want to accidentally get out-of-sync while we're changing things and end up with a tutorials job that isn't really doing its job without us noticing. The tutorials-fetching script can now also be re-used in a separate GitHub Actions workflow that will handle the full tutorials-included documentation build and deployment in the future. The notebook-convertion step is moved into Python space, using `nbconvert` as a library in order to parallelise the build process for the tutorials, and to allow CI and developers calling `tox` directly to specify the output directories for the built tutorials. * Retarget tutorial-conversion utility as executor This reorganises the tutorial "conversion" utility to make it clearer that what it's actually doing is just executing the tutorials. The script itself is changed to default to editing the files inplace, while the `tox` job is updated to write the files into a special directory, making it easier to clean up a dirty build directory and making it so subsequent local executions will not pick up the converted files. * Allow configuration of tutorials execution There was a worry that not being able to configure these would make it more unpleasant to use `tox` for the jobs locally. --------- Co-authored-by: Jake Lishman <jake.lishman@ibm.com> (cherry picked from commit df2ddca) Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com>
diff --git a/.azure/docs-linux.yml b/.azure/docs-linux.yml
@@ -24,7 +24,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install -U "tox<4.4.0"
           sudo apt-get update
-          sudo apt-get install -y graphviz
+          sudo apt-get install -y graphviz pandoc
         displayName: 'Install dependencies'
 
       - bash: |
diff --git a/.azure/tutorials-linux.yml b/.azure/tutorials-linux.yml
@@ -8,9 +8,7 @@ jobs:
     pool: {vmImage: 'ubuntu-latest'}
 
     variables:
-      QISKIT_SUPPRESS_PACKAGING_WARNINGS: Y
       PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
-      QISKIT_CELL_TIMEOUT: 300
 
     steps:
       - task: UsePythonVersion@0
@@ -20,40 +18,33 @@ jobs:
 
       - bash: |
           set -e
-          git clone https://github.com/Qiskit/qiskit-tutorials --depth=1
           python -m pip install --upgrade pip setuptools wheel
-          python -m pip install -U \
-            -c constraints.txt \
-            -r requirements.txt \
-            -r requirements-dev.txt \
-            -r requirements-optional.txt \
-            -r requirements-tutorials.txt \
-            -e .
+          python -m pip install -U "tox<4.4.0"
           sudo apt-get update
           sudo apt-get install -y graphviz pandoc
-          pip check
         displayName: 'Install dependencies'
-        env:
-          SETUPTOOLS_ENABLE_FEATURES: "legacy-editable"
 
-      - bash: |
-          set -e
-          cd qiskit-tutorials
-          sphinx-build -b html . _build/html
+      - bash: tools/prepare_tutorials.bash algorithms circuits circuits_advanced operators
+        displayName: 'Download current tutorials'
+
+      - bash: tox -e tutorials
+        displayName: "Execute tutorials"
         env:
-          QISKIT_PARALLEL: False
+          QISKIT_CELL_TIMEOUT: 300
 
       - task: ArchiveFiles@2
         inputs:
-          rootFolderOrFile: 'qiskit-tutorials/_build/html'
+          rootFolderOrFile: 'executed_tutorials'
           archiveType: tar
-          archiveFile: '$(Build.ArtifactStagingDirectory)/html_tutorials.tar.gz'
+          archiveFile: '$(Build.ArtifactStagingDirectory)/executed_tutorials.tar.gz'
           verbose: true
+        condition: succeededOrFailed()
 
       - task: PublishBuildArtifacts@1
-        displayName: 'Publish docs'
+        displayName: 'Publish updated tutorials'
         inputs:
           pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-          artifactName: 'html_tutorials'
+          artifactName: 'executed_tutorials'
           Parallel: true
           ParallelCount: 8
+        condition: succeededOrFailed()
diff --git a/.gitignore b/.gitignore
@@ -143,6 +143,7 @@ qiskit/transpiler/passes/**/cython/**/*.cpp
 qiskit/quantum_info/states/cython/*.cpp
 
 docs/stubs/*
+executed_tutorials/
 
 # Notebook testing images
 test/visual/mpl/circuit/circuit_results/*.png
diff --git a/docs/conf.py b/docs/conf.py
@@ -14,12 +14,12 @@
 
 """Sphinx documentation builder."""
 
-# -- General configuration ---------------------------------------------------
 import datetime
 import doctest
+import os
 
 project = "Qiskit"
-copyright = f"2017-{datetime.date.today().year}, Qiskit Development Team"  # pylint: disable=redefined-builtin
+project_copyright = f"2017-{datetime.date.today().year}, Qiskit Development Team"
 author = "Qiskit Development Team"
 
 # The short X.Y version
@@ -39,7 +39,8 @@
     "reno.sphinxext",
     "sphinx_design",
     "matplotlib.sphinxext.plot_directive",
-    "sphinx.ext.doctest",
+    "qiskit_sphinx_theme",
+    "nbsphinx",
 ]
 
 templates_path = ["_templates"]
@@ -77,7 +78,9 @@
     "matplotlib": ("https://matplotlib.org/stable/", None),
 }
 
-# -- Options for HTML output -------------------------------------------------
+# ----------------------------------------------------------------------------------
+# HTML theme
+# ----------------------------------------------------------------------------------
 
 html_theme = "qiskit_sphinx_theme"
 html_last_updated_fmt = "%Y/%m/%d"
@@ -88,8 +91,9 @@
     "style_external_links": True,
 }
 
-
-# -- Options for Autosummary and Autodoc -------------------------------------
+# ----------------------------------------------------------------------------------
+# Autodoc
+# ----------------------------------------------------------------------------------
 
 # Note that setting autodoc defaults here may not have as much of an effect as you may expect; any
 # documentation created by autosummary uses a template file (in autosummary in the templates path),
@@ -131,7 +135,9 @@
 napoleon_numpy_docstring = False
 
 
-# -- Options for Doctest --------------------------------------------------------
+# ----------------------------------------------------------------------------------
+# Doctest
+# ----------------------------------------------------------------------------------
 
 doctest_default_flags = (
     doctest.ELLIPSIS
@@ -145,3 +151,27 @@
 # >> code
 # output
 doctest_test_doctest_blocks = ""
+
+# ----------------------------------------------------------------------------------
+# Nbsphinx
+# ----------------------------------------------------------------------------------
+
+nbsphinx_timeout = int(os.getenv("QISKIT_CELL_TIMEOUT", "300"))
+nbsphinx_execute = os.getenv("QISKIT_DOCS_BUILD_TUTORIALS", "never")
+nbsphinx_widgets_path = ""
+nbsphinx_thumbnails = {"**": "_static/images/logo.png"}
+
+nbsphinx_prolog = """
+{% set docname = env.doc2path(env.docname, base=None) %}
+
+.. only:: html
+
+    .. role:: raw-html(raw)
+        :format: html
+
+    .. note::
+        This page was generated from `{{ docname }}`__.
+
+    __ https://github.com/Qiskit/qiskit-terra/blob/main/{{ docname }}
+
+"""
diff --git a/docs/index.rst b/docs/index.rst
@@ -7,6 +7,7 @@ Qiskit Terra documentation
   :hidden:
 
   How-to Guides <how_to/index>
+  tutorials
   API References <apidocs/terra>
   Explanation <explanation/index>
   Migration Guides <migration_guides/index>
diff --git a/docs/tutorials.rst b/docs/tutorials.rst
@@ -0,0 +1,37 @@
+.. _tutorials:
+
+=========
+Tutorials
+=========
+
+Quantum circuits
+================
+
+.. nbgallery::
+   :glob:
+
+   tutorials/circuits/*
+
+Advanced circuits
+=================
+
+.. nbgallery::
+   :glob:
+
+   tutorials/circuits_advanced/*
+
+Algorithms
+==========
+
+.. nbgallery::
+   :glob:
+
+   tutorials/algorithms/*
+
+Operators
+=========
+
+.. nbgallery::
+   :glob:
+
+   tutorials/operators/*
diff --git a/docs/tutorials/algorithms/placeholder.ipynb b/docs/tutorials/algorithms/placeholder.ipynb
@@ -0,0 +1,34 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Placeholder",
+                "\n",
+                "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration."
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.9.16"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4
+}
diff --git a/docs/tutorials/circuits/placeholder.ipynb b/docs/tutorials/circuits/placeholder.ipynb
@@ -0,0 +1,34 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Placeholder",
+                "\n",
+                "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration."
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.9.16"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4
+}
diff --git a/docs/tutorials/circuits_advanced/placeholder.ipynb b/docs/tutorials/circuits_advanced/placeholder.ipynb
@@ -0,0 +1,34 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Placeholder",
+                "\n",
+                "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration."
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.9.16"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4
+}
diff --git a/docs/tutorials/operators/placeholder.ipynb b/docs/tutorials/operators/placeholder.ipynb
@@ -0,0 +1,34 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Placeholder",
+                "\n",
+                "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration."
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.9.16"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4
+}
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -31,9 +31,10 @@ ddt>=1.2.0,!=1.4.0,!=1.4.3
 # components of Terra use some of its optional dependencies in order to document
 # themselves.  These are the requirements that are _only_ required for the docs
 # build, and are not used by Terra itself.
-
+Sphinx>=6.0
+qiskit-sphinx-theme~=1.13.0
+sphinx-design>=0.2.0
+nbsphinx~=0.9.2
+nbconvert~=7.7.1
 # TODO: switch to stable release when 4.1 is released
 reno @ git+https://github.com/openstack/reno.git@81587f616f17904336cdc431e25c42b46cd75b8f
-Sphinx>=5.0
-qiskit-sphinx-theme~=1.11.0
-sphinx-design>=0.2.0
diff --git a/requirements-tutorials.txt b/requirements-tutorials.txt
@@ -2,7 +2,7 @@
 # This may also include some requirements that are only in `requirements-dev.txt`, since those
 # aren't runtime dependencies or optionals of Terra.
 
-networkx>=2.2
+networkx>=2.3
 jupyter
 Sphinx
 nbsphinx
diff --git a/tools/execute_tutorials.py b/tools/execute_tutorials.py
diff --git a/tools/prepare_tutorials.bash b/tools/prepare_tutorials.bash
diff --git a/tox.ini b/tox.ini
