Merge pull request #4 from realtyem/psycopg3-adjust-settings

Author: clokep

.ci/scripts/auditwheel_wrapper.py

@@ -25,7 +25,6 @@
 import argparse
 import os
 import subprocess
-from typing import Optional
 from zipfile import ZipFile
 
 from packaging.tags import Tag
@@ -80,7 +79,7 @@ def cpython(wheel_file: str, name: str, version: Version, tag: Tag) -> str:
     return new_wheel_file
 
 
-def main(wheel_file: str, dest_dir: str, archs: Optional[str]) -> None:
+def main(wheel_file: str, dest_dir: str, archs: str | None) -> None:
     """Entry point"""
 
     # Parse the wheel file name into its parts. Note that `parse_wheel_filename`

.ci/scripts/calculate_jobs.py

@@ -35,64 +35,64 @@ def set_output(key: str, value: str):
 
 # First calculate the various trial jobs.
 #
-# For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.9 right now)
+# For PRs, we only run each type of test with the oldest and newest Python
+# version that's supported. The oldest version ensures we don't accidentally
+# introduce syntax or code that's too new, and the newest ensures we don't use
+# code that's been dropped in the latest supported Python version.
 
 trial_sqlite_tests = [
     {
-        "python-version": "3.9",
+        "python-version": "3.10",
         "database": "sqlite",
         "extras": "all",
-    }
+    },
+    {
+        "python-version": "3.14",
+        "database": "sqlite",
+        "extras": "all",
+    },
 ]
 
 if not IS_PR:
+    # Otherwise, check all supported Python versions.
+    #
+    # Avoiding running all of these versions on every PR saves on CI time.
     trial_sqlite_tests.extend(
         {
             "python-version": version,
             "database": "sqlite",
             "extras": "all",
         }
-        for version in ("3.10", "3.11", "3.12", "3.13")
+        for version in ("3.11", "3.12", "3.13")
     )
 
-# Run with both psycopg2 and psycopg.
+# Only test postgres against the earliest and latest Python versions that we
+# support in order to save on CI time.
 trial_postgres_tests = [
     {
-        "python-version": "3.9",
+        "python-version": "3.10",
         "database": "postgres",
-        "postgres-version": "13",
+        "postgres-version": "14",
         "extras": "all",
     },
     {
-        "python-version": "3.9",
+        "python-version": "3.10",
         "database": "psycopg",
-        "postgres-version": "13",
+        "postgres-version": "14",
+        "extras": "all",
+    },
+    {
+        "python-version": "3.14",
+        "database": "postgres",
+        "postgres-version": "17",
         "extras": "all",
     },
 ]
 
-if not IS_PR:
-    trial_postgres_tests.extend(
-        [
-            {
-                "python-version": "3.13",
-                "database": "postgres",
-                "postgres-version": "17",
-                "extras": "all",
-            },
-            {
-                "python-version": "3.13",
-                "database": "psycopg",
-                "postgres-version": "17",
-                "extras": "all",
-            },
-        ]
-    )
-
+# Ensure that Synapse passes unit tests even with no extra dependencies installed.
 trial_no_extra_tests = [
     {
-        "python-version": "3.9",
+        "python-version": "3.10",
         "database": "sqlite",
         "extras": "",
     }
@@ -114,24 +114,24 @@ def set_output(key: str, value: str):
 
 # First calculate the various sytest jobs.
 #
-# For each type of test we only run on bullseye on PRs
+# For each type of test we only run on bookworm on PRs
 
 
 sytest_tests = [
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "bookworm",
     },
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "bookworm",
         "postgres": "postgres",
     },
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "bookworm",
         "postgres": "multi-postgres",
         "workers": "workers",
     },
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "bookworm",
         "postgres": "multi-postgres",
         "workers": "workers",
         "reactor": "asyncio",
@@ -142,11 +142,11 @@ def set_output(key: str, value: str):
     sytest_tests.extend(
         [
             {
-                "sytest-tag": "bullseye",
+                "sytest-tag": "bookworm",
                 "reactor": "asyncio",
             },
             {
-                "sytest-tag": "bullseye",
+                "sytest-tag": "bookworm",
                 "postgres": "postgres",
                 "reactor": "asyncio",
             },

.ci/scripts/prepare_old_deps.sh

@@ -16,20 +16,23 @@ export VIRTUALENV_NO_DOWNLOAD=1
 # to select the lowest possible versions, rather than resorting to this sed script.
 
 # Patch the project definitions in-place:
-# - Replace all lower and tilde bounds with exact bounds
-# - Replace all caret bounds---but not the one that defines the supported Python version!
-# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - `-E` use extended regex syntax.
+# - Don't modify the line that defines required Python versions.
+# - Replace all lower and tilde bounds with exact bounds.
+# - Replace all caret bounds with exact bounds.
+# - Delete all lines referring to psycopg2 - so no testing of postgres support.
 # - Use pyopenssl 17.0, which is the oldest version that works with
 #   a `cryptography` compiled against OpenSSL 1.1.
 # - Omit systemd: we're not logging to journal here.
 
-sed -i \
-   -e "s/[~>]=/==/g" \
-   -e '/^python = "^/!s/\^/==/g' \
-   -e "/psycopg2/d" \
-   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
-   -e '/systemd/d' \
-   pyproject.toml
+sed -i -E '
+  /^\s*requires-python\s*=/b
+  s/[~>]=/==/g
+  s/\^/==/g
+  /psycopg2/d
+  s/pyOpenSSL\s*==\s*16\.0\.0"/pyOpenSSL==17.0.0"/
+  /systemd/d
+' pyproject.toml
 
 echo "::group::Patched pyproject.toml"
 cat pyproject.toml

.git-blame-ignore-revs

@@ -26,3 +26,8 @@ c4268e3da64f1abb5b31deaeb5769adb6510c0a7
 # Update black to 23.1.0 (https://github.com/matrix-org/synapse/pull/15103)
 9bb2eac71962970d02842bca441f4bcdbbf93a11
 
+# Use type hinting generics in standard collections (https://github.com/element-hq/synapse/pull/19046)
+fc244bb592aa481faf28214a2e2ce3bb4e95d990
+
+# Write union types as X | Y where possible (https://github.com/element-hq/synapse/pull/19111)
+fcac7e0282b074d4bd3414d1c9c181e9701875d9

.github/workflows/docker.yml

@@ -75,7 +75,7 @@ jobs:
           touch "${{ runner.temp }}/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: digests-${{ matrix.suffix }}
           path: ${{ runner.temp }}/digests/*
@@ -95,7 +95,7 @@ jobs:
       - build
     steps:
       - name: Download digests
-        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
         with:
           path: ${{ runner.temp }}/digests
           pattern: digests-*
@@ -120,10 +120,10 @@ jobs:
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
 
       - name: Install Cosign
-        uses: sigstore/cosign-installer@d7543c93d881b35a8faa02e8e3605f69b7a1ce62 # v3.10.0
+        uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0
 
       - name: Calculate docker image tag
-        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0
         with:
           images: ${{ matrix.repository }}
           flavor: |

.github/workflows/docs-pr.yaml

@@ -39,7 +39,7 @@ jobs:
           cp book/welcome_and_overview.html book/index.html
 
       - name: Upload Artifact
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         with:
           name: book
           path: book

.github/workflows/fix_lint.yaml

@@ -47,6 +47,6 @@ jobs:
       - run: cargo fmt
         continue-on-error: true
 
-      - uses: stefanzweifel/git-auto-commit-action@778341af668090896ca464160c2def5d1d1a3eb0 # v6.0.1
+      - uses: stefanzweifel/git-auto-commit-action@28e16e81777b558cc906c8750092100bbb34c5e3 # v7.0.0
         with:
           commit_message: "Attempt to fix linting"

.github/workflows/latest_deps.yml

@@ -139,9 +139,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: bullseye
+          - sytest-tag: bookworm
 
-          - sytest-tag: bullseye
+          - sytest-tag: bookworm
             postgres: postgres
             workers: workers
             redis: redis
@@ -173,7 +173,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})

.github/workflows/push_complement_image.yml

@@ -55,7 +55,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Work out labels for complement image
         id: meta
-        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0
         with:
           images: ghcr.io/${{ github.repository }}/complement-synapse
           tags: |

.github/workflows/release-artifacts.yml

@@ -101,7 +101,7 @@ jobs:
           echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT"
 
       - name: Upload debs as artifacts
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         with:
           name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
           path: debs/*
@@ -114,8 +114,8 @@ jobs:
         os:
           - ubuntu-24.04
           - ubuntu-24.04-arm
-          - macos-13 # This uses x86-64
           - macos-14 # This uses arm64
+          - macos-15-intel # This uses x86-64
         # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
         # It is not read by the rest of the workflow.
         is_pr:
@@ -124,7 +124,7 @@ jobs:
         exclude:
           # Don't build macos wheels on PR CI.
           - is_pr: true
-            os: "macos-13"
+            os: "macos-15-intel"
           - is_pr: true
             os: "macos-14"
           # Don't build aarch64 wheels on PR CI.
@@ -141,20 +141,25 @@ jobs:
           python-version: "3.x"
 
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==3.0.0
+        run: python -m pip install cibuildwheel==3.2.1
 
       - name: Only build a single wheel on PR
         if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp39-manylinux_*"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp310-manylinux_*"" >> $GITHUB_ENV
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse
         env:
-          # Skip testing for platforms which various libraries don't have wheels
-          # for, and so need extra build deps.
-          CIBW_TEST_SKIP: pp3*-* *i686* *musl*
+          # The platforms that we build for are determined by the
+          # `tool.cibuildwheel.skip` option in `pyproject.toml`.
 
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+          # We skip testing wheels for the following platforms in CI:
+          #
+          # pp3*-* (PyPy wheels) broke in CI (TODO: investigate).
+          # musl: (TODO: investigate).
+          CIBW_TEST_SKIP: pp3*-* *musl*
+
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         with:
           name: Wheel-${{ matrix.os }}
           path: ./wheelhouse/*.whl
@@ -175,7 +180,7 @@ jobs:
       - name: Build sdist
         run: python -m build --sdist
 
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         with:
           name: Sdist
           path: dist/*.tar.gz
@@ -191,7 +196,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download all workflow run artifacts
-        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
       - name: Build a tarball for the debs
         # We need to merge all the debs uploads into one folder, then compress
         # that.
@@ -200,16 +205,11 @@ jobs:
           mv debs*/* debs/
           tar -cvJf debs.tar.xz debs
       - name: Attach to release
-        # Pinned to work around https://github.com/softprops/action-gh-release/issues/445
-        uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # v0.1.15
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          files: |
-            Sdist/*
-            Wheel*/*
-            debs.tar.xz
-          # if it's not already published, keep the release as a draft.
-          draft: true
-          # mark it as a prerelease if the tag contains 'rc'.
-          prerelease: ${{ contains(github.ref, 'rc') }}
+        run: |
+          gh release upload "${{ github.ref_name }}" \
+            Sdist/* \
+            Wheel*/* \
+            debs.tar.xz \
+            --repo ${{ github.repository }}