Update path rewriting to support setuptools v64's PEP660 editable install mode

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Ensure path rewriting works when using setuptools v64's new PEP660-based editable install mode ([#1357](https://github.com/heroku/heroku-buildpack-python/pull/1357)).
 - Display an EOL warning for Python 3.4, 3.5 and 3.6 ([#1356](https://github.com/heroku/heroku-buildpack-python/pull/1356)).
 - Improve the EOL warning for Python 2.7 ([#1356](https://github.com/heroku/heroku-buildpack-python/pull/1356)).
 - Display a deprecation warning for PyPy support ([#1356](https://github.com/heroku/heroku-buildpack-python/pull/1356)).

bin/compile

@@ -304,14 +304,14 @@ if [[ \$HOME != "/app" ]]; then
 fi
 EOT
 
-# At runtime, rewrite paths in editable package .egg-link and .pth files from the build time paths
+# At runtime, rewrite paths in editable package .egg-link, .pth and finder files from the build time paths
 # (such as `/tmp/build_<hash>`) back to `/app`. This is not done during the build itself, since later
 # buildpacks still need the build time paths. The `/lib*/*/` glob is to ensure it matches against both:
 # - lib/python3.NN/site-packages/ (CPython)
 # - lib-python/3/site-packages/ (PyPy)
 if [[ "${BUILD_DIR}" != "/app" ]]; then
   cat <<EOT >> "$PROFILE_PATH"
-find .heroku/python/lib*/*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' \) -exec sed -i -e 's#${BUILD_DIR}#/app#' {} \+
+find .heroku/python/lib*/*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) -exec sed -i -e 's#${BUILD_DIR}#/app#' {} \+
 EOT
 fi
 

spec/fixtures/requirements_editable/bin/compile

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This file is run by the inline buildpack, and tests that editable reqirements are
+# This file is run by the inline buildpack, and tests that editable requirements are
 # usable by buildpacks that run after the Python buildpack during the build.
 
 set -euo pipefail

spec/fixtures/requirements_editable/bin/test-entrypoints

@@ -2,12 +2,15 @@
 
 set -euo pipefail
 
-# List the filenames and contents of all .egg-link and .pth files in site-packages.
-find .heroku/python/lib*/*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' \) | sort | xargs -exec tail -n +1
+# List the filenames and contents of all .egg-link, .pth, and finder files in site-packages.
+find .heroku/python/lib*/*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) | sort | xargs -exec tail -n +1
 echo
 
-echo -n "Running entrypoint for the local package: "
-local_package
+echo -n "Running entrypoint for the pyproject.toml-based local package: "
+local_package_pyproject_toml
+
+echo -n "Running entrypoint for the setup.py-based local package: "
+local_package_setup_py
 
 echo -n "Running entrypoint for the VCS package: "
 gunicorn --version

spec/fixtures/requirements_editable/packages/local_package_pyproject_toml/local_package_pyproject_toml/__init__.py

@@ -0,0 +1,2 @@
+def hello():
+    print("Hello pyproject.toml!")

spec/fixtures/requirements_editable/packages/local_package_pyproject_toml/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "local_package_pyproject_toml"
+version = "0.0.1"
+
+[project.scripts]
+local_package_pyproject_toml = "local_package_pyproject_toml:hello"

spec/fixtures/requirements_editable/packages/local_package_setup_py/local_package_setup_py/__init__.py

@@ -0,0 +1,2 @@
+def hello():
+    print("Hello setup.py!")

spec/fixtures/requirements_editable/packages/local_package_setup_py/setup.cfg

@@ -0,0 +1,10 @@
+[metadata]
+name = local_package_setup_py
+version = 0.0.1
+
+[options]
+packages = local_package_setup_py
+
+[options.entry_points]
+console_scripts =
+    local_package_setup_py = local_package_setup_py:hello

spec/fixtures/requirements_editable/requirements.txt

@@ -1,2 +1,5 @@
--e ./local_package
+# The packages have to be nested under `packages/` to work around:
+# https://github.com/pypa/setuptools/issues/3535
+-e ./packages/local_package_pyproject_toml
+-e ./packages/local_package_setup_py
 -e git+https://github.com/benoitc/[email protected]#egg=gunicorn

spec/hatchet/pip_spec.rb

@@ -91,99 +91,139 @@
     let(:buildpacks) { [:default, 'heroku-community/inline'] }
     let(:app) { Hatchet::Runner.new('spec/fixtures/requirements_editable', buildpacks: buildpacks) }
 
-    it 'rewrites .pth and .egg-link paths correctly for hooks, later buildpacks, runtime and cached builds' do
+    it 'rewrites .pth, .egg-link and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do
       app.deploy do |app|
-        expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
-          remote:        Successfully installed gunicorn-20.1.0 local-package-0.0.1
+        expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE))
+          remote:        Successfully installed gunicorn-20.1.0 local-package-pyproject-toml-0.0.1 local-package-setup-py-0.0.1
           remote: -----> Running post-compile hook
           remote: ==> .heroku/python/lib/python.*/site-packages/distutils-precedence.pth <==
           remote: .*
           remote: 
           remote: ==> .heroku/python/lib/python.*/site-packages/easy-install.pth <==
-          remote: /tmp/build_.*/local_package
+          remote: /tmp/build_.*/packages/local_package_setup_py
           remote: /app/.heroku/src/gunicorn
           remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable___local_package_pyproject_toml_0_0_1_finder.py <==
+          remote: .*
+          remote: MAPPING = \{'local_package_pyproject_toml': '/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'\}
+          remote: .*
+          remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable__.local_package_pyproject_toml-0.0.1.pth <==
+          remote: import __editable___.*
           remote: ==> .heroku/python/lib/python.*/site-packages/gunicorn.egg-link <==
           remote: /app/.heroku/src/gunicorn
           remote: .
-          remote: ==> .heroku/python/lib/python.*/site-packages/local-package.egg-link <==
-          remote: /tmp/build_.*/local_package
+          remote: ==> .heroku/python/lib/python.*/site-packages/local-package-setup-py.egg-link <==
+          remote: /tmp/build_.*/packages/local_package_setup_py
           remote: .
-          remote: Running entrypoint for the local package: Hello!
+          remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
+          remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
           remote: ==> .heroku/python/lib/python.*/site-packages/distutils-precedence.pth <==
           remote: .*
           remote: 
           remote: ==> .heroku/python/lib/python.*/site-packages/easy-install.pth <==
-          remote: /tmp/build_.*/local_package
+          remote: /tmp/build_.*/packages/local_package_setup_py
           remote: /app/.heroku/src/gunicorn
           remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable___local_package_pyproject_toml_0_0_1_finder.py <==
+          remote: .*
+          remote: MAPPING = \{'local_package_pyproject_toml': '/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'\}
+          remote: .*
+          remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable__.local_package_pyproject_toml-0.0.1.pth <==
+          remote: import __editable___.*
           remote: ==> .heroku/python/lib/python.*/site-packages/gunicorn.egg-link <==
           remote: /app/.heroku/src/gunicorn
           remote: .
-          remote: ==> .heroku/python/lib/python.*/site-packages/local-package.egg-link <==
-          remote: /tmp/build_.*/local_package
+          remote: ==> .heroku/python/lib/python.*/site-packages/local-package-setup-py.egg-link <==
+          remote: /tmp/build_.*/packages/local_package_setup_py
           remote: .
-          remote: Running entrypoint for the local package: Hello!
+          remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
+          remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
         REGEX
 
         # Test rewritten paths work at runtime.
-        expect(app.run('bin/test-entrypoints')).to match(Regexp.new(<<~REGEX))
+        expect(app.run('bin/test-entrypoints')).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE))
           ==> .heroku/python/lib/python.*/site-packages/distutils-precedence.pth <==
           .*
 
           ==> .heroku/python/lib/python.*/site-packages/easy-install.pth <==
-          /app/local_package
+          /app/packages/local_package_setup_py
           /app/.heroku/src/gunicorn
 
+          ==> .heroku/python/lib/python.*/site-packages/__editable___local_package_pyproject_toml_0_0_1_finder.py <==
+          .*
+          MAPPING = \{'local_package_pyproject_toml': '/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'\}
+          .*
+
+          ==> .heroku/python/lib/python.*/site-packages/__editable__.local_package_pyproject_toml-0.0.1.pth <==
+          import __editable___.*
           ==> .heroku/python/lib/python.*/site-packages/gunicorn.egg-link <==
           /app/.heroku/src/gunicorn
           .
-          ==> .heroku/python/lib/python.*/site-packages/local-package.egg-link <==
-          /app/local_package
+          ==> .heroku/python/lib/python.*/site-packages/local-package-setup-py.egg-link <==
+          /app/packages/local_package_setup_py
           .
-          Running entrypoint for the local package: Hello!
+          Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
+          Running entrypoint for the setup.py-based local package: Hello setup.py!
           Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
         REGEX
 
         # Test that the cached .pth files work correctly.
         app.commit!
         app.push!
-        expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
-          remote:        Successfully installed gunicorn-20.1.0 local-package-0.0.1
+        expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE))
+          remote:        Successfully installed gunicorn-20.1.0 local-package-pyproject-toml-0.0.1 local-package-setup-py-0.0.1
           remote: -----> Running post-compile hook
           remote: ==> .heroku/python/lib/python.*/site-packages/distutils-precedence.pth <==
           remote: .*
           remote: 
           remote: ==> .heroku/python/lib/python.*/site-packages/easy-install.pth <==
           remote: /app/.heroku/src/gunicorn
-          remote: /tmp/build_.*/local_package
+          remote: /tmp/build_.*/packages/local_package_setup_py
+          remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable___local_package_pyproject_toml_0_0_1_finder.py <==
+          remote: .*
+          remote: MAPPING = \{'local_package_pyproject_toml': '/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'\}
+          remote: .*
           remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable__.local_package_pyproject_toml-0.0.1.pth <==
+          remote: import __editable___.*
           remote: ==> .heroku/python/lib/python.*/site-packages/gunicorn.egg-link <==
           remote: /app/.heroku/src/gunicorn
           remote: .
-          remote: ==> .heroku/python/lib/python.*/site-packages/local-package.egg-link <==
-          remote: /tmp/build_.*/local_package
+          remote: ==> .heroku/python/lib/python.*/site-packages/local-package-setup-py.egg-link <==
+          remote: /tmp/build_.*/packages/local_package_setup_py
           remote: .
-          remote: Running entrypoint for the local package: Hello!
+          remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
+          remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
           remote: ==> .heroku/python/lib/python.*/site-packages/distutils-precedence.pth <==
           remote: .*
           remote: 
           remote: ==> .heroku/python/lib/python.*/site-packages/easy-install.pth <==
           remote: /app/.heroku/src/gunicorn
-          remote: /tmp/build_.*/local_package
+          remote: /tmp/build_.*/packages/local_package_setup_py
+          remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable___local_package_pyproject_toml_0_0_1_finder.py <==
+          remote: .*
+          remote: MAPPING = \{'local_package_pyproject_toml': '/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'\}
+          remote: .*
           remote: 
+          remote: ==> .heroku/python/lib/python.*/site-packages/__editable__.local_package_pyproject_toml-0.0.1.pth <==
+          remote: import __editable___.*
           remote: ==> .heroku/python/lib/python.*/site-packages/gunicorn.egg-link <==
           remote: /app/.heroku/src/gunicorn
           remote: .
-          remote: ==> .heroku/python/lib/python.*/site-packages/local-package.egg-link <==
-          remote: /tmp/build_.*/local_package
+          remote: ==> .heroku/python/lib/python.*/site-packages/local-package-setup-py.egg-link <==
+          remote: /tmp/build_.*/packages/local_package_setup_py
           remote: .
-          remote: Running entrypoint for the local package: Hello!
+          remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
+          remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
         REGEX
       end