Do not index preview URLs for searching (see contao#2550)

leofeyer · web-flow · commit 0df5442b5ff4 · 2020-12-08T14:03:08.000+01:00
Description ----------- | Q | A | -----------------| --- | Fixed issues | Fixes contao#2507 | Docs PR or issue | - @Toflar: If the `X-Robots-Tag` header or the meta robots tag contains `noindex`, the listener will not pass the document to the indexer at all. This means that potentially existing entries will not be deleted. Is this OK? Commits ------- 17d0e8a Do not index preview URLs for searching
diff --git a/core-bundle/src/EventListener/SearchIndexListener.php b/core-bundle/src/EventListener/SearchIndexListener.php
@@ -71,7 +71,26 @@ public function __invoke(TerminateEvent $event): void
             return;
         }
 
-        $document = Document::createFromRequestResponse($request, $event->getResponse());
+        $response = $event->getResponse();
+
+        // Do not index if the X-Robots-Tag header contains "noindex"
+        if (false !== strpos($response->headers->get('X-Robots-Tag', ''), 'noindex')) {
+            return;
+        }
+
+        $document = Document::createFromRequestResponse($request, $response);
+
+        try {
+            $robots = $document->getContentCrawler()->filterXPath('//head/meta[@name="robots"]')->first()->attr('content');
+
+            // Do not index if the meta robots tag contains "noindex"
+            if (false !== strpos($robots, 'noindex')) {
+                return;
+            }
+        } catch (\Exception $e) {
+            // No meta robots tag found
+        }
+
         $lds = $document->extractJsonLdScripts();
 
         // If there are no json ld scripts at all, this should not be handled by our indexer
diff --git a/core-bundle/tests/EventListener/SearchIndexListenerTest.php b/core-bundle/tests/EventListener/SearchIndexListenerTest.php
@@ -121,5 +121,24 @@ public function getRequestResponse(): \Generator
             false,
             false,
         ];
+
+        $response = new Response('<html><body><script type="application/ld+json">{"@context":"https:\/\/contao.org\/","@type":"Page","pageId":2,"noSearch":false,"protected":false,"groups":[],"fePreview":false}</script></body></html>', 403);
+        $response->headers->set('X-Robots-Tag', 'noindex');
+
+        yield 'Should not be handled because the X-Robots-Tag header contains "noindex" ' => [
+            Request::create('/foobar'),
+            $response,
+            SearchIndexListener::FEATURE_DELETE | SearchIndexListener::FEATURE_INDEX,
+            false,
+            false,
+        ];
+
+        yield 'Should not be handled because the meta robots tag contains "noindex" ' => [
+            Request::create('/foobar'),
+            new Response('<html><head><meta name="robots" content="noindex,nofollow"/></head><body><script type="application/ld+json">{"@context":"https:\/\/contao.org\/","@type":"Page","pageId":2,"noSearch":false,"protected":false,"groups":[],"fePreview":false}</script></body></html>', 403),
+            SearchIndexListener::FEATURE_DELETE | SearchIndexListener::FEATURE_INDEX,
+            false,
+            false,
+        ];
     }
 }
diff --git a/manager-bundle/src/Resources/skeleton/web/preview.php b/manager-bundle/src/Resources/skeleton/web/preview.php
@@ -30,6 +30,9 @@
 $kernel = ContaoKernel::fromRequest(\dirname(__DIR__), $request);
 $response = $kernel->handle($request);
 
+// Prevent preview URLs from being indexed
+$response->headers->set('X-Robots-Tag', 'noindex');
+
 // Force no-cache on all responses in the preview front controller
 $response->headers->set('Cache-Control', 'no-store');
 
