Improve Website Indexing

website/app/blog/[...slug]/page.tsx

@@ -9,6 +9,7 @@ import {
 } from "@/lib/blog";
 import { compileMdxContent, extractHeadings } from "@/lib/mdx";
 import { createMetadata } from "@/lib/metadata";
+import { siteMetadata } from "@/lib/site-config";
 import { BlogPostPage } from "@/lib/blog-post-page";
 import { BlogListPage } from "@/lib/blog-list-page";
 import { notFound } from "next/navigation";
@@ -50,12 +51,15 @@ export async function generateMetadata({ params }: PageProps) {
   if (slug.length === 4) {
     const postSlug = `/blog/${slug.join("/")}`;
     const post = getBlogPostBySlug(postSlug);
+    const postUrl = `${siteMetadata.siteUrl}${postSlug}/`;
 
     return createMetadata({
       title: post?.title || "Blog Post",
       description: post?.description,
       isArticle: true,
       imageUrl: post?.featuredImage,
+      pageUrl: postUrl,
+      canonicalUrl: postUrl,
     });
   }
 

website/app/docs/[...slug]/page.tsx

@@ -1,8 +1,14 @@
 import React from "react";
 
-import { getAllDocPages, getDocPageBySlug, getDocsConfig } from "@/lib/docs";
+import {
+  getAllDocPages,
+  getDocPageBySlug,
+  getDocsConfig,
+  getProductInfo,
+} from "@/lib/docs";
 import { compileMdxContent, extractHeadings } from "@/lib/mdx";
 import { createMetadata } from "@/lib/metadata";
+import { siteMetadata } from "@/lib/site-config";
 import { DocPageView } from "@/lib/doc-page-view";
 import { notFound } from "next/navigation";
 
@@ -26,9 +32,30 @@ export async function generateMetadata({ params }: PageProps) {
   const title =
     page?.frontmatter?.title || slug[slug.length - 1] || "Documentation";
 
+  const product = page?.product;
+  const version = page?.version;
+  const productInfo = product ? getProductInfo(product) : undefined;
+  const latestVersion = productInfo?.latestStableVersion;
+  const isLatest = !version || !latestVersion || version === latestVersion;
+
+  const pageUrl = `${siteMetadata.siteUrl}${fullSlug}/`;
+  let canonicalUrl: string;
+
+  if (isLatest) {
+    canonicalUrl = pageUrl;
+  } else {
+    const restOfPath = slug.slice(2).join("/");
+    canonicalUrl = restOfPath
+      ? `${siteMetadata.siteUrl}/docs/${product}/${latestVersion}/${restOfPath}/`
+      : `${siteMetadata.siteUrl}/docs/${product}/${latestVersion}/`;
+  }
+
   return createMetadata({
     title,
     description: page?.frontmatter?.description,
+    canonicalUrl,
+    pageUrl,
+    noIndex: !isLatest,
   });
 }
 

website/app/layout.tsx

@@ -16,6 +16,7 @@ const radioCanada = Radio_Canada({
 });
 
 export const metadata: Metadata = {
+  metadataBase: new URL(siteMetadata.siteUrl),
   title: {
     default: siteMetadata.title,
     template: `%s - ${siteMetadata.title}`,

website/app/sitemap.ts

@@ -9,7 +9,7 @@ import {
   getFilesRecursively,
   readMarkdownFile,
 } from "@/lib/content";
-import { getAllDocPages } from "@/lib/docs";
+import { getAllDocPages, getDocsConfig } from "@/lib/docs";
 import { siteMetadata } from "@/lib/site-config";
 
 export const dynamic = "force-static";
@@ -57,10 +57,12 @@ const STATIC_ROUTES: { route: string; pageFile: string }[] = [
   { route: "/services/training/", pageFile: "app/services/training/page.tsx" },
 ];
 
-const DOCS_DISALLOWED = [
-  /^\/docs\/hotchocolate\/v10(\/|$)/,
-  /^\/docs\/hotchocolate\/v11(\/|$)/,
-];
+const latestVersionByProduct = new Map<string, string>();
+for (const product of getDocsConfig()) {
+  if (product.latestStableVersion) {
+    latestVersionByProduct.set(product.path, product.latestStableVersion);
+  }
+}
 
 const mtimeCache = new Map<string, Date | undefined>();
 
@@ -194,8 +196,14 @@ export default function sitemap(): MetadataRoute.Sitemap {
 
   const docPages = getAllDocPages();
   for (const page of docPages) {
+    if (page.product && page.version) {
+      const latest = latestVersionByProduct.get(page.product);
+      if (latest && page.version !== latest) {
+        continue;
+      }
+    }
+
     const docPath = page.slug.endsWith("/") ? page.slug : `${page.slug}/`;
-    if (DOCS_DISALLOWED.some((re) => re.test(docPath))) continue;
 
     entries.push({
       url: `${BASE_URL}${docPath}`,

website/lib/metadata.ts

@@ -7,26 +7,39 @@ interface GenerateMetadataOptions {
   description?: string;
   imageUrl?: string;
   isArticle?: boolean;
+  canonicalUrl?: string;
+  pageUrl?: string;
+  noIndex?: boolean;
 }
 
 export function createMetadata({
   title,
   description,
   imageUrl,
   isArticle,
+  canonicalUrl,
+  pageUrl,
+  noIndex,
 }: GenerateMetadataOptions): Metadata {
   const metaDescription = description || siteMetadata.description;
   const metaAuthor = `@${siteMetadata.author}`;
   const metaImageUrl = imageUrl
     ? `${siteMetadata.siteUrl}${imageUrl}`
     : `${siteMetadata.siteUrl}/favicon-32x32.png`;
   const metaType = isArticle ? "article" : "website";
+  const resolvedUrl = pageUrl || siteMetadata.siteUrl;
 
   return {
     title,
     description: metaDescription,
+    ...(canonicalUrl && {
+      alternates: { canonical: canonicalUrl },
+    }),
+    ...(noIndex && {
+      robots: { index: false, follow: true },
+    }),
     openGraph: {
-      url: siteMetadata.siteUrl,
+      url: resolvedUrl,
       title,
       description: metaDescription,
       type: metaType as "website" | "article",

website/public/robots.txt

@@ -3,6 +3,16 @@ User-agent: *
 Allow: /
 Disallow: /docs/hotchocolate/v10/
 Disallow: /docs/hotchocolate/v11/
+Disallow: /docs/hotchocolate/v12/
+Disallow: /docs/hotchocolate/v13/
+Disallow: /docs/hotchocolate/v14/
+Disallow: /docs/hotchocolate/v15/
+Disallow: /docs/fusion/v14/
+Disallow: /docs/fusion/v15/
+Disallow: /docs/strawberryshake/v12/
+Disallow: /docs/strawberryshake/v13/
+Disallow: /docs/strawberryshake/v14/
+Disallow: /docs/strawberryshake/v15/
 
 # Algolia Crawler
 User-agent: Algolia Crawler