Feature(sitemap): named files chunking strategy (#14471)

* feat(sitemap): add chunking strategy for sitemaps

Adds the ability to split sitemap generation into chunks based on
customizable logic. This allows for better management of large
sitemaps and improved performance.

The new `chunks` option in the sitemap configuration allows users to
define functions that categorize sitemap items into different chunks.
Each chunk is then written to a separate sitemap file.

This change introduces a new `writeSitemapChunk` function to handle
the writing of individual sitemap chunks.

* feat(sitemap): add chunks option to sitemap config

Adds a `chunks` option to the sitemap configuration schema.
This allows users to define custom chunking strategies for
generating sitemaps, providing flexibility in how the sitemap
is split into multiple files.

* feat(sitemap): add sitemap chunk writing functionality

* fix(sitemap): fix empty callback in writeSitemap

The empty callback function in the `writeSitemap` function was
causing unnecessary function calls. This commit fixes this by
removing the empty callback.

* feat(sitemap): add test fixture for sitemap chunking

This commit adds a test fixture to verify the sitemap chunking
functionality. It includes a configuration file, dependencies,
and several pages to simulate a real-world scenario.

* test(sitemap): add test for sitemap chunking with files

* feat(sitemap): add changeset for sitemap chunking

Adds changeset to document the new sitemap chunking feature.
This feature allows splitting sitemap generation into chunks
based on customizable logic, improving management of large
sitemaps and performance.

* build: update dependencies and add astro

* chore: remove unused astro dependency

* chore: remove unused entries from lockfile

* refactor(sitemap): improve import ordering and formatting

* refactor(sitemap): improve import ordering

The import order of `AstroConfig` has been moved to align
with other imports, improving code readability and
consistency. This change ensures that type imports are
grouped together, making the codebase easier to maintain.

* refactor(sitemap): improve import ordering

* refactor(sitemap): improve import ordering

* refactor(sitemap): improve import ordering

* refactor(sitemap): improve chunk file test readability

Simplify the chunk file test by using `path.resolve` and
`includes` for better readability and maintainability.
This change improves the test's clarity without altering
its functionality.

* test(sitemap): fix flaky chunk file tests

The tests were failing intermittently because the `readXML` function
was not properly resolving the file path. This commit updates the
`readXML` function to use `fixture.readFile` to ensure that the file
path is resolved correctly. Additionally, the `flatMapUrls` function
is now async to ensure that the `readXML` function is awaited.

* refactor(sitemap): improve import ordering

* Update .changeset/floppy-times-grab.md

Co-authored-by: Matt Kane <m@mk.gg>

* chore(sitemap): update changeset to minor

The previous changeset incorrectly marked the sitemap chunking feature as a major change. This commit corrects the changeset to reflect that it is a minor feature addition.

* feat(sitemap): add chunking support for sitemap generation

* fix: attempt to fix lockfile

* fix: conflict

* fix: lockfile

---------

Co-authored-by: Matt Kane <m@mk.gg>
Co-authored-by: Princesseuh <3019731+Princesseuh@users.noreply.github.com>

Author: 3 people

.changeset/floppy-times-grab.md

@@ -0,0 +1,36 @@
+---
+'@astrojs/sitemap': minor
+---
+
+Adds the ability to split sitemap generation into chunks based on customizable logic. This allows for better management of large sitemaps and improved performance. The new `chunks` option in the sitemap configuration allows users to define functions that categorize sitemap items into different chunks. Each chunk is then written to a separate sitemap file. 
+
+```
+integrations: [
+  sitemap({
+    serialize(item) { th
+      return item
+    },
+    chunks: { // this property will be treated last on the configuration
+      'blog': (item) => {  // will produce a sitemap file with `blog` name (sitemap-blog-0.xml)
+        if (/blog/.test(item.url)) { // filter path that will be included in this specific sitemap file 
+          item.changefreq = 'weekly';
+          item.lastmod = new Date();
+          item.priority = 0.9; // define specific properties for this filtered path
+          return item;
+        }
+      },
+      'glossary': (item) => {
+        if (/glossary/.test(item.url)) {
+          item.changefreq = 'weekly';
+          item.lastmod = new Date();
+          item.priority = 0.7;
+          return item;
+        }
+      }
+
+      // the rest of the path will be stored in `sitemap-pages.0.xml`
+    },
+  }),
+],
+
+  ```

packages/integrations/sitemap/src/index.ts

@@ -7,8 +7,10 @@ import { ZodError } from 'zod';
 import { generateSitemap } from './generate-sitemap.js';
 import { validateOptions } from './validate-options.js';
 import { writeSitemap } from './write-sitemap.js';
+import { writeSitemapChunk } from './write-sitemap-chunk.js';
 
 export { EnumChangefreq as ChangeFreqEnum } from 'sitemap';
+
 export type ChangeFreq = `${EnumChangefreq}`;
 export type SitemapItem = Pick<
 	SitemapItemLoose,
@@ -18,36 +20,35 @@ export type LinkItem = LinkItemBase;
 
 export type SitemapOptions =
 	| {
-			filenameBase?: string;
-			filter?(page: string): boolean;
-			customSitemaps?: string[];
-			customPages?: string[];
-
-			i18n?: {
-				defaultLocale: string;
-				locales: Record<string, string>;
-			};
-			// number of entries per sitemap file
-			entryLimit?: number;
-
-			// sitemap specific
-			changefreq?: ChangeFreq;
-			lastmod?: Date;
-			priority?: number;
-
-			// called for each sitemap item just before to save them on disk, sync or async
-			serialize?(item: SitemapItem): SitemapItem | Promise<SitemapItem | undefined> | undefined;
-
-			xslURL?: string;
-
-			// namespace configuration
-			namespaces?: {
-				news?: boolean;
-				xhtml?: boolean;
-				image?: boolean;
-				video?: boolean;
-			};
-	  }
+		filenameBase?: string;
+		filter?(page: string): boolean;
+		customSitemaps?: string[];
+		customPages?: string[];
+
+		i18n?: {
+			defaultLocale: string;
+			locales: Record<string, string>;
+		};
+		// number of entries per sitemap file
+		entryLimit?: number;
+		// sitemap specific
+		changefreq?: ChangeFreq;
+		lastmod?: Date;
+		priority?: number;
+
+		// called for each sitemap item just before to save them on disk, sync or async
+		serialize?(item: SitemapItem): SitemapItem | Promise<SitemapItem | undefined> | undefined;
+
+		xslURL?: string;
+		chunks?: Record<string, (item: SitemapItem) => SitemapItem | Promise<SitemapItem | undefined> | undefined>
+		// namespace configuration
+		namespaces?: {
+			news?: boolean;
+			xhtml?: boolean;
+			image?: boolean;
+			video?: boolean;
+		};
+	}
 	| undefined;
 
 function formatConfigErrorMessage(err: ZodError) {
@@ -102,8 +103,7 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => {
 
 					const opts = validateOptions(config.site, options);
 
-					const { filenameBase, filter, customPages, customSitemaps, serialize, entryLimit } = opts;
-
+					const { filenameBase, filter, customPages, customSitemaps, serialize, entryLimit, chunks } = opts;
 					const outFile = `${filenameBase}-index.xml`;
 					const finalSiteUrl = new URL(config.base, config.site);
 					const shouldIgnoreStatus = isStatusCodePage(Object.keys(opts.i18n?.locales ?? {}));
@@ -179,9 +179,53 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => {
 							return;
 						}
 					}
+
 					const destDir = fileURLToPath(dir);
 					const lastmod = opts.lastmod?.toISOString();
 					const xslURL = opts.xslURL ? new URL(opts.xslURL, finalSiteUrl).href : undefined;
+
+					if (chunks) {
+						try {
+							let groupedUrlCollection: SitemapItem['url'][] = []
+							const chunksItem: Record<string, SitemapItem[]> = {};
+							for (const [key, cb] of Object.entries(chunks)) {
+								// Create a new, separate collection for each key
+								const collection: SitemapItem[] = [];
+
+								for (const item of urlData) {
+									// Await the asynchronous operation
+									const collect = await Promise.resolve(cb(item));
+									if (collect) {
+										collection.push(collect);
+									}
+								}
+
+								// Assign the specific collection to its key
+								chunksItem[key] = collection;
+								groupedUrlCollection = [...groupedUrlCollection, ...collection.map((coll) => coll.url)]
+							}
+							chunksItem['pages'] = urlData.filter((urlDataItem) => !(groupedUrlCollection.includes(urlDataItem.url)))
+								// Process each chunk here
+								await writeSitemapChunk({
+									filenameBase,
+									hostname: finalSiteUrl.href,
+									sitemapHostname: finalSiteUrl.href,
+									sourceData: chunksItem,
+									destinationDir: destDir,
+									publicBasePath: config.base,
+									customSitemaps,
+									limit: entryLimit,
+									xslURL,
+									lastmod,
+									namespaces: opts.namespaces,
+								}, config);
+								logger.info(`\`${outFile}\` created at \`${path.relative(process.cwd(), destDir)}\``);
+								return
+						} catch (err) {
+							logger.error(`Error chunking sitemaps\n${(err as any).toString()}`);
+							return;
+						}
+					}
 					await writeSitemap(
 						{
 							filenameBase: filenameBase,

packages/integrations/sitemap/src/schema.ts

@@ -47,6 +47,7 @@ export const SitemapOptionsSchema = z
 			})
 			.optional()
 			.default(SITEMAP_CONFIG_DEFAULTS.namespaces),
-	})
+		chunks: z.record(z.function().args(z.any()).returns(z.any())).optional(),
+		})
 	.strict()
 	.default(SITEMAP_CONFIG_DEFAULTS);

packages/integrations/sitemap/src/write-sitemap-chunk.ts

@@ -0,0 +1,128 @@
+import { createWriteStream, type WriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { normalize, resolve } from 'node:path';
+import { pipeline, Readable } from 'node:stream';
+import { promisify } from 'node:util';
+import type { AstroConfig } from 'astro';
+import { SitemapAndIndexStream, SitemapIndexStream, SitemapStream } from 'sitemap';
+import replace from 'stream-replace-string';
+import type { SitemapItem } from './index.js';
+
+
+type WriteSitemapChunkConfig = {
+	filenameBase: string;
+	hostname: string;
+	sitemapHostname?: string;
+	sourceData: Record<string, SitemapItem[]>;
+	destinationDir: string;
+	customSitemaps?: string[];
+	publicBasePath?: string;
+	limit?: number;
+	xslURL?: string;
+	lastmod?: string;
+	namespaces?: {
+		news?: boolean;
+		xhtml?: boolean;
+		image?: boolean;
+		video?: boolean;
+	};
+};
+
+// adapted from sitemap.js/sitemap-simple
+export async function writeSitemapChunk(
+	{
+		filenameBase,
+		hostname,
+		sitemapHostname = hostname,
+		sourceData,
+		destinationDir,
+		limit = 50000,
+		customSitemaps = [],
+		publicBasePath = './',
+		xslURL: xslUrl,
+		lastmod,
+		namespaces = { news: true, xhtml: true, image: true, video: true },
+	}: WriteSitemapChunkConfig,
+	astroConfig: AstroConfig,
+) {
+	await mkdir(destinationDir, { recursive: true });
+	
+	// Normalize publicBasePath
+	let normalizedPublicBasePath = publicBasePath;
+	if (!normalizedPublicBasePath.endsWith('/')) {
+		normalizedPublicBasePath += '/';
+	}
+
+	// Array to collect all sitemap URLs for the index
+	const sitemapUrls: Array<{ url: string; lastmod?: string }> = [];
+
+	// Process each chunk separately
+	for (const [chunkName, items] of Object.entries(sourceData)) {
+		const sitemapAndIndexStream = new SitemapAndIndexStream({
+			limit,
+			xslUrl,
+			getSitemapStream: (i) => {
+				const sitemapStream = new SitemapStream({
+					hostname,
+					xslUrl,
+					// Custom namespace handling
+					xmlns: {
+						news: namespaces?.news !== false,
+						xhtml: namespaces?.xhtml !== false,
+						image: namespaces?.image !== false,
+						video: namespaces?.video !== false,
+					},
+				});
+				
+				const path = `./${filenameBase}-${chunkName}-${i}.xml`;
+				const writePath = resolve(destinationDir, path);
+				const publicPath = normalize(normalizedPublicBasePath + path);
+
+				let stream: WriteStream;
+				if (astroConfig.trailingSlash === 'never' || astroConfig.build.format === 'file') {
+					// workaround for trailing slash issue in sitemap.js
+					const host = hostname.endsWith('/') ? hostname.slice(0, -1) : hostname;
+					const searchStr = `<loc>${host}/</loc>`;
+					const replaceStr = `<loc>${host}</loc>`;
+					stream = sitemapStream
+						.pipe(replace(searchStr, replaceStr))
+						.pipe(createWriteStream(writePath));
+				} else {
+					stream = sitemapStream.pipe(createWriteStream(writePath));
+				}
+
+				const url = new URL(publicPath, sitemapHostname).toString();
+				
+				// Collect this sitemap URL for the index
+				sitemapUrls.push({ url, lastmod });
+				
+				return [{ url, lastmod }, sitemapStream, stream];
+			},
+		});
+
+		// Create a readable stream from this chunk's items
+		const dataStream = Readable.from(items);
+
+		// Write this chunk's sitemap(s)
+		await promisify(pipeline)(dataStream, sitemapAndIndexStream);
+	}
+
+	// Now create the sitemap index with all the generated sitemaps
+	const indexStream = new SitemapIndexStream({ xslUrl });
+	const indexPath = resolve(destinationDir, `./${filenameBase}-index.xml`);
+	const indexWriteStream = createWriteStream(indexPath);
+
+	// Add custom sitemaps to the index
+	for (const url of customSitemaps) {
+		indexStream.write({ url, lastmod });
+	}
+
+	// Add all generated sitemaps to the index
+	for (const sitemapUrl of sitemapUrls) {
+		indexStream.write(sitemapUrl);
+	}
+
+	indexStream.end();
+
+	return await promisify(pipeline)(indexStream, indexWriteStream);
+}

packages/integrations/sitemap/src/write-sitemap.ts

@@ -93,7 +93,7 @@ export async function writeSitemap(
 			sitemapAndIndexStream,
 			{ url, lastmod },
 			'utf8',
-			() => {},
+			() => { },
 		);
 	}
 	return promisify(pipeline)(src, sitemapAndIndexStream, createWriteStream(indexPath));

packages/integrations/sitemap/test/chunks-files.test.js

@@ -0,0 +1,61 @@
+import assert from 'node:assert/strict';
+import { before, describe, it } from 'node:test';
+import { sitemap } from './fixtures/static/deps.mjs';
+import { loadFixture, readXML } from './test-utils.js';
+
+describe('Sitemap with chunked files', () => {
+	/** @type {import('./test-utils.js').Fixture} */
+	let fixture;
+	/** @type {string[]} */
+	let blogUrls;
+	let glossaryUrls;
+	let pagesUrls;
+
+	before(async () => {
+		fixture = await loadFixture({
+			root: './fixtures/chunks/',
+			integrations: [
+				sitemap({
+					serialize(item) {
+						return item
+					},
+					chunks: {
+						'blog': (item) => {
+							if (item.url.includes('blog')) {
+								item.changefreq = 'weekly';
+								item.lastmod = new Date();
+								item.priority = 0.9;
+								return item;
+							}
+						},
+						'glossary': (item) => {
+							if (item.url.includes('glossary')) {
+								item.changefreq = 'weekly';
+								item.lastmod = new Date();
+								item.priority = 0.9;
+								return item;
+							}
+						}
+					},
+				}),
+			],
+		});
+		await fixture.build();
+		const flatMapUrls = async (file) => {
+			const data = await readXML(fixture.readFile(file))
+			return data.urlset.url.map((url) => url.loc[0])
+		};
+		blogUrls = await flatMapUrls('sitemap-blog-0.xml');
+		glossaryUrls = await flatMapUrls('sitemap-glossary-0.xml')
+		pagesUrls = await flatMapUrls('sitemap-pages-0.xml')
+	});
+
+	it('includes defined custom pages', async () => {
+		assert.equal(blogUrls.includes('http://example.com/blog/one/'), true);
+		assert.equal(blogUrls.includes('http://example.com/blog/two/'), true);
+		assert.equal(glossaryUrls.includes('http://example.com/glossary/one/'), true);
+		assert.equal(glossaryUrls.includes('http://example.com/glossary/two/'), true);
+		assert.equal(pagesUrls.includes('http://example.com/one/'), true);
+		assert.equal(pagesUrls.includes('http://example.com/two/'), true);
+	});
+});