Merge remote-tracking branch 'origin/develop' into renovate/vector-im

Author: dbkr

apps/web/package.json

@@ -89,7 +89,7 @@
         "opus-recorder": "^8.0.3",
         "pako": "^2.0.3",
         "png-chunks-extract": "^1.0.0",
-        "posthog-js": "1.364.7",
+        "posthog-js": "1.369.3",
         "qrcode": "1.5.4",
         "re-resizable": "6.11.2",
         "react": "catalog:",
@@ -214,7 +214,7 @@
         "postcss-preset-env": "11.2.1",
         "postcss-scss": "4.0.9",
         "postcss-simple-vars": "7.0.1",
-        "prettier": "3.8.1",
+        "prettier": "3.8.3",
         "process": "^0.11.10",
         "raw-loader": "^4.0.2",
         "semver": "^7.5.2",

apps/web/playwright/e2e/messages/messages.spec.ts

@@ -252,6 +252,7 @@ test.describe("Message url previews", () => {
                         "og:title": "A simple site",
                         "og:description": "And with a brief description",
                         "og:image": mxc,
+                        "og:image:alt": "The riot logo",
                     },
                 });
             });

apps/web/playwright/snapshots/messages/messages.spec.ts/preview-basic-linux.png

@@ -13,7 +13,6 @@ import { type Interaction as InteractionEvent } from "@matrix-org/analytics-even
 import { type PinUnpinAction } from "@matrix-org/analytics-events/types/typescript/PinUnpinAction";
 import { type RoomListSortingAlgorithmChanged } from "@matrix-org/analytics-events/types/typescript/RoomListSortingAlgorithmChanged";
 import { type UrlPreviewRendered } from "@matrix-org/analytics-events/types/typescript/UrlPreviewRendered";
-import { type UrlPreview } from "@element-hq/web-shared-components";
 
 import PageType from "./PageTypes";
 import Views from "./Views";
@@ -151,7 +150,7 @@ export default class PosthogTrackers {
      * @param isEncrypted Whether the event (and effectively the room) was encrypted.
      * @param previews The previews generated from the event.
      */
-    public trackUrlPreview(eventId: string, isEncrypted: boolean, previews: UrlPreview[]): void {
+    public trackUrlPreview(eventId: string, isEncrypted: boolean, previews: { image?: unknown }[]): void {
         // Discount any previews that we have already tracked.
         if (this.previewedEventIds.get(eventId)) {
             return;

apps/web/playwright/snapshots/messages/messages.spec.ts/preview-with-thumb-linux.png

@@ -34,8 +34,10 @@ export interface UrlPreviewGroupViewModelProps {
 }
 
 export const MAX_PREVIEWS_WHEN_LIMITED = 2;
-export const PREVIEW_WIDTH = 100;
-export const PREVIEW_HEIGHT = 100;
+export const PREVIEW_WIDTH_PX = 478;
+export const PREVIEW_HEIGHT_PX = 200;
+export const MIN_PREVIEW_PX = 96;
+export const MIN_IMAGE_SIZE_BYTES = 8192;
 
 export enum PreviewVisibility {
     /**
@@ -100,28 +102,77 @@ export class UrlPreviewGroupViewModel
             typeof response["og:description"] === "string" && response["og:description"].trim()
                 ? response["og:description"].trim()
                 : undefined;
-        let siteName =
+        const siteName =
             typeof response["og:site_name"] === "string" && response["og:site_name"].trim()
                 ? response["og:site_name"].trim()
-                : undefined;
+                : new URL(link).hostname;
 
+        // If there is no title, use the description as the title.
         if (!title && description) {
             title = description;
             description = undefined;
         } else if (!title && siteName) {
             title = siteName;
-            siteName = undefined;
         } else if (!title) {
             title = link;
         }
 
+        // If the description matches the site name, don't bother with a description.
+        if (description && description.toLowerCase() === siteName.toLowerCase()) {
+            description = undefined;
+        }
+
         return {
             title,
             description: description && decode(description),
             siteName,
         };
     }
 
+    /**
+     * Calculate the best possible author from an opengraph response.
+     * @param response The opengraph response
+     * @returns The author value, or undefined if no valid author could be found.
+     */
+    private static getAuthorFromResponse(response: IPreviewUrlResponse): UrlPreview["author"] {
+        let calculatedAuthor: string | undefined;
+        if (response["og:type"] === "article") {
+            if (typeof response["article:author"] === "string" && response["article:author"]) {
+                calculatedAuthor = response["article:author"];
+            }
+            // Otherwise fall through to check the profile.
+        }
+        if (typeof response["profile:username"] === "string" && response["profile:username"]) {
+            calculatedAuthor = response["profile:username"];
+        }
+        if (calculatedAuthor && URL.canParse(calculatedAuthor)) {
+            // Some sites return URLs as authors which doesn't look good in Element, so discard it.
+            return;
+        }
+        return calculatedAuthor;
+    }
+
+    /**
+     * Calculate whether the provided image from the preview response is an full size preview or
+     * a site icon.
+     * @returns `true` if the image should be used as a preview, otherwise `false`
+     */
+    private static isImagePreview(width?: number, height?: number, bytes?: number): boolean {
+        // We can't currently distinguish from a preview image and a favicon. Neither OpenGraph nor Matrix
+        // have a clear distinction, so we're using a heuristic here to check the dimensions & size of the file and
+        // deciding whether to render it as a full preview or icon.
+        if (width && width < MIN_PREVIEW_PX) {
+            return false;
+        }
+        if (height && height < MIN_PREVIEW_PX) {
+            return false;
+        }
+        if (bytes && bytes < MIN_IMAGE_SIZE_BYTES) {
+            return false;
+        }
+        return true;
+    }
+
     /**
      * Determine if an anchor element can be rendered into a preview.
      * If it can, return the value of `href`
@@ -278,38 +329,54 @@ export class UrlPreviewGroupViewModel
         }
 
         const { title, description, siteName } = UrlPreviewGroupViewModel.getBaseMetadataFromResponse(preview, link);
+        const author = UrlPreviewGroupViewModel.getAuthorFromResponse(preview);
         const hasImage = preview["og:image"] && typeof preview?.["og:image"] === "string";
         // Ensure we have something relevant to render.
         // The title must not just be the link, or we must have an image.
         if (title === link && !hasImage) {
             return null;
         }
         let image: UrlPreview["image"];
+        let siteIcon: string | undefined;
         if (typeof preview["og:image"] === "string" && this.visibility > PreviewVisibility.MediaHidden) {
             const media = mediaFromMxc(preview["og:image"], this.client);
             const declaredHeight = UrlPreviewGroupViewModel.getNumberFromOpenGraph(preview["og:image:height"]);
             const declaredWidth = UrlPreviewGroupViewModel.getNumberFromOpenGraph(preview["og:image:width"]);
-            const width = Math.min(declaredWidth ?? PREVIEW_WIDTH, PREVIEW_WIDTH);
-            const height = thumbHeight(width, declaredHeight, PREVIEW_WIDTH, PREVIEW_WIDTH) ?? PREVIEW_WIDTH;
-            const thumb = media.getThumbnailOfSourceHttp(PREVIEW_WIDTH, PREVIEW_HEIGHT, "scale");
-            // No thumb, no preview.
-            if (thumb) {
-                image = {
-                    imageThumb: thumb,
-                    imageFull: media.srcHttp ?? thumb,
-                    width,
-                    height,
-                    fileSize: UrlPreviewGroupViewModel.getNumberFromOpenGraph(preview["matrix:image:size"]),
-                };
+            const imageSize = UrlPreviewGroupViewModel.getNumberFromOpenGraph(preview["matrix:image:size"]);
+            const alt = typeof preview["og:image:alt"] === "string" ? preview["og:image:alt"] : undefined;
+
+            const isImagePreview = UrlPreviewGroupViewModel.isImagePreview(declaredWidth, declaredHeight, imageSize);
+            if (isImagePreview) {
+                const width = Math.min(declaredWidth ?? PREVIEW_WIDTH_PX, PREVIEW_WIDTH_PX);
+                const height =
+                    thumbHeight(width, declaredHeight, PREVIEW_WIDTH_PX, PREVIEW_WIDTH_PX) ?? PREVIEW_WIDTH_PX;
+                const thumb = media.getThumbnailOfSourceHttp(PREVIEW_WIDTH_PX, PREVIEW_HEIGHT_PX, "scale");
+                const playable = !!preview["og:video"] || !!preview["og:video:type"] || !!preview["og:audio"];
+                // No thumb, no preview.
+                if (thumb) {
+                    image = {
+                        imageThumb: thumb,
+                        imageFull: media.srcHttp ?? thumb,
+                        width,
+                        height,
+                        fileSize: UrlPreviewGroupViewModel.getNumberFromOpenGraph(preview["matrix:image:size"]),
+                        alt,
+                        playable,
+                    };
+                }
+            } else if (media.srcHttp) {
+                siteIcon = media.srcHttp;
             }
         }
 
         const result = {
             link,
             title,
+            author,
             description,
             siteName,
-            showTooltipOnLink: link !== title && PlatformPeg.get()?.needsUrlTooltips(),
+            siteIcon,
+            showTooltipOnLink: !!(link !== title && PlatformPeg.get()?.needsUrlTooltips()),
             image,
         } satisfies UrlPreview;
         this.previewCache.set(link, result);

apps/web/src/PosthogTrackers.ts

@@ -18,20 +18,10 @@ describe("PosthogTrackers", () => {
         const tracker = new PosthogTrackers();
         tracker.trackUrlPreview("$123456", false, [
             {
-                title: "A preview",
-                image: {
-                    imageThumb: "abc",
-                    imageFull: "abc",
-                },
-                link: "a-link",
-            },
-        ]);
-        tracker.trackUrlPreview("$123456", false, [
-            {
-                title: "A second preview",
-                link: "a-link",
+                image: {},
             },
         ]);
+        tracker.trackUrlPreview("$123456", false, [{}]);
         // Ignores subsequent calls.
         expect(PosthogAnalytics.instance.trackEvent).toHaveBeenCalledWith({
             eventName: "UrlPreviewRendered",

apps/web/src/viewmodels/message-body/UrlPreviewGroupViewModel.ts

@@ -125,6 +125,32 @@ describe("UrlPreviewGroupViewModel", () => {
         await vm.updateEventElement(msg);
         expect(vm.getSnapshot()).toMatchSnapshot();
     });
+    it.each<Partial<IPreviewUrlResponse>>([
+        { "matrix:image:size": 8191 },
+        { "og:image:width": 95 },
+        { "og:image:height": 95 },
+    ])("should preview a URL with a site icon", async (extraResp) => {
+        const { vm, client } = getViewModel();
+        client.getUrlPreview.mockResolvedValueOnce({
+            "og:title": "This is an example!",
+            "og:type": "document",
+            "og:url": "https://example.org",
+            "og:image": IMAGE_MXC,
+            "og:image:height": 128,
+            "og:image:width": 128,
+            "matrix:image:size": 8193,
+            ...extraResp,
+        });
+        // eslint-disable-next-line no-restricted-properties
+        client.mxcUrlToHttp.mockImplementation((url) => {
+            expect(url).toEqual(IMAGE_MXC);
+            return "https://example.org/image/src";
+        });
+        const msg = document.createElement("div");
+        msg.innerHTML = '<a href="https://example.org">Test</a>';
+        await vm.updateEventElement(msg);
+        expect(vm.getSnapshot().previews[0].siteIcon).toBeTruthy();
+    });
     it("should ignore media when mediaVisible is false", async () => {
         const { vm, client } = getViewModel({ mediaVisible: false, visible: true });
         client.getUrlPreview.mockResolvedValueOnce({
@@ -200,6 +226,41 @@ describe("UrlPreviewGroupViewModel", () => {
         expect(vm.getSnapshot()).toMatchSnapshot();
     });
 
+    describe("calculates author", () => {
+        it("should use the profile:username if provided", async () => {
+            const { vm, client } = getViewModel();
+            client.getUrlPreview.mockResolvedValueOnce({ ...BASIC_PREVIEW_OGDATA, "profile:username": "my username" });
+            const msg = document.createElement("div");
+            msg.innerHTML = '<a href="https://example.org">Test</a>';
+            await vm.updateEventElement(msg);
+            expect(vm.getSnapshot().previews[0].author).toEqual("my username");
+        });
+        it("should use author if the og:type is an article", async () => {
+            const { vm, client } = getViewModel();
+            client.getUrlPreview.mockResolvedValueOnce({
+                ...BASIC_PREVIEW_OGDATA,
+                "og:type": "article",
+                "article:author": "my name",
+            });
+            const msg = document.createElement("div");
+            msg.innerHTML = '<a href="https://example.org">Test</a>';
+            await vm.updateEventElement(msg);
+            expect(vm.getSnapshot().previews[0].author).toEqual("my name");
+        });
+        it("should NOT use author if the author is a URL", async () => {
+            const { vm, client } = getViewModel();
+            client.getUrlPreview.mockResolvedValueOnce({
+                ...BASIC_PREVIEW_OGDATA,
+                "og:type": "article",
+                "article:author": "https://junk.example.org/foo",
+            });
+            const msg = document.createElement("div");
+            msg.innerHTML = '<a href="https://example.org">Test</a>';
+            await vm.updateEventElement(msg);
+            expect(vm.getSnapshot().previews[0].author).toBeUndefined();
+        });
+    });
+
     it.each([
         { text: "", href: "", hasPreview: false },
         { text: "test", href: "noprotocol.example.org", hasPreview: false },
@@ -232,7 +293,7 @@ describe("UrlPreviewGroupViewModel", () => {
             // API *may* return a string, so check we parse correctly.
             "og:image:height": "500" as unknown as number,
             "og:image:width": 500,
-            "matrix:image:size": 1024,
+            "matrix:image:size": 10000,
             "og:image": IMAGE_MXC,
         },
     ])("handles different kinds of opengraph responses %s", async (og) => {
@@ -251,4 +312,25 @@ describe("UrlPreviewGroupViewModel", () => {
         await vm.updateEventElement(msg);
         expect(vm.getSnapshot().previews[0]).toMatchSnapshot();
     });
+
+    it.each<string>(["og:video", "og:video:type", "og:audio"])("detects playable links via %s", async (property) => {
+        const { vm, client } = getViewModel();
+        // eslint-disable-next-line no-restricted-properties
+        client.mxcUrlToHttp.mockImplementation((url, width) => {
+            expect(url).toEqual(IMAGE_MXC);
+            if (width) {
+                return "https://example.org/image/thumb";
+            }
+            return "https://example.org/image/src";
+        });
+        client.getUrlPreview.mockResolvedValueOnce({
+            ...BASIC_PREVIEW_OGDATA,
+            "og:image": IMAGE_MXC,
+            [property]: "anything",
+        });
+        const msg = document.createElement("div");
+        msg.innerHTML = `<a href="https://example.org">test</a>`;
+        await vm.updateEventElement(msg);
+        expect(vm.getSnapshot().previews[0].image?.playable).toEqual(true);
+    });
 });