594 lines
16 KiB
TypeScript
594 lines
16 KiB
TypeScript
// Copyright 2020-2021 Signal Messenger, LLC
|
|
// SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
import { RequestInit, Response } from 'node-fetch';
|
|
import type { AbortSignal as AbortSignalForNodeFetch } from 'abort-controller';
|
|
|
|
import {
|
|
IMAGE_GIF,
|
|
IMAGE_ICO,
|
|
IMAGE_JPEG,
|
|
IMAGE_PNG,
|
|
IMAGE_WEBP,
|
|
MIMEType,
|
|
stringToMIMEType,
|
|
} from '../types/MIME';
|
|
import * as log from '../logging/log';
|
|
|
|
const USER_AGENT = 'WhatsApp/2';
|
|
|
|
const MAX_REQUEST_COUNT_WITH_REDIRECTS = 20;
|
|
|
|
// Lifted from the `fetch` spec [here][0].
|
|
// [0]: https://fetch.spec.whatwg.org/#redirect-status
|
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
|
|
const MAX_CONTENT_TYPE_LENGTH_TO_PARSE = 100;
|
|
|
|
// Though we'll accept HTML of any Content-Length (including no specified length), we
|
|
// will only load some of the HTML. So we might start loading a 99 gigabyte HTML page
|
|
// but only parse the first 1000 kilobytes. However, if the Content-Length is less than
|
|
// this, we won't waste space.
|
|
const MAX_HTML_BYTES_TO_LOAD = 1000 * 1024;
|
|
|
|
// `<title>x` is 8 bytes. Nothing else (meta tags, etc) will even fit, so we can ignore
|
|
// it. This is mostly to protect us against empty response bodies.
|
|
const MIN_HTML_CONTENT_LENGTH = 8;
|
|
|
|
// Similar to the above. We don't want to show tiny images (even though the more likely
|
|
// case is that the Content-Length is 0).
|
|
const MIN_IMAGE_CONTENT_LENGTH = 8;
|
|
const MAX_IMAGE_CONTENT_LENGTH = 1024 * 1024;
|
|
const VALID_IMAGE_MIME_TYPES: Set<MIMEType> = new Set([
|
|
IMAGE_GIF,
|
|
IMAGE_ICO,
|
|
IMAGE_JPEG,
|
|
IMAGE_PNG,
|
|
IMAGE_WEBP,
|
|
]);
|
|
|
|
// We want to discard unreasonable dates. Update this in ~950 years. (This may discard
|
|
// some reasonable dates, which is okay because it is only for link previews.)
|
|
const MIN_DATE = 0;
|
|
const MAX_DATE = new Date(3000, 0, 1).valueOf();
|
|
|
|
const emptyContentType = { type: null, charset: null };
|
|
|
|
export type FetchFn = (href: string, init: RequestInit) => Promise<Response>;
|
|
|
|
export type LinkPreviewMetadata = {
|
|
title: string;
|
|
description: null | string;
|
|
date: null | number;
|
|
imageHref: null | string;
|
|
};
|
|
|
|
export type LinkPreviewImage = {
|
|
data: Uint8Array;
|
|
contentType: MIMEType;
|
|
};
|
|
|
|
type ParsedContentType =
|
|
| { type: null; charset: null }
|
|
| { type: MIMEType; charset: null | string };
|
|
|
|
// This throws non-helpful errors because (1) it logs (2) it will be immediately caught.
|
|
async function fetchWithRedirects(
|
|
fetchFn: FetchFn,
|
|
href: string,
|
|
options: RequestInit
|
|
): Promise<Response> {
|
|
const urlsSeen = new Set<string>();
|
|
|
|
let nextHrefToLoad = href;
|
|
for (let i = 0; i < MAX_REQUEST_COUNT_WITH_REDIRECTS; i += 1) {
|
|
if (urlsSeen.has(nextHrefToLoad)) {
|
|
log.warn('fetchWithRedirects: found a redirect loop');
|
|
throw new Error('redirect loop');
|
|
}
|
|
urlsSeen.add(nextHrefToLoad);
|
|
|
|
// This `await` is deliberately inside of a loop.
|
|
// eslint-disable-next-line no-await-in-loop
|
|
const response = await fetchFn(nextHrefToLoad, {
|
|
...options,
|
|
redirect: 'manual',
|
|
});
|
|
|
|
if (!REDIRECT_STATUSES.has(response.status)) {
|
|
return response;
|
|
}
|
|
|
|
const location = response.headers.get('location');
|
|
if (!location) {
|
|
log.warn(
|
|
'fetchWithRedirects: got a redirect status code but no Location header; bailing'
|
|
);
|
|
throw new Error('no location with redirect');
|
|
}
|
|
|
|
const newUrl = maybeParseUrl(location, nextHrefToLoad);
|
|
if (newUrl?.protocol !== 'https:') {
|
|
log.warn(
|
|
'fetchWithRedirects: got a redirect status code and an invalid Location header'
|
|
);
|
|
throw new Error('invalid location');
|
|
}
|
|
|
|
nextHrefToLoad = newUrl.href;
|
|
}
|
|
|
|
log.warn('fetchWithRedirects: too many redirects');
|
|
throw new Error('too many redirects');
|
|
}
|
|
|
|
function maybeParseUrl(href: string, base: string): null | URL {
|
|
let result: URL;
|
|
try {
|
|
result = new URL(href, base);
|
|
} catch (err) {
|
|
return null;
|
|
}
|
|
// We never need the hash
|
|
result.hash = '';
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Parses a Content-Type header value. Refer to [RFC 2045][0] for details (though this is
|
|
* a simplified version for link previews.
|
|
* [0]: https://tools.ietf.org/html/rfc2045
|
|
*/
|
|
const parseContentType = (headerValue: string | null): ParsedContentType => {
|
|
if (!headerValue || headerValue.length > MAX_CONTENT_TYPE_LENGTH_TO_PARSE) {
|
|
return emptyContentType;
|
|
}
|
|
|
|
const [rawType, ...rawParameters] = headerValue
|
|
.toLowerCase()
|
|
.split(/;/g)
|
|
.map(part => part.trim())
|
|
.filter(Boolean);
|
|
if (!rawType) {
|
|
return emptyContentType;
|
|
}
|
|
|
|
let charset: null | string = null;
|
|
for (let i = 0; i < rawParameters.length; i += 1) {
|
|
const rawParameter = rawParameters[i];
|
|
const parsed = new URLSearchParams(rawParameter);
|
|
const parsedCharset = parsed.get('charset')?.trim();
|
|
if (parsedCharset) {
|
|
charset = parsedCharset;
|
|
break;
|
|
}
|
|
}
|
|
|
|
return {
|
|
type: stringToMIMEType(rawType),
|
|
charset,
|
|
};
|
|
};
|
|
|
|
const isInlineContentDisposition = (headerValue: string | null): boolean =>
|
|
!headerValue || headerValue.split(';', 1)[0] === 'inline';
|
|
|
|
const parseContentLength = (headerValue: string | null): number => {
|
|
// No need to parse gigantic Content-Lengths; only parse the first 10 digits.
|
|
if (typeof headerValue !== 'string' || !/^\d{1,10}$/g.test(headerValue)) {
|
|
return Infinity;
|
|
}
|
|
const result = parseInt(headerValue, 10);
|
|
return Number.isNaN(result) ? Infinity : result;
|
|
};
|
|
|
|
type ValidCharset =
|
|
| 'ascii'
|
|
| 'utf8'
|
|
| 'utf-8'
|
|
| 'utf16le'
|
|
| 'ucs2'
|
|
| 'ucs-2'
|
|
| 'base64'
|
|
| 'latin1'
|
|
| 'binary'
|
|
| 'hex';
|
|
|
|
const VALID_CHARSETS = new Set([
|
|
'ascii',
|
|
'utf8',
|
|
'utf-8',
|
|
'utf16le',
|
|
'ucs2',
|
|
'ucs-2',
|
|
'base64',
|
|
'latin1',
|
|
'binary',
|
|
'hex',
|
|
]);
|
|
|
|
const checkCharset = (charSet: string | null): charSet is ValidCharset => {
|
|
if (!charSet) {
|
|
return false;
|
|
}
|
|
return VALID_CHARSETS.has(charSet);
|
|
};
|
|
|
|
const emptyHtmlDocument = (): HTMLDocument =>
|
|
new DOMParser().parseFromString('', 'text/html');
|
|
|
|
// The charset behavior here follows the [W3 guidelines][0]. The priority is BOM, HTTP
|
|
// header, `http-equiv` meta tag, `charset` meta tag, and finally a UTF-8 fallback.
|
|
// (This fallback could, perhaps, be smarter based on user locale.)
|
|
// [0]: https://www.w3.org/International/questions/qa-html-encoding-declarations.en
|
|
const parseHtmlBytes = (
|
|
bytes: Readonly<Uint8Array>,
|
|
httpCharset: string | null
|
|
): HTMLDocument => {
|
|
const hasBom = bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf;
|
|
|
|
let isSureOfCharset: boolean;
|
|
let decoder: TextDecoder;
|
|
if (hasBom) {
|
|
decoder = new TextDecoder();
|
|
isSureOfCharset = true;
|
|
} else if (httpCharset) {
|
|
try {
|
|
decoder = new TextDecoder(httpCharset);
|
|
isSureOfCharset = true;
|
|
} catch (err) {
|
|
decoder = new TextDecoder();
|
|
isSureOfCharset = false;
|
|
}
|
|
} else {
|
|
decoder = new TextDecoder();
|
|
isSureOfCharset = false;
|
|
}
|
|
|
|
let decoded: string;
|
|
try {
|
|
decoded = decoder.decode(bytes);
|
|
} catch (err) {
|
|
decoded = '';
|
|
}
|
|
|
|
let document: HTMLDocument;
|
|
try {
|
|
document = new DOMParser().parseFromString(decoded, 'text/html');
|
|
} catch (err) {
|
|
document = emptyHtmlDocument();
|
|
}
|
|
|
|
if (!isSureOfCharset) {
|
|
const httpEquiv = document
|
|
.querySelector('meta[http-equiv="content-type"]')
|
|
?.getAttribute('content');
|
|
if (httpEquiv) {
|
|
const httpEquivCharset = parseContentType(httpEquiv).charset;
|
|
if (httpEquivCharset) {
|
|
return parseHtmlBytes(bytes, httpEquivCharset);
|
|
}
|
|
}
|
|
|
|
const metaCharset = document
|
|
.querySelector('meta[charset]')
|
|
?.getAttribute('charset');
|
|
if (metaCharset) {
|
|
return parseHtmlBytes(bytes, metaCharset);
|
|
}
|
|
}
|
|
|
|
return document;
|
|
};
|
|
|
|
const getHtmlDocument = async (
|
|
body: AsyncIterable<string | Uint8Array>,
|
|
contentLength: number,
|
|
httpCharset: string | null,
|
|
abortSignal: AbortSignal
|
|
): Promise<HTMLDocument> => {
|
|
let result: HTMLDocument = emptyHtmlDocument();
|
|
|
|
const maxHtmlBytesToLoad = Math.min(contentLength, MAX_HTML_BYTES_TO_LOAD);
|
|
const buffer = new Uint8Array(maxHtmlBytesToLoad);
|
|
let bytesLoadedSoFar = 0;
|
|
|
|
try {
|
|
for await (let chunk of body) {
|
|
if (abortSignal.aborted) {
|
|
break;
|
|
}
|
|
|
|
// This check exists to satisfy TypeScript; chunk should always be a Buffer.
|
|
if (typeof chunk === 'string') {
|
|
if (!checkCharset(httpCharset)) {
|
|
throw new Error(`Invalid charset: ${httpCharset}`);
|
|
}
|
|
chunk = Buffer.from(chunk, httpCharset || 'utf8');
|
|
}
|
|
|
|
const truncatedChunk = chunk.slice(
|
|
0,
|
|
maxHtmlBytesToLoad - bytesLoadedSoFar
|
|
);
|
|
buffer.set(truncatedChunk, bytesLoadedSoFar);
|
|
bytesLoadedSoFar += truncatedChunk.byteLength;
|
|
|
|
result = parseHtmlBytes(buffer.slice(0, bytesLoadedSoFar), httpCharset);
|
|
|
|
const hasLoadedMaxBytes = bytesLoadedSoFar >= maxHtmlBytesToLoad;
|
|
if (hasLoadedMaxBytes) {
|
|
break;
|
|
}
|
|
}
|
|
} catch (err) {
|
|
log.warn(
|
|
'getHtmlDocument: error when reading body; continuing with what we got'
|
|
);
|
|
}
|
|
|
|
return result;
|
|
};
|
|
|
|
const getOpenGraphContent = (
|
|
document: HTMLDocument,
|
|
properties: ReadonlyArray<string>
|
|
): string | null => {
|
|
for (let i = 0; i < properties.length; i += 1) {
|
|
const property = properties[i];
|
|
const content = document
|
|
.querySelector(`meta[property="${property}"]`)
|
|
?.getAttribute('content')
|
|
?.trim();
|
|
if (content) {
|
|
return content;
|
|
}
|
|
}
|
|
return null;
|
|
};
|
|
|
|
const getLinkHrefAttribute = (
|
|
document: HTMLDocument,
|
|
rels: ReadonlyArray<string>
|
|
): string | null => {
|
|
for (let i = 0; i < rels.length; i += 1) {
|
|
const rel = rels[i];
|
|
const href = document
|
|
.querySelector(`link[rel="${rel}"]`)
|
|
?.getAttribute('href')
|
|
?.trim();
|
|
if (href) {
|
|
return href;
|
|
}
|
|
}
|
|
return null;
|
|
};
|
|
|
|
const parseMetadata = (
|
|
document: HTMLDocument,
|
|
href: string
|
|
): LinkPreviewMetadata | null => {
|
|
const title =
|
|
getOpenGraphContent(document, ['og:title']) || document.title.trim();
|
|
if (!title) {
|
|
log.warn("parseMetadata: HTML document doesn't have a title; bailing");
|
|
return null;
|
|
}
|
|
|
|
const description =
|
|
getOpenGraphContent(document, ['og:description']) ||
|
|
document
|
|
.querySelector('meta[name="description"]')
|
|
?.getAttribute('content')
|
|
?.trim() ||
|
|
null;
|
|
|
|
const rawImageHref =
|
|
getOpenGraphContent(document, ['og:image', 'og:image:url']) ||
|
|
getLinkHrefAttribute(document, [
|
|
'apple-touch-icon',
|
|
'apple-touch-icon-precomposed',
|
|
'shortcut icon',
|
|
'icon',
|
|
]);
|
|
const imageUrl = rawImageHref ? maybeParseUrl(rawImageHref, href) : null;
|
|
const imageHref = imageUrl ? imageUrl.href : null;
|
|
|
|
let date: number | null = null;
|
|
const rawDate = getOpenGraphContent(document, [
|
|
'og:published_time',
|
|
'article:published_time',
|
|
'og:modified_time',
|
|
'article:modified_time',
|
|
]);
|
|
if (rawDate) {
|
|
const parsed = Date.parse(rawDate);
|
|
if (parsed > MIN_DATE && parsed < MAX_DATE) {
|
|
date = parsed;
|
|
}
|
|
}
|
|
|
|
return {
|
|
title,
|
|
description,
|
|
imageHref,
|
|
date,
|
|
};
|
|
};
|
|
|
|
/**
|
|
* This attempts to fetch link preview metadata, returning `null` if it cannot be found
|
|
* for any reason.
|
|
*
|
|
* NOTE: This does NOT validate the incoming URL for safety. For example, it may fetch an
|
|
* insecure HTTP href. It also does not offer a timeout; that is up to the caller.
|
|
*
|
|
* At a high level, it:
|
|
*
|
|
* 1. Makes a GET request, following up to 20 redirects (`fetch`'s default).
|
|
* 2. Checks the response status code and headers to make sure it's a normal HTML
|
|
* response.
|
|
* 3. Streams up to `MAX_HTML_BYTES_TO_LOAD`, stopping when (1) it has loaded all of the
|
|
* HTML (2) loaded the maximum number of bytes (3) finished loading the `<head>`.
|
|
* 4. Parses the resulting HTML with `DOMParser`.
|
|
* 5. Grabs the title, description, image URL, and date.
|
|
*/
|
|
export async function fetchLinkPreviewMetadata(
|
|
fetchFn: FetchFn,
|
|
href: string,
|
|
abortSignal: AbortSignal
|
|
): Promise<null | LinkPreviewMetadata> {
|
|
let response: Response;
|
|
try {
|
|
response = await fetchWithRedirects(fetchFn, href, {
|
|
headers: {
|
|
Accept: 'text/html,application/xhtml+xml',
|
|
'User-Agent': USER_AGENT,
|
|
},
|
|
signal: abortSignal as AbortSignalForNodeFetch,
|
|
});
|
|
} catch (err) {
|
|
log.warn(
|
|
'fetchLinkPreviewMetadata: failed to fetch link preview HTML; bailing'
|
|
);
|
|
return null;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
log.warn(
|
|
`fetchLinkPreviewMetadata: got a ${response.status} status code; bailing`
|
|
);
|
|
return null;
|
|
}
|
|
|
|
if (!response.body) {
|
|
log.warn('fetchLinkPreviewMetadata: no response body; bailing');
|
|
return null;
|
|
}
|
|
|
|
if (
|
|
!isInlineContentDisposition(response.headers.get('Content-Disposition'))
|
|
) {
|
|
log.warn(
|
|
'fetchLinkPreviewMetadata: Content-Disposition header is not inline; bailing'
|
|
);
|
|
return null;
|
|
}
|
|
|
|
if (abortSignal.aborted) {
|
|
return null;
|
|
}
|
|
|
|
const contentLength = parseContentLength(
|
|
response.headers.get('Content-Length')
|
|
);
|
|
if (contentLength < MIN_HTML_CONTENT_LENGTH) {
|
|
log.warn('fetchLinkPreviewMetadata: Content-Length is too short; bailing');
|
|
return null;
|
|
}
|
|
|
|
const contentType = parseContentType(response.headers.get('Content-Type'));
|
|
if (contentType.type !== 'text/html') {
|
|
log.warn('fetchLinkPreviewMetadata: Content-Type is not HTML; bailing');
|
|
return null;
|
|
}
|
|
|
|
const document = await getHtmlDocument(
|
|
response.body,
|
|
contentLength,
|
|
contentType.charset,
|
|
abortSignal
|
|
);
|
|
|
|
// [The Node docs about `ReadableStream.prototype[Symbol.asyncIterator]`][0] say that
|
|
// the stream will be destroyed if you `break` out of the loop, but I could not
|
|
// reproduce this. Also [`destroy` is a documented method][1] but it is not in the
|
|
// Node types, which is why we do this cast to `any`.
|
|
// [0]: https://nodejs.org/docs/latest-v12.x/api/stream.html#stream_readable_symbol_asynciterator
|
|
// [1]: https://nodejs.org/docs/latest-v12.x/api/stream.html#stream_readable_destroy_error
|
|
try {
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
(response.body as any).destroy();
|
|
} catch (err) {
|
|
// Ignored.
|
|
}
|
|
|
|
if (abortSignal.aborted) {
|
|
return null;
|
|
}
|
|
|
|
return parseMetadata(document, response.url);
|
|
}
|
|
|
|
/**
|
|
* This attempts to fetch an image, returning `null` if it fails for any reason.
|
|
*
|
|
* NOTE: This does NOT validate the incoming URL for safety. For example, it may fetch an
|
|
* insecure HTTP href. It also does not offer a timeout; that is up to the caller.
|
|
*/
|
|
export async function fetchLinkPreviewImage(
|
|
fetchFn: FetchFn,
|
|
href: string,
|
|
abortSignal: AbortSignal
|
|
): Promise<null | LinkPreviewImage> {
|
|
let response: Response;
|
|
try {
|
|
response = await fetchWithRedirects(fetchFn, href, {
|
|
headers: {
|
|
'User-Agent': USER_AGENT,
|
|
},
|
|
size: MAX_IMAGE_CONTENT_LENGTH,
|
|
signal: abortSignal as AbortSignalForNodeFetch,
|
|
});
|
|
} catch (err) {
|
|
log.warn('fetchLinkPreviewImage: failed to fetch image; bailing');
|
|
return null;
|
|
}
|
|
|
|
if (abortSignal.aborted) {
|
|
return null;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
log.warn(
|
|
`fetchLinkPreviewImage: got a ${response.status} status code; bailing`
|
|
);
|
|
return null;
|
|
}
|
|
|
|
const contentLength = parseContentLength(
|
|
response.headers.get('Content-Length')
|
|
);
|
|
if (contentLength < MIN_IMAGE_CONTENT_LENGTH) {
|
|
log.warn('fetchLinkPreviewImage: Content-Length is too short; bailing');
|
|
return null;
|
|
}
|
|
if (contentLength > MAX_IMAGE_CONTENT_LENGTH) {
|
|
log.warn(
|
|
'fetchLinkPreviewImage: Content-Length is too large or is unset; bailing'
|
|
);
|
|
return null;
|
|
}
|
|
|
|
const { type: contentType } = parseContentType(
|
|
response.headers.get('Content-Type')
|
|
);
|
|
if (!contentType || !VALID_IMAGE_MIME_TYPES.has(contentType)) {
|
|
log.warn('fetchLinkPreviewImage: Content-Type is not an image; bailing');
|
|
return null;
|
|
}
|
|
|
|
let data: Uint8Array;
|
|
try {
|
|
data = await response.buffer();
|
|
} catch (err) {
|
|
log.warn('fetchLinkPreviewImage: failed to read body; bailing');
|
|
return null;
|
|
}
|
|
|
|
if (abortSignal.aborted) {
|
|
return null;
|
|
}
|
|
|
|
return { data, contentType };
|
|
}
|