Fix getHtmlDocument for compressed documents

This commit is contained in:
Fedor Indutny 2021-12-20 17:45:57 +01:00 committed by GitHub
parent 3228e22428
commit 67339001a8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -283,14 +283,12 @@ const parseHtmlBytes = (
const getHtmlDocument = async ( const getHtmlDocument = async (
body: AsyncIterable<string | Uint8Array>, body: AsyncIterable<string | Uint8Array>,
contentLength: number,
httpCharset: string | null, httpCharset: string | null,
abortSignal: AbortSignal abortSignal: AbortSignal
): Promise<HTMLDocument> => { ): Promise<HTMLDocument> => {
let result: HTMLDocument = emptyHtmlDocument(); let result: HTMLDocument = emptyHtmlDocument();
const maxHtmlBytesToLoad = Math.min(contentLength, MAX_HTML_BYTES_TO_LOAD); const buffer = new Uint8Array(MAX_HTML_BYTES_TO_LOAD);
const buffer = new Uint8Array(maxHtmlBytesToLoad);
let bytesLoadedSoFar = 0; let bytesLoadedSoFar = 0;
try { try {
@ -307,16 +305,13 @@ const getHtmlDocument = async (
chunk = Buffer.from(chunk, httpCharset || 'utf8'); chunk = Buffer.from(chunk, httpCharset || 'utf8');
} }
const truncatedChunk = chunk.slice( const truncatedChunk = chunk.slice(0, buffer.length - bytesLoadedSoFar);
0,
maxHtmlBytesToLoad - bytesLoadedSoFar
);
buffer.set(truncatedChunk, bytesLoadedSoFar); buffer.set(truncatedChunk, bytesLoadedSoFar);
bytesLoadedSoFar += truncatedChunk.byteLength; bytesLoadedSoFar += truncatedChunk.byteLength;
result = parseHtmlBytes(buffer.slice(0, bytesLoadedSoFar), httpCharset); result = parseHtmlBytes(buffer.slice(0, bytesLoadedSoFar), httpCharset);
const hasLoadedMaxBytes = bytesLoadedSoFar >= maxHtmlBytesToLoad; const hasLoadedMaxBytes = bytesLoadedSoFar >= buffer.length;
if (hasLoadedMaxBytes) { if (hasLoadedMaxBytes) {
break; break;
} }
@ -495,7 +490,6 @@ export async function fetchLinkPreviewMetadata(
const document = await getHtmlDocument( const document = await getHtmlDocument(
response.body, response.body,
contentLength,
contentType.charset, contentType.charset,
abortSignal abortSignal
); );