Fix getHtmlDocument for compressed documents
This commit is contained in:
parent
3228e22428
commit
67339001a8
1 changed files with 3 additions and 9 deletions
|
@ -283,14 +283,12 @@ const parseHtmlBytes = (
|
||||||
|
|
||||||
const getHtmlDocument = async (
|
const getHtmlDocument = async (
|
||||||
body: AsyncIterable<string | Uint8Array>,
|
body: AsyncIterable<string | Uint8Array>,
|
||||||
contentLength: number,
|
|
||||||
httpCharset: string | null,
|
httpCharset: string | null,
|
||||||
abortSignal: AbortSignal
|
abortSignal: AbortSignal
|
||||||
): Promise<HTMLDocument> => {
|
): Promise<HTMLDocument> => {
|
||||||
let result: HTMLDocument = emptyHtmlDocument();
|
let result: HTMLDocument = emptyHtmlDocument();
|
||||||
|
|
||||||
const maxHtmlBytesToLoad = Math.min(contentLength, MAX_HTML_BYTES_TO_LOAD);
|
const buffer = new Uint8Array(MAX_HTML_BYTES_TO_LOAD);
|
||||||
const buffer = new Uint8Array(maxHtmlBytesToLoad);
|
|
||||||
let bytesLoadedSoFar = 0;
|
let bytesLoadedSoFar = 0;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -307,16 +305,13 @@ const getHtmlDocument = async (
|
||||||
chunk = Buffer.from(chunk, httpCharset || 'utf8');
|
chunk = Buffer.from(chunk, httpCharset || 'utf8');
|
||||||
}
|
}
|
||||||
|
|
||||||
const truncatedChunk = chunk.slice(
|
const truncatedChunk = chunk.slice(0, buffer.length - bytesLoadedSoFar);
|
||||||
0,
|
|
||||||
maxHtmlBytesToLoad - bytesLoadedSoFar
|
|
||||||
);
|
|
||||||
buffer.set(truncatedChunk, bytesLoadedSoFar);
|
buffer.set(truncatedChunk, bytesLoadedSoFar);
|
||||||
bytesLoadedSoFar += truncatedChunk.byteLength;
|
bytesLoadedSoFar += truncatedChunk.byteLength;
|
||||||
|
|
||||||
result = parseHtmlBytes(buffer.slice(0, bytesLoadedSoFar), httpCharset);
|
result = parseHtmlBytes(buffer.slice(0, bytesLoadedSoFar), httpCharset);
|
||||||
|
|
||||||
const hasLoadedMaxBytes = bytesLoadedSoFar >= maxHtmlBytesToLoad;
|
const hasLoadedMaxBytes = bytesLoadedSoFar >= buffer.length;
|
||||||
if (hasLoadedMaxBytes) {
|
if (hasLoadedMaxBytes) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -495,7 +490,6 @@ export async function fetchLinkPreviewMetadata(
|
||||||
|
|
||||||
const document = await getHtmlDocument(
|
const document = await getHtmlDocument(
|
||||||
response.body,
|
response.body,
|
||||||
contentLength,
|
|
||||||
contentType.charset,
|
contentType.charset,
|
||||||
abortSignal
|
abortSignal
|
||||||
);
|
);
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue