111 lines
3.2 KiB
JavaScript
111 lines
3.2 KiB
JavaScript
// Split chunks line-wise for generators passed to the `std*` options
|
|
export const getSplitLinesGenerator = (binary, preserveNewlines, skipped, state) => binary || skipped
|
|
? undefined
|
|
: initializeSplitLines(preserveNewlines, state);
|
|
|
|
// Same but for synchronous methods
|
|
export const splitLinesSync = (chunk, preserveNewlines, objectMode) => objectMode
|
|
? chunk.flatMap(item => splitLinesItemSync(item, preserveNewlines))
|
|
: splitLinesItemSync(chunk, preserveNewlines);
|
|
|
|
const splitLinesItemSync = (chunk, preserveNewlines) => {
|
|
const {transform, final} = initializeSplitLines(preserveNewlines, {});
|
|
return [...transform(chunk), ...final()];
|
|
};
|
|
|
|
const initializeSplitLines = (preserveNewlines, state) => {
|
|
state.previousChunks = '';
|
|
return {
|
|
transform: splitGenerator.bind(undefined, state, preserveNewlines),
|
|
final: linesFinal.bind(undefined, state),
|
|
};
|
|
};
|
|
|
|
// This imperative logic is much faster than using `String.split()` and uses very low memory.
|
|
const splitGenerator = function * (state, preserveNewlines, chunk) {
|
|
if (typeof chunk !== 'string') {
|
|
yield chunk;
|
|
return;
|
|
}
|
|
|
|
let {previousChunks} = state;
|
|
let start = -1;
|
|
|
|
for (let end = 0; end < chunk.length; end += 1) {
|
|
if (chunk[end] === '\n') {
|
|
const newlineLength = getNewlineLength(chunk, end, preserveNewlines, state);
|
|
let line = chunk.slice(start + 1, end + 1 - newlineLength);
|
|
|
|
if (previousChunks.length > 0) {
|
|
line = concatString(previousChunks, line);
|
|
previousChunks = '';
|
|
}
|
|
|
|
yield line;
|
|
start = end;
|
|
}
|
|
}
|
|
|
|
if (start !== chunk.length - 1) {
|
|
previousChunks = concatString(previousChunks, chunk.slice(start + 1));
|
|
}
|
|
|
|
state.previousChunks = previousChunks;
|
|
};
|
|
|
|
const getNewlineLength = (chunk, end, preserveNewlines, state) => {
|
|
if (preserveNewlines) {
|
|
return 0;
|
|
}
|
|
|
|
state.isWindowsNewline = end !== 0 && chunk[end - 1] === '\r';
|
|
return state.isWindowsNewline ? 2 : 1;
|
|
};
|
|
|
|
const linesFinal = function * ({previousChunks}) {
|
|
if (previousChunks.length > 0) {
|
|
yield previousChunks;
|
|
}
|
|
};
|
|
|
|
// Unless `preserveNewlines: true` is used, we strip the newline of each line.
|
|
// This re-adds them after the user `transform` code has run.
|
|
export const getAppendNewlineGenerator = ({binary, preserveNewlines, readableObjectMode, state}) => binary || preserveNewlines || readableObjectMode
|
|
? undefined
|
|
: {transform: appendNewlineGenerator.bind(undefined, state)};
|
|
|
|
const appendNewlineGenerator = function * ({isWindowsNewline = false}, chunk) {
|
|
const {unixNewline, windowsNewline, LF, concatBytes} = typeof chunk === 'string' ? linesStringInfo : linesUint8ArrayInfo;
|
|
|
|
if (chunk.at(-1) === LF) {
|
|
yield chunk;
|
|
return;
|
|
}
|
|
|
|
const newline = isWindowsNewline ? windowsNewline : unixNewline;
|
|
yield concatBytes(chunk, newline);
|
|
};
|
|
|
|
const concatString = (firstChunk, secondChunk) => `${firstChunk}${secondChunk}`;
|
|
|
|
const linesStringInfo = {
|
|
windowsNewline: '\r\n',
|
|
unixNewline: '\n',
|
|
LF: '\n',
|
|
concatBytes: concatString,
|
|
};
|
|
|
|
const concatUint8Array = (firstChunk, secondChunk) => {
|
|
const chunk = new Uint8Array(firstChunk.length + secondChunk.length);
|
|
chunk.set(firstChunk, 0);
|
|
chunk.set(secondChunk, firstChunk.length);
|
|
return chunk;
|
|
};
|
|
|
|
const linesUint8ArrayInfo = {
|
|
windowsNewline: new Uint8Array([0x0D, 0x0A]),
|
|
unixNewline: new Uint8Array([0x0A]),
|
|
LF: 0x0A,
|
|
concatBytes: concatUint8Array,
|
|
};
|