Explain why we need to stream and why not use readline

This commit is contained in:
Asger F
2024-11-19 13:28:41 +01:00
parent 2cde3b9c2f
commit bb1da9c6ff

View File

@@ -15,6 +15,8 @@ export async function readJsonlFile<T>(
handler: (value: T) => Promise<void>,
logger?: { log: (message: string) => void },
): Promise<void> {
// Stream the data as large evaluator logs won't fit in memory.
// Also avoid using 'readline' as it is slower than our manual line splitting.
void logger?.log(
`Parsing ${path} (${(await stat(path)).size / 1024 / 1024} MB)...`,
);