How to test a large data export

Raise the row count until the behaviour changes, then name the limit. Run node checkexport.mjs http://127.0.0.1:8931/buffered 50000 100000 150000. Under a 64 MB heap the buffered export finished at 100 000 rows and killed the server at 150 000 in eight runs of ten, where the streaming one delivered a million.

Why check this

An export that passes on a thousand rows and fails on a million fails in a way that names its own cause. Run it when the export is written and after any change to how it is assembled.

Two failures it prevents. A service that dies on a heap limit takes every request in flight down with it. And a transfer that breaks after the response headers arrives as HTTP 200 with a short file, which passes a status assertion.

The size is not the question. Which ceiling arrives first is: memory, a deadline, or the client.

Prerequisites

• Node 22.23.2 and curl 8.21.0 on Windows 11 build 22631, eight cores.
• Port 8931 on 127.0.0.1 and nothing else. Every size runs against the target below.
• How to count rows in a csv file owns the counting. This page uses one number from it, the data row count.
• Every millisecond and megabyte below moves between runs. Byte counts, row counts, status codes and exit codes do not.
• --max-old-space-size sets the heap ceiling, so a laptop reaches the real failure in seconds.

server.mjs is the target. It serves one export two ways.

import { createServer } from 'node:http';
import { Readable } from 'node:stream';

const PORT = 8931;
const LF = String.fromCharCode(10);
const HEAD = 'id,sku,customer,amount,note';
const row = (i) => i + ',SKU-' + String(i).padStart(7, '0') + ',customer ' + i + ',' +
  ((i % 997) / 100).toFixed(2) + ',order line ' + i;
const tail = (n) => '#end,' + n + ',,,';

const mb = (b) => (b < 0 ? '-' : '+') + (Math.abs(b) / 1048576).toFixed(1);
const log = (route, rows, sent, heap, note) => console.log(
  route.padEnd(9) + String(rows).padStart(8) + String(sent).padStart(10) +
  mb(process.memoryUsage().heapUsed - heap).padStart(8) + '  ' + note);

// Buffered. The whole file exists before the first byte leaves. `page` is how many
// rows are built between two turns of the event loop, the way a paged read works.
async function buffered(rows, page) {
  const out = [HEAD];
  for (let i = 1; i <= rows; i += 1) {
    out.push(row(i));
    if (i % page === 0) await new Promise((r) => setImmediate(r));
  }
  out.push(tail(rows));
  return Buffer.from(out.join(LF) + LF, 'utf8');
}

// Streamed. One row at a time, nothing kept. `fail` kills the source mid-file.
const put = (r, s) => { r.sent += Buffer.byteLength(s); r.push(s); };
function streamed(rows, fail) {
  let i = 0;
  const src = new Readable({ read() {
    if (i === 0) { put(this, HEAD + LF); i = 1; return; }
    if (fail > 0 && i === fail) { this.destroy(new Error('export worker died at row ' + i)); return; }
    if (i > rows) { put(this, tail(rows) + LF); this.push(null); return; }
    put(this, row(i) + LF); i += 1;
  } });
  src.sent = 0;
  return src;
}

// connectionsCheckingInterval decides how often the two timeouts below are enforced.
// Its default is 30000 ms, which is longer than either of them.
const srv = createServer({ connectionsCheckingInterval: 500 }, async (req, res) => {
  const u = new URL(req.url, 'http://127.0.0.1:' + PORT);
  const n = (k, d) => Number(u.searchParams.get(k) ?? d);
  const rows = n('rows', 1000), page = n('page', 50000), fail = n('fail', 0), deadline = n('deadline', 0);
  const base = process.memoryUsage().heapUsed;
  res.on('error', (e) => log('socket', rows, -1, base, 'response error ' + e.code));
  req.on('error', (e) => log('socket', rows, -1, base, 'request error ' + e.code));
  let timer = null;
  if (deadline > 0) timer = setTimeout(() => {
    if (res.headersSent) { res.destroy(); log('deadline', rows, -1, base, 'fired after the first byte, the status was already 200'); }
    else {
      res.writeHead(504, { 'content-type': 'application/json' });
      res.end('{"error":"export deadline exceeded","rows":' + rows + '}');
      log('deadline', rows, 0, base, 'fired before the first byte, sent 504');
    }
  }, deadline);

  if (u.pathname === '/buffered') {
    const body = await buffered(rows, page);
    if (res.writableEnded || res.destroyed) return;
    clearTimeout(timer);
    res.writeHead(200, { 'content-type': 'text/csv', 'content-length': body.length });
    res.end(body);
    log('buffered', rows, body.length, base, 'content-length declared');
  } else if (u.pathname === '/stream') {
    res.writeHead(200, { 'content-type': 'text/csv' });
    const src = streamed(rows, fail);
    src.on('error', (e) => { clearTimeout(timer); res.destroy(); log('stream', rows, src.sent, base, e.message + ', the client already has a 200'); });
    res.on('finish', () => { clearTimeout(timer); log('stream', rows, src.sent, base, 'chunked, no content-length'); });
    src.pipe(res);
  } else if (u.pathname === '/import') {
    let got = 0;
    req.on('data', (c) => { got += c.length; });
    req.on('end', () => { clearTimeout(timer); res.writeHead(200, { 'content-type': 'application/json' }); res.end('{"received":' + got + '}'); });
  } else {
    clearTimeout(timer);
    res.writeHead(404).end();
  }
});

srv.headersTimeout = 5000;
srv.requestTimeout = 8000;
srv.listen(PORT, '127.0.0.1', () => console.log(
  'export target on 127.0.0.1:' + PORT + '  headersTimeout ' + srv.headersTimeout +
  ' ms  requestTimeout ' + srv.requestTimeout + ' ms'));

checkexport.mjs is the check: one row per size, non-zero exit on an incomplete file.

import { get } from 'node:http';

const LF = String.fromCharCode(10);
const base = process.argv[2];
const sizes = process.argv.slice(3).map(Number);

function run(url, rows) {
  return new Promise((resolve) => {
    const t0 = process.hrtime.bigint();
    const ms = () => Math.round(Number(process.hrtime.bigint() - t0) / 1e6);
    let ttfb = -1, bytes = 0, lines = 0, end = '', declared = '-', code = 0, err = '';
    let settled = false;
    const done = () => {
      if (settled) return; settled = true;
      const total = ms();
      const last = end.split(LF).filter(Boolean).pop() ?? '';
      const terminated = last === '#end,' + rows + ',,,';
      const read = Math.max(lines - (terminated ? 2 : 1), 0);
      const lengthOk = declared === '-' || Number(declared) === bytes;
      const ok = code === 200 && terminated && read === rows && lengthOk && err === '';
      const why = code === 0 ? err : code !== 200 ? 'HTTP ' + code : err !== '' ? err
        : !terminated ? 'no end line' : read !== rows ? 'short by ' + (rows - read) + ' rows'
        : !lengthOk ? 'length mismatch' : 'complete';
      resolve({ rows, ttfb, total, bytes, declared, read, terminated, ok, why });
    };
    const req = get(url, (res) => {
      code = res.statusCode;
      declared = res.headers['content-length'] ?? '-';
      res.on('data', (c) => {
        if (ttfb < 0) ttfb = ms();
        bytes += c.length;
        const s = c.toString('latin1');
        for (let i = 0; i < s.length; i += 1) if (s.charCodeAt(i) === 10) lines += 1;
        end = (end + s).slice(-40);
      });
      res.on('end', done);
      res.on('error', (e) => { err = e.message; done(); });
    });
    req.on('error', (e) => { err = e.message; done(); });
  });
}

console.log('   rows  ttfb ms  total ms       bytes    declared  rows read  end line  verdict');
let bad = 0;
for (const rows of sizes) {
  const sep = base.includes('?') ? '&' : '?';
  const r = await run(base + sep + 'rows=' + rows, rows);
  if (!r.ok) bad += 1;
  console.log(
    String(r.rows).padStart(7) + (r.ttfb < 0 ? '-' : String(r.ttfb)).padStart(9) + String(r.total).padStart(10) +
    String(r.bytes).padStart(12) + String(r.declared).padStart(12) + String(r.read).padStart(11) +
    (r.terminated ? 'yes' : 'no').padStart(10) + '  ' + r.why);
}
console.log(sizes.length + ' sizes, ' + (sizes.length - bad) + ' complete, ' + bad + ' incomplete');
process.exit(bad === 0 ? 0 : 1);

slowreq.mjs holds a request open, to reach the server's own timeouts.

import { connect } from 'node:net';

const CRLF = String.fromCharCode(13) + String.fromCharCode(10);
const mode = process.argv[2];
const t0 = Date.now();
const at = () => String(Math.round((Date.now() - t0) / 100) / 10) + ' s';

const s = connect(8931, '127.0.0.1', () => {
  if (mode === 'head') {
    s.write('GET /buffered?rows=10 HTTP/1.1' + CRLF + 'host: 127.0.0.1' + CRLF);
    console.log('sent request headers without the blank line, waiting');
  } else {
    s.write('POST /import HTTP/1.1' + CRLF + 'host: 127.0.0.1' + CRLF +
      'content-length: 2000' + CRLF + CRLF + 'id,sku' + CRLF);
    console.log('sent full headers and 8 bytes of a 2000 byte body, waiting');
  }
});
s.on('data', (d) => console.log(at() + '  server sent: ' + d.toString().split(CRLF)[0]));
s.on('close', () => console.log(at() + '  connection closed by the server'));
s.on('error', (e) => console.log(at() + '  socket error ' + e.code));

Steps

1. 1

   Step 1.

   Confirm nothing owns the port.

   netstat -ano | grep ":8931 " | grep -c LISTENING
   

   0

   Any other number means a process is there. Pick another port.

2. 2

   Step 2.

   Start the target and read what it bound.

   node server.mjs > server.log 2>&1 & sleep 2 && netstat -ano | grep "127.0.0.1:8931 " | grep LISTENING
   

     TCP    127.0.0.1:8931         0.0.0.0:0              LISTENING       32408

   The address is 127.0.0.1, so nothing off this machine reaches it. The last column is the process id step 14 stops.

3. 3

   Step 3.

   Measure the buffered export at three sizes.

   node checkexport.mjs http://127.0.0.1:8931/buffered 50000 200000 500000
   

      rows  ttfb ms  total ms       bytes    declared  rows read  end line  verdict
   50000       74       141     2716724     2716724      50000       yes  complete
   200000      256       325    11266728    11266728     200000       yes  complete
   500000      474       731    28666728    28666728     500000       yes  complete
   3 sizes, 3 complete, 0 incomplete

   Time to first byte rises with the row count: the whole file exists before any of it is sent. That wait is what a proxy deadline sees.

4. 4

   Step 4.

   Measure the streaming export at the same sizes.

   node checkexport.mjs http://127.0.0.1:8931/stream 50000 200000 500000
   

      rows  ttfb ms  total ms       bytes    declared  rows read  end line  verdict
   50000       13       272     2716724           -      50000       yes  complete
   200000       10       939    11266728           -     200000       yes  complete
   500000        4      1626    28666728           -     500000       yes  complete
   3 sizes, 3 complete, 0 incomplete

   Byte for byte the same files, first byte from 4 ms. declared is empty: a chunked response carries no content-length.

5. 5

   Step 5.

   Read what the server recorded.

   cat server.log
   

   export target on 127.0.0.1:8931  headersTimeout 5000 ms  requestTimeout 8000 ms
   buffered    50000   2716724   +22.8  content-length declared
   buffered   200000  11266728   +89.6  content-length declared
   buffered   500000  28666728  +217.4  content-length declared
   stream      50000   2716724    +7.3  chunked, no content-length
   stream     200000  11266728    -4.3  chunked, no content-length
   stream     500000  28666728   +14.0  chunked, no content-length

   Heap at the last byte minus heap at the handler's first line. One export cost +89.6 MB buffered and released memory streamed.

6. 6

   Step 6.

   Kill an export halfway.

   curl -s -o partial.csv -w "http_code %{http_code}  bytes %{size_download}\n" "http://127.0.0.1:8931/stream?rows=200000&fail=120000"; echo "curl exit $?"; tail -c 46 partial.csv
   

   http_code 200  bytes 6614185
   curl exit 18
   119784,customer 119784,1.44,order line 119784

   The status went out before the failure existed, so it reads 200. The file ends on an ordinary data row.

7. 7

   Step 7.

   Put that URL through the check.

   node checkexport.mjs "http://127.0.0.1:8931/stream?fail=120000" 200000
   

      rows  ttfb ms  total ms       bytes    declared  rows read  end line  verdict
   200000       12       281     6614185           -     119784        no  aborted
   1 sizes, 0 complete, 1 incomplete

   No content-length to compare against, the dash says so. What is left is the row count, short by 80 216, and the missing last line.

8. 8

   Step 8.

   Give the buffered export a 200 ms deadline.

   curl -s -o dead.json -w "http_code %{http_code}  bytes %{size_download}  time %{time_total}\n" "http://127.0.0.1:8931/buffered?rows=500000&deadline=200"; echo "curl exit $?"; cat dead.json
   

   http_code 504  bytes 50  time 0.207856
   curl exit 0
   {"error":"export deadline exceeded","rows":500000}

   504 and a 50 byte body. Buffering buys one thing: the status can still carry the failure.

9. 9

   Step 9.

   Same deadline on a stream with a slow reader.

   curl -s -o slow.csv --limit-rate 2M -w "http_code %{http_code}  bytes %{size_download}  time %{time_total}\n" "http://127.0.0.1:8931/stream?rows=1000000&deadline=1000"; echo "curl exit $?"; tail -c 39 slow.csv
   

   http_code 200  bytes 4139285  time 1.013298
   curl exit 18
   5,customer 75865,0.93,order line 75865

   Opposite report. The 200 left with the first row, so the deadline could only destroy the socket.

10. 10

    Step 10.

    Time the client out instead of the server.

    curl -s -o cut.csv --max-time 1 -w "http_code %{http_code}  bytes %{size_download}\n" "http://127.0.0.1:8931/stream?rows=1000000"; echo "curl exit $?"; wc -c < cut.csv
    

    http_code 200  bytes 30044329
    curl exit 28
    30044329

    Exit 28 is the client giving up, exit 18 in step 9 was the server stopping. Both leave a partial file.

11. 11

    Step 11.

    Reach the server's own two timeouts.

    node slowreq.mjs head; node slowreq.mjs body
    

    sent request headers without the blank line, waiting
    5.5 s  server sent: HTTP/1.1 408 Request Timeout
    5.5 s  connection closed by the server
    sent full headers and 8 bytes of a 2000 byte body, waiting
    8.1 s  server sent: HTTP/1.1 408 Request Timeout
    8.1 s  connection closed by the server

    Both answer 408 at the value set for them. Neither produces a short export: the handler never ran.

12. 12

    Step 12.

    Restart under a heap cap and raise the row count.

    P=$(netstat -ano | grep "127.0.0.1:8931 " | grep LISTENING | awk '{print $NF}'); powershell -Command "Stop-Process -Id $P -Force"; sleep 1; { node --max-old-space-size=64 server.mjs; echo "exit code $?"; } > cap.log 2>&1 & sleep 2 && node checkexport.mjs http://127.0.0.1:8931/buffered 50000 100000 150000 200000 250000; sleep 1; grep -E "^buffered|FATAL ERROR|^exit code" cap.log
    

       rows  ttfb ms  total ms       bytes    declared  rows read  end line  verdict
    50000       67        80     2716724     2716724      50000       yes  complete
    100000       78       100     5466728     5466728     100000       yes  complete
    150000        -       197           0           -          0        no  read ECONNRESET
    200000        -         1           0           -          0        no  connect ECONNREFUSED 127.0.0.1:8931
    250000        -         1           0           -          0        no  connect ECONNREFUSED 127.0.0.1:8931
    5 sizes, 2 complete, 3 incomplete
    buffered    50000   2716724   +21.8  content-length declared
    buffered   100000   5466728   +39.6  content-length declared
    FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
    exit code 134

    Across ten repeats the first failure came at 150 000 rows in eight and at 200 000 in two, so the limit is a band. The process left with code 134.

13. 13

    Step 13.

    Run the same sizes against the streaming route.

    node --max-old-space-size=64 server.mjs > cap2.log 2>&1 & sleep 2 && node checkexport.mjs http://127.0.0.1:8931/stream 150000 250000 1000000; grep "^stream" cap2.log
    

       rows  ttfb ms  total ms       bytes    declared  rows read  end line  verdict
    150000       15       377     8366728           -     150000       yes  complete
    250000        4       502    14166728           -     250000       yes  complete
    1000000        6      1933    57666732           -    1000000       yes  complete
    3 sizes, 3 complete, 0 incomplete
    stream     150000   8366728    +5.1  chunked, no content-length
    stream     250000  14166728   +18.0  chunked, no content-length
    stream    1000000  57666732    +8.3  chunked, no content-length

    The port was free: the previous server is dead. Under the same cap this route delivered a million rows for +8.3 MB. The ceiling was the implementation.

14. 14

    Step 14.

    Stop the target by process id.

    P=$(netstat -ano | grep "127.0.0.1:8931 " | grep LISTENING | awk '{print $NF}'); powershell -Command "Stop-Process -Id $P -Force"; sleep 1; netstat -ano | grep ":8931 " | grep -c LISTENING
    

    0

    By id, never by image name. Another server may be running here.

How to read the result

| What you see | What it means | What to do | | --- | --- | --- | | Time to first byte rises with the row count | The export is assembled before it is sent | Expect memory to rise with it. Step 3 read 74, 256 and 474 ms for the same three files step 4 started sending in 4 ms. | | A declared column of - | The response is chunked, so there is no content-length to compare | Count rows and check the last line. Step 7 does both on a file that lost 80 216 rows. | | 200 with a file that stops mid-export | The failure happened after the response headers | Read the exit code, not the status. curl 18 in steps 6 and 9, aborted in step 7. | | 504 with a JSON body | The deadline was reached before the first byte | The status code is usable here and nowhere later. Step 8. | | curl exit 28 | The client gave up, the server is still working | Raise --max-time and re-measure before calling it a server fault. Step 10. | | 408 before any data | A server header or request timeout, not an export timeout | The handler never ran, so no partial file exists. Step 11. | | A reset with no status, then connection refused | The process died | Read the server's own output for a heap limit and an exit code. Step 12 exits 134. |

Common mistakes

Thresholds

What to check next

• How to count rows in a csv file: the count compared here.
• How to compare two csv files: whether those rows are right.
• How to check memory growth under load: the heap question over small requests.
• How to test API timeout handling: the client half of step 10.
• How to test csv import validation: the other side of a short file.

FAQ

What is the csv export limit?

A property of the implementation, not of CSV. At a 64 MB heap cap the same data first failed at 150 000 or 200 000 rows buffered, and passed a million streamed.

How do I find the export row limit?

Raise the row count until it changes behaviour, then repeat the sweep: in ten runs the failing size moved.

Is there a csv row count limit?

No limit in the format. The ceilings measured here are heap, a server deadline and a client timeout.

Why does a long export finish with 200 and a short file?

Because the status goes out with the first byte. In step 9 a deadline fires a second in, and the only marks are curl exit 18 and a missing last line.

Verified