How to check if a cron job is still running

Record a start and an end timestamp for every run, then sweep the ledger for intervals that intersect. node audit.mjs overlap.db overlap reports max concurrent 3 for a 1400 ms job on a 500 ms schedule, and invoices.total 4 after 11 runs, lost 7, which is what two copies of one job cost.

Why check this

Run this on staging sign-off for any service that owns a schedule, and after a deploy puts the scheduler on a second machine. A scheduler has three answers when a run is due and the last one has not finished: drop the instant, queue it, or start it anyway. The third writes one row from two processes. Eleven runs below each added 1 to a counter, which reached 4.

Prerequisites

• Node 22.23.2 produced every figure here. node:sqlite prints an ExperimentalWarning, cut from the output blocks below.
• No cron daemon and no broker. The target is the scheduler below: fire, claim, work, record.
• A ledger with a start and an end timestamp per run. One timestamp per run cannot answer this.
• Both programs in one directory, every command run from it, .db files deleted between experiments.
• Windows PowerShell 5.1.22621.6133 ran step 6. The holder check in step 7 uses process.kill(pid, 0).

// sched.mjs <policy: skip|queue|overlap> <db>
// env: EVERY_MS JOB_MS STOP_MS HOST LOCKDB LOCK_TTL_MS
import { DatabaseSync } from 'node:sqlite';

const [policy, dbPath] = process.argv.slice(2);
const every = Number(process.env.EVERY_MS ?? 500);
const jobMs = Number(process.env.JOB_MS ?? 1400);
const stopMs = Number(process.env.STOP_MS ?? 6000);
const host = process.env.HOST ?? 'host-a';
const ttl = Number(process.env.LOCK_TTL_MS ?? 0);
const holder = `${host}:${process.pid}`;

const db = new DatabaseSync(dbPath);
db.exec(`PRAGMA busy_timeout = 5000;
CREATE TABLE IF NOT EXISTS runs(policy TEXT, host TEXT, pid INT, n INT, due INT, started INT, ended INT);
CREATE TABLE IF NOT EXISTS skipped(policy TEXT, host TEXT, n INT, due INT, reason TEXT);
CREATE TABLE IF NOT EXISTS invoices(id INT PRIMARY KEY, total INT);
INSERT OR IGNORE INTO invoices(id,total) VALUES(1,0);`);

const lockDb = process.env.LOCKDB ? new DatabaseSync(process.env.LOCKDB) : null;
lockDb?.exec(`PRAGMA busy_timeout = 5000;
CREATE TABLE IF NOT EXISTS joblock(name TEXT PRIMARY KEY, holder TEXT, acquired INT)`);

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const t0 = Date.now();
const rel = (t) => ((t - t0) / 1000).toFixed(3);

function acquire(id, due) {
  if (!lockDb) return true;
  try {
    lockDb.prepare('INSERT INTO joblock(name,holder,acquired) VALUES(?,?,?)').run('invoices', holder, Date.now());
    return true;
  } catch {
    const row = lockDb.prepare('SELECT holder,acquired FROM joblock WHERE name=?').get('invoices');
    const age = Date.now() - row.acquired;
    if (ttl && age > ttl) {
      lockDb.prepare('UPDATE joblock SET holder=?,acquired=? WHERE name=?').run(holder, Date.now(), 'invoices');
      console.log(`took over lock from ${row.holder} held ${(age / 1000).toFixed(1)}s`);
      return true;
    }
    console.log(`blocked ${id} lock held by ${row.holder} for ${(age / 1000).toFixed(1)}s`);
    db.prepare('INSERT INTO skipped VALUES(?,?,?,?,?)').run(policy, host, id, due, `lock held by ${row.holder}`);
    return false;
  }
}
const release = () => lockDb?.prepare('DELETE FROM joblock WHERE name=? AND holder=?').run('invoices', holder);

let n = 0;
let active = 0;
let chain = Promise.resolve();

async function job(id, due) {
  active += 1;
  const started = Date.now();
  const before = db.prepare('SELECT total FROM invoices WHERE id=1').get().total;
  await sleep(jobMs);
  db.prepare('UPDATE invoices SET total=? WHERE id=1').run(before + 1);
  const ended = Date.now();
  db.prepare('INSERT INTO runs VALUES(?,?,?,?,?,?,?)').run(policy, host, process.pid, id, due, started, ended);
  console.log(`run ${id} start ${rel(started)} end ${rel(ended)} concurrent ${active}`);
  active -= 1;
  release();
}

function fire() {
  n += 1;
  const id = n;
  const due = Date.now();
  console.log(`fire ${id} at ${rel(due)} active ${active}`);
  if (policy === 'skip' && active > 0) {
    db.prepare('INSERT INTO skipped VALUES(?,?,?,?,?)').run(policy, host, id, due, 'previous run still going');
    console.log(`skip ${id} previous run still going`);
    return;
  }
  if (policy === 'queue') {
    chain = chain.then(() => (acquire(id, due) ? job(id, due) : null));
    return;
  }
  if (acquire(id, due)) job(id, due);
}

console.log(`scheduler pid ${process.pid} policy ${policy} host ${host} every ${every}ms job ${jobMs}ms`);
const timer = setInterval(fire, every);
setTimeout(async () => {
  clearInterval(timer);
  await chain;
  await sleep(jobMs + 200);
  console.log(`stopped, invoices.total ${db.prepare('SELECT total FROM invoices WHERE id=1').get().total}`);
  process.exit(0);
}, stopMs);

// audit.mjs <db> <policy>
import { DatabaseSync } from 'node:sqlite';

const [dbPath, policy] = process.argv.slice(2);
const db = new DatabaseSync(dbPath);
const runs = db.prepare('SELECT * FROM runs WHERE policy=? ORDER BY due').all(policy);
const skips = db.prepare('SELECT * FROM skipped WHERE policy=?').all(policy);
if (runs.length === 0) { console.log('no runs recorded'); process.exit(0); }

const t0 = Math.min(...runs.map((r) => r.due));
const rel = (t) => ((t - t0) / 1000).toFixed(3).padStart(7);

// Sweep line: +1 when a run starts, -1 when it ends, ends first on a tie.
const events = runs.flatMap((r) => [{ t: r.started, d: 1 }, { t: r.ended, d: -1 }]);
events.sort((a, b) => a.t - b.t || a.d - b.d);
let live = 0;
let max = 0;
let peakAt = 0;
for (const e of events) { live += e.d; if (live > max) { max = live; peakAt = e.t; } }

console.log('  n     due   start     end   lag_s  at_start  host:pid');
for (const r of runs) {
  const atStart = runs.filter((o) => o.started <= r.started && o.ended > r.started).length;
  console.log(`${String(r.n).padStart(3)} ${rel(r.due)} ${rel(r.started)} ${rel(r.ended)} ${((r.started - r.due) / 1000).toFixed(3).padStart(7)} ${String(atStart).padStart(9)}  ${r.host}:${r.pid}`);
}
const total = db.prepare('SELECT total FROM invoices WHERE id=1').get().total;
console.log(`due ${runs.length + skips.length} ran ${runs.length} skipped ${skips.length} max concurrent ${max} at ${rel(peakAt).trim()}s`);
console.log(`invoices.total ${total} after ${runs.length} runs, lost ${runs.length - total}`);
for (const r of db.prepare('SELECT reason, count(*) c FROM skipped WHERE policy=? GROUP BY reason').all(policy)) {
  console.log(`skipped ${r.c} ${r.reason}`);
}

The schedule fires every 500 ms and the job takes 1400 ms, so each experiment ends inside 20 seconds. Fire times in step 1 land 502 to 527 ms apart, never the nominal 500 ms, so the audit reads the ledger.

Steps

1. 1

   Step 1.

   Start the scheduler with the policy that fires whether or not the last run has finished.

   node sched.mjs overlap overlap.db
   

   scheduler pid 32720 policy overlap host host-a every 500ms job 1400ms
   fire 1 at 0.514 active 0
   fire 2 at 1.019 active 1
   fire 3 at 1.527 active 2
   run 1 start 0.514 end 1.920 concurrent 3
   fire 4 at 2.041 active 2
   run 2 start 1.019 end 2.430 concurrent 3
   fire 5 at 2.554 active 2
   run 3 start 1.527 end 2.929 concurrent 3
   …
   run 10 start 5.117 end 6.521 concurrent 2
   run 11 start 5.625 end 7.046 concurrent 1
   stopped, invoices.total 4

   active 2 on fire 4 means two runs were already going when a third started.

2. 2

   Step 2.

   Audit the ledger. The scheduler does not have to be running.

   node audit.mjs overlap.db overlap
   

     n     due   start     end   lag_s  at_start  host:pid
   1   0.000   0.000   1.406   0.000         1  host-a:32720
   2   0.505   0.505   1.916   0.000         2  host-a:32720
   3   1.013   1.013   2.415   0.000         3  host-a:32720
   4   1.527   1.527   2.979   0.000         3  host-a:32720
   5   2.040   2.041   3.444   0.001         3  host-a:32720
   6   2.555   2.555   3.970   0.000         3  host-a:32720
   7   3.082   3.085   4.501   0.003         3  host-a:32720
   8   3.584   3.584   4.988   0.000         3  host-a:32720
   9   4.095   4.095   5.509   0.000         3  host-a:32720
   10   4.603   4.603   6.007   0.000         3  host-a:32720
   11   5.111   5.111   6.532   0.000         3  host-a:32720
   due 11 ran 11 skipped 0 max concurrent 3 at 1.013s
   invoices.total 4 after 11 runs, lost 7

   at_start counts the runs already going when each began. The peak is 3, and 7 of 11 increments are gone.

3. 3

   Step 3.

   Audit a ledger written by the policy that drops the instant, node sched.mjs skip skip.db.

   node audit.mjs skip.db skip
   

     n     due   start     end   lag_s  at_start  host:pid
   1   0.000   0.000   1.409   0.000         1  host-a:20532
   4   1.531   1.531   2.943   0.000         1  host-a:20532
   7   3.063   3.063   4.468   0.000         1  host-a:20532
   10   4.588   4.588   6.002   0.000         1  host-a:20532
   due 11 ran 4 skipped 7 max concurrent 1 at 0.000s
   invoices.total 4 after 4 runs, lost 0
   skipped 7 previous run still going

   Four runs from eleven instants, with a reason recorded for each absence.

4. 4

   Step 4.

   Audit a ledger written by the policy that queues the instant, node sched.mjs queue queue.db.

   node audit.mjs queue.db queue
   

     n     due   start     end   lag_s  at_start  host:pid
   1   0.000   0.000   1.408   0.000         1  host-a:44060
   2   0.503   1.411   2.817   0.908         1  host-a:44060
   3   1.017   2.820   4.232   1.803         1  host-a:44060
   4   1.528   4.234   5.638   2.706         1  host-a:44060
   5   2.036   5.641   7.049   3.605         1  host-a:44060
   6   2.548   7.051   8.464   4.503         1  host-a:44060
   7   3.048   8.467   9.880   5.419         1  host-a:44060
   8   3.560   9.884  11.289   6.324         1  host-a:44060
   9   4.072  11.293  12.709   7.221         1  host-a:44060
   10   4.586  12.712  14.125   8.126         1  host-a:44060
   11   5.094  14.128  15.545   9.034         1  host-a:44060
   due 11 ran 11 skipped 0 max concurrent 1 at 0.000s
   invoices.total 11 after 11 runs, lost 0

   Nothing overlaps and nothing is lost. Run 11 started 9.034 s late, and the backlog grows 900 ms per fire.

5. 5

   Step 5.

   Put a lock row in front of the work with LOCKDB=lock-shared.db node sched.mjs overlap locked.db, then audit.

   node audit.mjs locked.db overlap
   

     n     due   start     end   lag_s  at_start  host:pid
   1   0.000   0.003   1.414   0.003         1  host-a:3280
   4   1.535   1.539   2.947   0.004         1  host-a:3280
   7   3.067   3.070   4.483   0.003         1  host-a:3280
   10   4.602   4.605   6.015   0.003         1  host-a:3280
   due 11 ran 4 skipped 7 max concurrent 1 at 0.003s
   invoices.total 4 after 4 runs, lost 0
   skipped 7 lock held by host-a:3280

   The unique primary key on joblock is the whole guard, and every refusal names the holder.

6. 6

   Step 6.

   Kill the holder while it is inside the work.

   $env:LOCKDB='lock-stale.db'; $env:JOB_MS='8000'; $env:EVERY_MS='1000'; $env:STOP_MS='60000'
   $p = Start-Process node -ArgumentList 'sched.mjs','overlap','stale.db' -PassThru -NoNewWindow -RedirectStandardOutput 'stale1.log' -RedirectStandardError 'stale1.err'
   Start-Sleep -Seconds 3; Stop-Process -Id $p.Id -Force; Get-Content stale1.log
   

   scheduler pid 25072 policy overlap host host-a every 1000ms job 8000ms
   fire 1 at 1.017 active 0
   fire 2 at 2.030 active 1
   blocked 2 lock held by host-a:25072 for 1.0s

   Stop-Process -Force runs no handler, so the DELETE inside release never happens.

7. 7

   Step 7.

   Read the lock row and ask the operating system whether its holder exists.

   node -e "const {DatabaseSync}=require('node:sqlite');const r=new DatabaseSync('lock-stale.db').prepare('SELECT * FROM joblock').get();const pid=Number(r.holder.split(':')[1]);let alive=true;try{process.kill(pid,0)}catch(e){alive=e.code}console.log('lock',r.name,'holder',r.holder,'age',((Date.now()-r.acquired)/1000).toFixed(1)+'s','holder alive',alive)"
   

   lock invoices holder host-a:25072 age 277.2s holder alive ESRCH

   ESRCH is no such process. Against a lock a running scheduler held, the same command printed age 7.7s holder alive true.

8. 8

   Step 8.

   Start a fresh scheduler against that lock.

   EVERY_MS=1000 JOB_MS=1400 STOP_MS=4000 LOCKDB=lock-stale.db node sched.mjs overlap stale2.db
   

   scheduler pid 34344 policy overlap host host-a every 1000ms job 1400ms
   fire 1 at 1.014 active 0
   blocked 1 lock held by host-a:25072 for 20.3s
   fire 2 at 2.019 active 0
   blocked 2 lock held by host-a:25072 for 21.3s
   fire 3 at 3.030 active 0
   blocked 3 lock held by host-a:25072 for 22.3s
   stopped, invoices.total 0

   Every instant is refused and the age climbs. A healthy process that will never run reads downstream like Why is my cron job not running.

9. 9

   Step 9.

   Audit one ledger written by two schedulers with their own lock files, HOST=host-a LOCKDB=lock-a.db and HOST=host-b LOCKDB=lock-b.db.

   node audit.mjs twohost-local.db overlap
   

     n     due   start     end   lag_s  at_start  host:pid
   1   0.000   0.004   1.414   0.004         1  host-a:30732
   1   0.015   0.018   1.426   0.003         2  host-b:25364
   4   1.535   1.538   2.947   0.003         1  host-a:30732
   4   1.550   1.553   2.959   0.003         2  host-b:25364
   due 10 ran 4 skipped 6 max concurrent 2 at 0.018s
   invoices.total 2 after 4 runs, lost 2
   skipped 3 lock held by host-a:30732
   skipped 3 lock held by host-b:25364

   Two copies of instant 1 began 14 ms apart on different pids, each refused only by itself.

10. 10

    Step 10.

    Audit the same pair run against one lock both hosts share, LOCKDB=lock-cluster.db.

    node audit.mjs twohost-shared.db overlap
    

      n     due   start     end   lag_s  at_start  host:pid
    1   0.000   0.004   1.421   0.004         1  host-b:26192
    4   1.529   1.533   2.948   0.004         1  host-a:7568
    due 10 ran 2 skipped 8 max concurrent 1 at 0.004s
    invoices.total 2 after 2 runs, lost 0
    skipped 8 lock held by host-a:7568

    Concurrency is 1 and the hosts take turns. The cluster lock costs throughput: 2 runs from 10 instants, against 4 from 11 in step 5.

How to read the result

| What you see | What it means | What to do | | --- | --- | --- | | max concurrent 2 or higher | Two runs of one job overlapped | Take a lock around the work, then require max concurrent 1 from a rerun. | | max concurrent 1 with skipped N previous run still going | The scheduler drops the instant while the last run continues | Decide per job whether that work may be dropped. If it may not, queue the instant. | | max concurrent 1, nothing skipped, lag_s climbing row after row | The scheduler queues instants and the backlog is not draining | Compare the run duration with the interval. A job slower than its schedule never catches up. | | lost N above zero | Two runs read one row and the later write overwrote the earlier | Make the write atomic, or hold the lock across the read and the write. | | no runs recorded while the scheduler is up | A lock is held and never released | Read the lock row for a holder, then check whether that pid exists. | | Rows from two host:pid values interleaved | The lock is per host, not per cluster | Move the lock into storage both hosts write to. | | at_start above 1 on a few rows only | Overlap happens only when the run is slow | Compare those runs' durations with the interval before changing the schedule. |

Common mistakes

What to check next

• Why is my cron job not running: the same absence, counted per instant.
• How to check if a cron job ran: one instant and its last result.
• How to test duplicate job execution: the same work twice from a queue, not from a schedule.
• How to test graceful shutdown: where the lock is released, when the process is asked to stop.
• How to validate cron expression: the interval the run duration is measured against.

FAQ

How to check if a cron job is currently running?

Ask storage, not the process. A run row with a start and no end is a run in progress, and a lock row names its holder. Then check that the holder exists: in step 7 it did not.

How do I keep two copies of one job from running?

Take a lock in storage both machines write to, and release it after the work. Step 5 held concurrency at 1 with a unique row in SQLite. A file per machine does not: step 9 ran two instances while both logs reported success.

Why did eleven runs leave the counter at 4?

Each run read the total, worked for 1400 ms, then wrote what it had read plus one. Three runs overlapped, so the last writer overwrote the other two. Overlap did not double the work here, it destroyed most of it.

Should an instant be queued or dropped?

Queueing loses nothing and adds lag: instant 11 started 9.034 s late. Dropping keeps the schedule honest and loses the work. Choose per job, and record in the ledger which one the scheduler did.

Verified