From 22f9315c817ad982b66cecf6e2a05984fa5fecd5 Mon Sep 17 00:00:00 2001 From: johnnyq Date: Tue, 17 Nov 2026 18:43:26 -0500 Subject: [PATCH] Maintenance Cron: admins see a heartbeat in the top bar when cron is not running, is switched off, or a job failed, is stuck or has fallen behind; the new Cron Watch job emails the admins or set addresses when a problem starts, once a day while it lasts and when it is fixed; health.php answers OK or 503 for an uptime monitor given its key in the address or an X-Health-Key header; the Alerts card sets the emails, sends a test and makes a new key; database 3.7.9 adds the cron alert settings; 26.11.9 changelog entries --- CHANGELOG.md | 6 +- admin/cron.php | 60 ++++++ admin/database_updates/3.7.9.php | 40 ++++ admin/post/cron.php | 101 +++++++++ cron/cron_watch.php | 40 ++++ css/itflow_custom.css | 8 + db.sql | 4 + functions.php | 1 + functions/cron_health.php | 337 +++++++++++++++++++++++++++++++ functions/demo.php | 2 +- functions/email_preview.php | 5 + health.php | 56 +++++ includes/cron_jobs.php | 9 + includes/top_nav.php | 28 +++ 14 files changed, 695 insertions(+), 2 deletions(-) create mode 100644 admin/database_updates/3.7.9.php create mode 100644 cron/cron_watch.php create mode 100644 functions/cron_health.php create mode 100644 health.php diff --git a/CHANGELOG.md b/CHANGELOG.md index f72431532..9b76d987a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ This file documents all notable changes made to ITFlow. ### Upgrading -Update from Maintenance > Update — Queue Update hands the job to cron, which updates the files and applies this release's twelve database changes (3.6.6 to 3.7.8) in the same pass. They add four tables for RMM providers and their alerts, a setting for device actions and one for API keys in the address, a column marking the emails the mail queue clears once sent, a column and a table for agent banners, two columns for ticket statuses with a setting for the "ticket assigned" message, a column on tickets with a setting for the "no response" message, a table for who a ticket's email replies go to, the settings for Quick tickets, columns on recurring tickets and recurring invoices for booking and making them ahead, and two columns on ticket replies for when their time was worked, and change nothing that is there. Nothing else is required. This release has eleven security fixes - see Security. On Apache, ITFlow's own .htaccess now sends security headers: if you show ITFlow inside a frame on another website, it will now refuse to load there. On nginx, add the same headers to your server block if you want them - see Developer Updates. +Update from Maintenance > Update — Queue Update hands the job to cron, which updates the files and applies this release's thirteen database changes (3.6.6 to 3.7.9) in the same pass. They add four tables for RMM providers and their alerts, a setting for device actions and one for API keys in the address, a column marking the emails the mail queue clears once sent, a column and a table for agent banners, two columns for ticket statuses with a setting for the "ticket assigned" message, a column on tickets with a setting for the "no response" message, a table for who a ticket's email replies go to, the settings for Quick tickets, columns on recurring tickets and recurring invoices for booking and making them ahead, two columns on ticket replies for when their time was worked, and the settings for cron alerts, and change nothing that is there. Nothing else is required. This release has eleven security fixes - see Security. On Apache, ITFlow's own .htaccess now sends security headers: if you show ITFlow inside a frame on another website, it will now refuse to load there. On nginx, add the same headers to your server block if you want them - see Developer Updates. ### Security @@ -88,6 +88,9 @@ Update from Maintenance > Update — Queue Update hands the job to cron, which u - Tickets: a note with time worked keeps when the work was done. Under Time worked, When it was worked has the date, start and end - filled in from the timer, ending now, until you type them, and typed times set the time worked, for a visit or a call written up later. The conversation shows it beside the time ("25m (Tue Nov 17, 2:05 - 2:30 PM)"), and Edit changes it: new times set the time worked, and a new time worked alone moves the start. An end before the start runs past midnight. A note with time worked needs a date, start and end to send. From Mountaineer IT's build of ITFlow. - Recurring tickets: on Schedule, Book the work for takes a time and Remote or Onsite - each ticket raised is booked for its run day at that time, with the emails, calendar invite and note booking it on the ticket gives. Make it ahead raises each ticket so many days before its run day, which stays the day the work is for - or so many days before the 1st of its month, so every run in a month is raised on one day (monthly and longer repeats). The window says when the next one will be made, and the list shows the time and days ahead. Raise now books the ticket for its run too. From Mountaineer IT's build of ITFlow. - Recurring invoices: Make it ahead makes each invoice so many days before its date, dated the day it's made, and Due, when made ahead sets it due on its date plus the client's terms, or on its date. With 0 days nothing changes. From Mountaineer IT's build of ITFlow. +- Maintenance > Cron: admins see a heartbeat in the top bar, beside the bell, when cron needs looking at - cron isn't running, it's switched off, or a job failed, is still running two hours after it started, or has fallen behind. Opening it lists what's wrong, with a job's error, and links to Maintenance > Cron. Nobody else sees it, and it's gone once all is well. From Mountaineer IT's build of ITFlow. +- Maintenance > Cron: the new Cron Watch job, every 5 minutes, emails your admins when a job fails, gets stuck or falls behind - once when it starts, again once a day while it lasts, and once more when it's fixed. The Alerts card turns the emails off, sends them to other addresses instead, and sends a test. From Mountaineer IT's build of ITFlow. +- Maintenance > Cron: an email can't say cron has stopped, since cron sends it. The Alerts card has a private health address for an uptime monitor such as Uptime Kuma or UptimeRobot: it answers OK while all is well and an error when cron isn't running or a job is in trouble. Check it every 5 minutes. Make a new address if the old one gets out. From Mountaineer IT's build of ITFlow. ### Bug Fixes @@ -205,6 +208,7 @@ Update from Maintenance > Update — Queue Update hands the job to cron, which u - Database 3.7.6: `config_quick_ticket_on`, `_assign` ('me', 'nobody' or a user id), `_status`, `_priority`, `_category`, `_billable` (-1 is `config_ticket_default_billable`), `_timer` (make, open or off), `_primary`, `_open_tickets` and `_email_client` in `settings`, read by `quickTicketSettings()` (`functions/quick_ticket.php`). `includes/top_nav.php` has the + (an `ajax-modal` of `agent/modals/ticket/ticket_quick.php`, with `agent/js/ticket_quick.js`) and the timers menu (`js/ticket_timers.js`, loaded from `includes/footer.php`). The window posts `add_ticket` with `quick_ticket`, plain-text `details` (made HTML by `quickTicketDetailsHtml()`), `status`, `category_id`, `billable` and `quick_ticket_timer`, the browser's clock in ms; `add_ticket` skips `ticketMailCreated()` when `_email_client` is off and calls `quickTicketAfterCreate()`, which logs the history line and keeps the start in the session for `quickTicketTimerScript()`, printed on the ticket page before `agent/js/ticket_time_tracking.js`. `agent/ajax.php`: `ticket_also_open` takes `work` (who has each, when it last moved, a Work on this one button with `data-ticket-work`, up to 10, and a line when there are none), `get_client_contacts` also returns `contact_phone` and `contact_mobile`, and `ticket_timer_tickets` returns the reference, subject, client and state of the tickets behind the timers the agent can open. The timers read and write the same `-startTime`, `-pausedTime` and `ticket-timer-running-` keys, and on the ticket's own page press its `#startStopTimer`. From Mountaineer IT's build of ITFlow. - Database 3.7.7: `recurring_tickets.recurring_ticket_schedule_time` (NULL books nothing), `_schedule_onsite`, `_lead_days` and `_lead_mode` (days or month), and `recurring_invoices.recurring_invoice_lead_days` and `_due_mode` (terms or next) - `recurringAheadReady()`. The nightly job takes recurring tickets by `recurringTicketDueSql()` and recurring invoices up to `recurring_invoice_lead_days` ahead, made with `recurringInvoiceGenerate()`'s `$early` when the date is still to come. `recurringTicketRunNow()` calls `recurringTicketBook()`, which books through `ticketSetSchedule()` for the run day at the time (today for a missed run), as "Recurring ticket" from the cron; `recurringInvoiceGenerate()` dates an invoice made ahead with lead days due by `recurringInvoiceAheadDue()`. Days ahead are kept short of one step by `recurringTicketLeadDays()` and `recurringInvoiceLeadDays()` (a month is 28 days, 90 at most; 27 before the 1st of the month, month-based repeats only). The fields are `schedule_time`, `schedule_onsite`, `lead_days` and `lead_mode` on the recurring ticket windows (`agent/includes/inc_recurring_ticket_ahead.php`), and `lead_days` and `due_mode` on the recurring invoice form; a post without `lead_days` keeps what is saved. From Mountaineer IT's build of ITFlow. - Database 3.7.8: `ticket_replies.ticket_reply_work_start` and `ticket_reply_work_end`, in company time - `ticketWorkTimesReady()`. `add_ticket_reply` takes `work_date`, `work_start` and `work_end` (`ticketWorkRange()`; an end before the start is the next day, under 24 hours) for a note with time worked, and `ticketWorkTimeWorked()` makes the time worked their length unless it agrees to the minute; without them, or from the bulk reply, the API or email, they stay NULL - it ended when it was posted. `edit_ticket_reply` takes the same fields: changed times set the time worked, the same times with a new time worked move the start, times left as `ticketWorkTimesShown()` filled them on a note without any keep it without, and no time worked clears them. The reply box's time row is Time worked and When it was worked, above Status and Send; `agent/js/ticket_work_times.js` follows `#hours` / `#minutes` / `#seconds` from `ticket_time_tracking.js` (at least a minute), in company time from `#workTimes` `data-now`, and refuses the form in the capture phase, before the timer resets, when a note with time worked has no valid times. From Mountaineer IT's build of ITFlow. +- Database 3.7.9: `config_cron_alert_email` (default 1), `config_cron_alert_to`, `config_cron_health_key` and `config_cron_alert_state` (JSON: `sent`, each problem's key and when it was last emailed; `waiting`, each job that is on and has never run and since when) - `cronHealthReady()`. `functions/cron_health.php`: `cronHealthProblems()` reads the heartbeat, `config_enable_cron` and each enabled registry job's row (not those a manager has paused) into problems with a key, kind (`not_running`, `off`, `failed`, `stuck` after `CRON_HEALTH_STUCK_MINUTES`, `overdue` - past two intervals and at least ten minutes over, or two hours past a daily time; a job never run is late once `waiting` is older than that), text and error. `includes/top_nav.php` shows them to admins as `#itflowCronHealth`. `cron/cron_watch.php` is the `cron_watch` job, every 5 minutes: `cronHealthWatch()` records `waiting`, then emails the problems that are new, a day old or gone with `cronHealthAlert()` through `cronHealthMail()` (`emailQueue()`, to `config_cron_alert_to`, else active admins - `cronHealthRecipients()`), leaving out `not_running` and its own; Admin > Mail previews it as `cron_alert`. `health.php` answers 200 `OK` or 503 with a line per problem, given `config_cron_health_key` as `?key=` or an `X-Health-Key` header, and 403 otherwise. `admin/post/cron.php` has `edit_cron_alerts`, `test_cron_alert` (to the addresses typed) and `renew_cron_health_key`, locked in the demo. From Mountaineer IT's build of ITFlow. ## [26.11.8] diff --git a/admin/cron.php b/admin/cron.php index fe575a31a..68fab59b2 100644 --- a/admin/cron.php +++ b/admin/cron.php @@ -509,6 +509,66 @@ $status_looks = [ + + +
+
+

Alerts

+
+
+

+ When cron stops, or a job fails, gets stuck or falls behind, admins see in the top bar. + The Cron Watch job also emails about it - when it starts, once a day while it lasts, and when it's fixed. +

+ +

Cron Watch is off, so no alerts are emailed - turn it on in Jobs.

+ +

No mail server is set up, so no alerts are emailed - Settings > Mail.

+ +
+ +
+ > + +
+ +
+ + +
+
+ Email addresses, with commas between. Left empty, they go to every active admin. +
+
+ + +
+
+
+

Uptime monitor

+

+ An email can't say cron has stopped - it's cron that sends it. Point an uptime monitor (Uptime Kuma, UptimeRobot...) at this address every 5 minutes: + it answers OK while all is well, and an error (HTTP 503) when cron isn't running or a job is in trouble. Keep it private - it shows which jobs are failing. +

+
+ + + + + + +
+
Or send the key in an X-Health-Key header to /health.php, which keeps it out of the web server's log.
+
+
+ +
diff --git a/admin/database_updates/3.7.9.php b/admin/database_updates/3.7.9.php new file mode 100644 index 000000000..925f928ac --- /dev/null +++ b/admin/database_updates/3.7.9.php @@ -0,0 +1,40 @@ + + * Cron, the key in the health address an uptime monitor checks, and what the + * Cron Watch job last emailed about. + * settings.config_cron_alert_email 1 = email the alerts (the default) + * settings.config_cron_alert_to addresses; empty = every active admin + * settings.config_cron_health_key the health address's key, made when first needed + * settings.config_cron_alert_state JSON - problems emailed, jobs waiting for a first run + * + * Re-runnable: each column is added only where it isn't there yet. + */ + +defined('FROM_DB_UPDATER') || die("Direct file access is not allowed"); + + $itflow_column = function ($table, $column) use ($mysqli) { + $result = mysqli_query($mysqli, "SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '$table' AND COLUMN_NAME = '$column' LIMIT 1"); + return $result && mysqli_num_rows($result) > 0; + }; + + if (!$itflow_column('settings', 'config_cron_alert_email')) { + mysqli_query($mysqli, "ALTER TABLE `settings` ADD `config_cron_alert_email` tinyint(1) NOT NULL DEFAULT 1 AFTER `config_cron_last_dispatch_at`"); + } + + if (!$itflow_column('settings', 'config_cron_alert_to')) { + mysqli_query($mysqli, "ALTER TABLE `settings` ADD `config_cron_alert_to` varchar(500) DEFAULT NULL AFTER `config_cron_alert_email`"); + } + + if (!$itflow_column('settings', 'config_cron_health_key')) { + mysqli_query($mysqli, "ALTER TABLE `settings` ADD `config_cron_health_key` varchar(200) DEFAULT NULL AFTER `config_cron_alert_to`"); + } + + if (!$itflow_column('settings', 'config_cron_alert_state')) { + mysqli_query($mysqli, "ALTER TABLE `settings` ADD `config_cron_alert_state` text DEFAULT NULL AFTER `config_cron_health_key`"); + } diff --git a/admin/post/cron.php b/admin/post/cron.php index 84a4e4922..cf8c353a3 100644 --- a/admin/post/cron.php +++ b/admin/post/cron.php @@ -158,3 +158,104 @@ if (isset($_GET['clear_cron_error'])) { redirect(); } + +/* + * The Alerts card (functions/cron_health.php): whether the Cron Watch job emails, and who to, + * a test email to them, and a new key for the health address an uptime monitor checks. + */ + +if (isset($_POST['edit_cron_alerts'])) { + + validateCSRFToken(); + + if (!cronHealthReady()) { + flashAlert("Cron alerts need the database update - Maintenance > Update.", 'error'); + redirect(); + } + + [$addresses, $not_addresses] = cronHealthAddresses($_POST['cron_alert_to'] ?? ''); + + if ($not_addresses) { + flashAlert("Not an email address: " . escapeHtml(implode(', ', $not_addresses)), 'error'); + redirect(); + } + + $alert_email = isset($_POST['cron_alert_email']) ? 1 : 0; + $alert_to = escapeSql(substr(implode(', ', $addresses), 0, 500)); + + mysqli_query($mysqli, "UPDATE settings SET config_cron_alert_email = $alert_email, config_cron_alert_to = " . ($alert_to === '' ? 'NULL' : "'$alert_to'") . " WHERE company_id = 1"); + + logAudit("Cron", "Edit", "$session_name set cron alerts " . ($alert_email ? 'to email ' . ($alert_to === '' ? 'every active admin' : $alert_to) : 'not to email')); + + flashAlert("Cron alerts saved."); + + redirect(); + +} + +if (isset($_POST['test_cron_alert'])) { + + validateCSRFToken(); + + if (!cronHealthReady()) { + flashAlert("Cron alerts need the database update - Maintenance > Update.", 'error'); + redirect(); + } + + if (!emailReady()) { + flashAlert("Set up outgoing mail first - Admin > Settings > Mail.", 'error'); + redirect(); + } + + // To the addresses in the box, saved or not - so they can be tried before they're saved + [$addresses, $not_addresses] = cronHealthAddresses($_POST['cron_alert_to'] ?? ''); + + if ($not_addresses) { + flashAlert("Not an email address: " . escapeHtml(implode(', ', $not_addresses)), 'error'); + redirect(); + } + + $recipients = cronHealthRecipients(implode(', ', $addresses)); + + if (!$recipients) { + flashAlert("Nobody to send to - add an address, or give an admin an email address.", 'error'); + redirect(); + } + + cronHealthMail("ITFlow cron alerts: test", [ + 'tone' => 'primary', 'title' => 'Cron alerts: test', + 'intro' => ['This is a test from Maintenance > Cron. When a cron job fails, gets stuck or falls behind, the alert comes to this address.'], + ], implode(', ', $addresses)); + + logAudit("Cron", "Edit", "$session_name sent a test cron alert"); + + // Typed but not saved yet - say so, or the alerts still go where they did + $saved = array_map('strtolower', cronHealthAddresses(cronHealthSettings()['config_cron_alert_to'] ?? '')[0]); + $typed = array_map('strtolower', $addresses); + sort($saved); + sort($typed); + + flashAlert("Test queued for " . escapeHtml(implode(', ', array_keys($recipients))) . " - it goes out with the mail queue, within a minute while cron runs." . ($saved !== $typed ? " Save for the alerts to go there too." : '')); + + redirect(); + +} + +if (isset($_GET['renew_cron_health_key'])) { + + validateCSRFToken(); + + if (!cronHealthReady()) { + flashAlert("The health address needs the database update - Maintenance > Update.", 'error'); + redirect(); + } + + cronHealthKey(true); + + logAudit("Cron", "Edit", "$session_name made a new health address for cron"); + + flashAlert("New health address made - the old one no longer works, so give your uptime monitor the new one."); + + redirect(); + +} diff --git a/cron/cron_watch.php b/cron/cron_watch.php new file mode 100644 index 000000000..53c59a97c --- /dev/null +++ b/cron/cron_watch.php @@ -0,0 +1,40 @@ + Update)"); +} + +foreach (cronHealthWatch() as $cron_watch_line) { + echo "$cron_watch_line\n"; +} diff --git a/css/itflow_custom.css b/css/itflow_custom.css index b492f7fb9..c30bbc192 100644 --- a/css/itflow_custom.css +++ b/css/itflow_custom.css @@ -2483,6 +2483,14 @@ body { font-variant-numeric: tabular-nums; font-weight: 600; } +/* Cron health in the top bar (functions/cron_health.php) - a problem a row */ +.itflow-cron-row { + padding: .6rem 1rem; + border-bottom: 1px solid var(--bs-border-color); +} +.itflow-cron-row:last-child { + border-bottom: 0; +} /* Quick ticket's list of the contact's open tickets - a long one scrolls in place */ .itflow-also-open-list { max-height: 15rem; diff --git a/db.sql b/db.sql index 645d60ef4..468e5d9b7 100644 --- a/db.sql +++ b/db.sql @@ -3421,6 +3421,10 @@ CREATE TABLE `settings` ( `config_ticket_ordering` tinyint(1) NOT NULL DEFAULT 0, `config_ticket_moving_columns` tinyint(1) NOT NULL DEFAULT 1, `config_cron_last_dispatch_at` datetime DEFAULT NULL, + `config_cron_alert_email` tinyint(1) NOT NULL DEFAULT 1, + `config_cron_alert_to` varchar(500) DEFAULT NULL, + `config_cron_health_key` varchar(200) DEFAULT NULL, + `config_cron_alert_state` text DEFAULT NULL, `config_backup_retention_days` int(11) NOT NULL DEFAULT 30, `config_backup_retention_count` int(11) NOT NULL DEFAULT 5, `config_backup_cron_type` varchar(20) NOT NULL DEFAULT 'full', diff --git a/functions.php b/functions.php index e8cc96207..8e6c8eeca 100644 --- a/functions.php +++ b/functions.php @@ -113,6 +113,7 @@ require_once __DIR__ . '/functions/quick_ticket.php'; require_once __DIR__ . '/functions/getting_started.php'; require_once __DIR__ . '/functions/settings.php'; require_once __DIR__ . '/functions/mail_queue.php'; +require_once __DIR__ . '/functions/cron_health.php'; require_once __DIR__ . '/functions/managed.php'; require_once __DIR__ . '/functions/demo.php'; diff --git a/functions/cron_health.php b/functions/cron_health.php new file mode 100644 index 000000000..b0c729f32 --- /dev/null +++ b/functions/cron_health.php @@ -0,0 +1,337 @@ + Cron + * already records each job's last run, how it went and its error (cron_jobs); + * this reads them and says so where people look: + * - a warning in the top bar for admins (includes/top_nav.php), + * - an email when a job fails, gets stuck or falls behind, again once a day + * while it lasts, and when it's fixed (the Cron Watch job, cron/cron_watch.php), + * - a health address for an uptime monitor (/health.php) - the only thing + * that can tell anyone cron itself has stopped, since cron sends the email. + * + * Settings (database 3.7.9): config_cron_alert_email (1 = email), + * config_cron_alert_to (addresses; empty = every active admin), + * config_cron_health_key (the health address's key) and config_cron_alert_state + * (JSON: 'sent' - each problem's key => when it was last emailed; 'waiting' - + * each job that's on but has never run => since when). Set on Maintenance > + * Cron's Alerts card (admin/post/cron.php). + */ + +// A job still running after this many minutes counts as stuck +const CRON_HEALTH_STUCK_MINUTES = 120; + +// The dispatcher writes its heartbeat every minute - older than this and cron isn't running +const CRON_HEALTH_HEARTBEAT_SECONDS = 300; + +/** Whether the 3.7.9 update has run - the alert settings are there. */ +function cronHealthReady() { + global $mysqli; + static $ready = null; + if ($ready === null) { + if (defined('CURRENT_DATABASE_VERSION')) { + $version = CURRENT_DATABASE_VERSION; + } else { + $result = mysqli_query($mysqli, "SELECT config_current_database_version FROM settings WHERE company_id = 1"); + $version = $result && ($row = mysqli_fetch_row($result)) ? (string) $row[0] : '0'; + } + $ready = version_compare($version, '3.7.9', '>='); + } + return $ready; +} + +/** The cron settings this reads, fresh from the database (a job runs in a long process). */ +function cronHealthSettings() { + global $mysqli; + $row = mysqli_fetch_assoc(mysqli_query($mysqli, "SELECT config_enable_cron, config_cron_last_dispatch_at, config_cron_alert_email, + config_cron_alert_to, config_cron_health_key, config_cron_alert_state FROM settings WHERE company_id = 1")); + $state = json_decode((string) ($row['config_cron_alert_state'] ?? ''), true); + $state = is_array($state) ? $state : []; + $row['state'] = ['sent' => (array) ($state['sent'] ?? []), 'waiting' => (array) ($state['waiting'] ?? [])]; + return $row; +} + +function cronHealthSaveState(array $state) { + global $mysqli; + $json = escapeSql(json_encode(['sent' => (object) $state['sent'], 'waiting' => (object) $state['waiting']])); + mysqli_query($mysqli, "UPDATE settings SET config_cron_alert_state = '$json' WHERE company_id = 1"); +} + +/* + * The switched-on jobs cron is expected to run, with their rows: name => row + label. + * A job a manager has paused (a read-only or suspended managed instance) isn't expected to. + */ +function cronHealthJobs() { + global $mysqli; + require_once __DIR__ . '/../includes/cron_jobs.php'; + $mode = managedMode(); + $kept = $mode === 'active' ? null : (MANAGED_KEEP_JOBS[$mode] ?? []); + $rows = []; + $result = mysqli_query($mysqli, "SELECT cron_job_name, cron_job_schedule, cron_job_interval_minutes, cron_job_daily_at, cron_job_last_run_at, + cron_job_last_status, cron_job_last_error, cron_job_last_error_at FROM cron_jobs WHERE cron_job_enabled = 1"); + while ($row = mysqli_fetch_assoc($result)) { + $rows[$row['cron_job_name']] = $row; + } + $jobs = []; + foreach (cronJobRegistry() as $entry) { + if (!isset($rows[$entry['name']]) || ($kept !== null && !in_array($entry['name'], $kept, true))) { + continue; + } + $jobs[$entry['name']] = $rows[$entry['name']] + ['label' => $entry['label']]; + } + return $jobs; +} + +/* + * Whether a job has gone too long without a run. An interval job may miss one + * run - late is past two intervals, and never less than ten minutes over. A + * daily job is late two hours after its time went by without a run. + */ +function cronHealthOverdue(array $job) { + $last_run = $job['cron_job_last_run_at'] ? strtotime($job['cron_job_last_run_at']) : 0; + if ($job['cron_job_schedule'] === 'Daily') { + $occurrence = strtotime(date('Y-m-d') . ' ' . substr((string) $job['cron_job_daily_at'], 0, 5) . ':00'); + if ($occurrence > time()) { + $occurrence -= 86400; + } + return $last_run < $occurrence && time() - $occurrence > 2 * 3600; + } + $interval = max(1, intval($job['cron_job_interval_minutes'])); + return time() - $last_run > ($interval + max(10, $interval)) * 60; +} + +/** The longest a job may wait for its first run, in seconds - as long as it may go between runs. */ +function cronHealthFirstRunGap(array $job) { + if ($job['cron_job_schedule'] === 'Daily') { + return 26 * 3600; + } + $interval = max(1, intval($job['cron_job_interval_minutes'])); + return ($interval + max(10, $interval)) * 60; +} + +/* + * What's wrong, as a list of problems - empty when all is well. Each: + * key the same for as long as the problem lasts (what the emails go by) + * job the job's name, or '' for cron itself + * label the job's label, or 'Cron' + * kind not_running, off, failed, stuck or overdue + * text a sentence for people + * detail the job's error, or '' + * $fresh reads again - the answer is kept for the rest of the request. + */ +function cronHealthProblems($fresh = false) { + static $problems = null; + if ($problems !== null && !$fresh) { + return $problems; + } + $problems = []; + if (!cronHealthReady()) { + return $problems; + } + + $settings = cronHealthSettings(); + $heartbeat = $settings['config_cron_last_dispatch_at'] ?? null; + $running = $heartbeat && time() - strtotime($heartbeat) < CRON_HEALTH_HEARTBEAT_SECONDS; + if (!$running) { + $problems[] = ['key' => 'not_running', 'job' => '', 'label' => 'Cron', 'kind' => 'not_running', 'detail' => '', + 'text' => ($heartbeat ? 'Cron isn\'t running - it last ran ' . timeAgo($heartbeat) . '.' : 'Cron has never run on this server.') + . ' No email goes out, no email becomes a ticket and nothing is invoiced until it does.']; + } + if (intval($settings['config_enable_cron'] ?? 0) !== 1) { + // Every job stops itself, so none of them is late either + $problems[] = ['key' => 'off', 'job' => '', 'label' => 'Cron', 'kind' => 'off', 'detail' => '', + 'text' => 'Cron is switched off on Maintenance > Cron - every job stops itself until it\'s turned back on.']; + return $problems; + } + + foreach (cronHealthJobs() as $name => $job) { + $label = $job['label']; + $status = (string) $job['cron_job_last_status']; + + if (!$job['cron_job_last_run_at']) { + // Never run: late once it has waited longer than it may go between runs, counted from when the + // Cron Watch job first saw it waiting (cron_jobs keeps no "switched on since") + $waiting = intval($settings['state']['waiting'][$name] ?? 0); + if ($running && $waiting && time() - $waiting > cronHealthFirstRunGap($job)) { + $problems[] = ['key' => "never:$name", 'job' => $name, 'label' => $label, 'kind' => 'overdue', 'detail' => '', + 'text' => "$label hasn't run yet - it has waited " . preg_replace('/ ago$/', '', timeAgo(date('Y-m-d H:i:s', $waiting))) . ' for its first run.']; + } + continue; + } + + if ($status === 'Failed') { + $problems[] = ['key' => "failed:$name", 'job' => $name, 'label' => $label, 'kind' => 'failed', + 'text' => "$label failed " . timeAgo($job['cron_job_last_error_at'] ?: $job['cron_job_last_run_at']) . '.', + 'detail' => mb_substr(trim((string) $job['cron_job_last_error']), 0, 500)]; + } elseif ($status === 'Running' && time() - strtotime($job['cron_job_last_run_at']) > CRON_HEALTH_STUCK_MINUTES * 60) { + $problems[] = ['key' => "stuck:$name", 'job' => $name, 'label' => $label, 'kind' => 'stuck', 'detail' => '', + 'text' => "$label started " . timeAgo($job['cron_job_last_run_at']) . ' and is still running - it may have stopped part way.']; + } elseif ($running && cronHealthOverdue($job)) { + // Late only means something while cron is running - otherwise "isn't running" says it all + $problems[] = ['key' => "overdue:$name", 'job' => $name, 'label' => $label, 'kind' => 'overdue', 'detail' => '', + 'text' => "$label is behind - it last ran " . timeAgo($job['cron_job_last_run_at']) . '.']; + } + } + return $problems; +} + +/** Who the alerts go to: name keyed by address - the addresses set, or every active admin with one. */ +function cronHealthRecipients($to = null) { + global $mysqli; + $to = $to ?? (string) (cronHealthReady() ? (cronHealthSettings()['config_cron_alert_to'] ?? '') : ''); + $set = cronHealthAddresses($to)[0]; + if ($set) { + return array_fill_keys($set, ''); + } + $recipients = []; + $result = mysqli_query($mysqli, "SELECT user_name, user_email FROM users JOIN user_roles ON role_id = user_role_id + WHERE role_is_admin = 1 AND user_type = 1 AND user_status = 1 AND user_archived_at IS NULL ORDER BY user_name"); + while ($row = mysqli_fetch_assoc($result)) { + if (filter_var($row['user_email'], FILTER_VALIDATE_EMAIL)) { + $recipients[$row['user_email']] = $row['user_name']; + } + } + return $recipients; +} + +/** Addresses typed with commas, semicolons or spaces between: [good, not addresses]. */ +function cronHealthAddresses($text) { + $good = $bad = []; + foreach (preg_split('/[\s,;]+/', trim((string) $text)) as $address) { + if ($address === '') { + continue; + } + if (filter_var($address, FILTER_VALIDATE_EMAIL)) { + $good[strtolower($address)] = $address; + } else { + $bad[] = $address; + } + } + return [array_values($good), $bad]; +} + +/** Queue an alert to everyone it goes to - or to $to's addresses (a test of ones typed). Returns how many it was queued for. */ +function cronHealthMail($subject, array $email, $to = null) { + global $config_base_url; + $to = $to ?? (string) (cronHealthSettings()['config_cron_alert_to'] ?? ''); + $footnote = cronHealthAddresses($to)[0] ? 'This address is set to get cron alerts on Maintenance > Cron.' : 'You get this as an admin of ITFlow - Maintenance > Cron says who cron alerts go to.'; + $recipients = []; + foreach (cronHealthRecipients($to) as $address => $name) { + $recipients[] = ['email' => $address, 'name' => $name, 'footnote' => $footnote]; + } + if (!$recipients) { + return 0; + } + $email += ['kind' => 'Cron', 'button' => ['label' => 'Open Maintenance > Cron', 'url' => 'https://' . $config_base_url . '/admin/cron.php']]; + return count(emailQueue($recipients, $subject, $email)); +} + +/* + * The Cron Watch job: email about problems that are new, still there a day + * later, or gone. Cron is running when this runs, so "isn't running" never + * comes up here - that's what the health address is for. Also notes when it + * first saw each job waiting for its first run. Returns lines for the log. + */ +function cronHealthWatch() { + $settings = cronHealthSettings(); + $state = $settings['state']; + + // Jobs that are on and have never run: since when (kept), the rest let go + $waiting = []; + foreach (cronHealthJobs() as $name => $job) { + if (!$job['cron_job_last_run_at']) { + $waiting[$name] = intval($state['waiting'][$name] ?? 0) ?: time(); + } + } + $state['waiting'] = $waiting; + cronHealthSaveState($state); + + $problems = array_values(array_filter(cronHealthProblems(true), function ($problem) { + return $problem['kind'] !== 'not_running' && $problem['job'] !== 'cron_watch'; + })); + + $now = time(); + $new = $reminders = $sent = []; + foreach ($problems as $problem) { + $last = intval($state['sent'][$problem['key']] ?? 0); + if (!$last) { + $new[] = $problem; + $sent[$problem['key']] = $now; + } elseif ($now - $last >= 86400) { + $reminders[] = $problem; + $sent[$problem['key']] = $now; + } else { + $sent[$problem['key']] = $last; + } + } + $fixed = array_diff_key($state['sent'], $sent); + $state['sent'] = $sent; + cronHealthSaveState($state); + + if (!$new && !$reminders && !$fixed) { + return [$problems ? count($problems) . ' problem(s), already emailed' : 'All well']; + } + if (intval($settings['config_cron_alert_email'] ?? 1) !== 1) { + return ['Email alerts are off - ' . count($problems) . ' problem(s)']; + } + if (!emailReady()) { + return ['No mail server is set up - ' . count($problems) . ' problem(s)']; + } + + $count = cronHealthAlert($new, $reminders, count($fixed), count($problems)); + return ['Emailed ' . $count . ' - new ' . count($new) . ', reminders ' . count($reminders) . ', fixed ' . count($fixed)]; +} + +/* + * The alert email: the problems that are new and those still there a day + * later, how many were fixed, and how many there are now - an all clear when + * none. Returns how many it was queued for. + */ +function cronHealthAlert(array $new, array $reminders, $fixed, $total) { + $list = function (array $items) { + $html = '
    '; + foreach ($items as $problem) { + $html .= '
  • ' . escapeHtml($problem['text']) . ($problem['detail'] !== '' ? '
    ' . escapeHtml($problem['detail']) . '' : '') . '
  • '; + } + return $html . '
'; + }; + if (!$total) { + return cronHealthMail('ITFlow cron: all clear', [ + 'tone' => 'success', 'title' => 'Cron: all clear', + 'intro' => ['Every cron job is keeping up again - nothing needs looking at.'], + ]); + } + $content = ''; + if ($new) { + $content .= '

New:

' . $list($new); + } + if ($reminders) { + $content .= '

Still not fixed:

' . $list($reminders); + } + if ($fixed) { + $content .= '

Fixed since the last email: ' . $fixed . ' problem' . ($fixed === 1 ? '' : 's') . '.

'; + } + return cronHealthMail("ITFlow cron needs attention ($total)", [ + 'tone' => 'danger', 'title' => 'Cron needs attention', + 'intro' => [$total === 1 ? 'A cron job on ITFlow needs looking at.' : "$total cron jobs on ITFlow need looking at."], + 'content' => $content, + ]); +} + +/** The key in the health address - made the first time it's asked for, or new with $renew. */ +function cronHealthKey($renew = false) { + global $mysqli; + $key = (string) (cronHealthSettings()['config_cron_health_key'] ?? ''); + if ($key === '' || $renew) { + $key = bin2hex(random_bytes(20)); + mysqli_query($mysqli, "UPDATE settings SET config_cron_health_key = '$key' WHERE company_id = 1"); + } + return $key; +} + +function cronHealthUrl() { + global $config_base_url; + return 'https://' . $config_base_url . '/health.php?key=' . cronHealthKey(); +} diff --git a/functions/demo.php b/functions/demo.php index 16706e2ad..da1c19b00 100644 --- a/functions/demo.php +++ b/functions/demo.php @@ -79,7 +79,7 @@ const DEMO_MODE_LOCKS = [ ], [ 'message' => "Cron can't be changed in the demo - it runs the nightly reset.", - 'keys' => ['clear_cron_error', 'disable_cron', 'disable_cron_job', 'edit_cron_job', 'enable_cron', 'enable_cron_job', 'run_cron_job'], + 'keys' => ['clear_cron_error', 'disable_cron', 'disable_cron_job', 'edit_cron_alerts', 'edit_cron_job', 'enable_cron', 'enable_cron_job', 'renew_cron_health_key', 'run_cron_job', 'test_cron_alert'], ], [ 'message' => "Card payments and payment providers are turned off in the demo.", diff --git a/functions/email_preview.php b/functions/email_preview.php index 8f2162e53..239cfa4a5 100644 --- a/functions/email_preview.php +++ b/functions/email_preview.php @@ -408,6 +408,11 @@ function emailPreviewCatalog() { }], // ---- Other + 'cron_alert' => ['group' => 'Other', 'label' => 'Cron needs attention', 'to' => 'Every admin, or the addresses on Maintenance > Cron', 'when' => 'A cron job fails, gets stuck or falls behind, again a day later, and when it is fixed - the Cron Watch job', + 'run' => function ($s) { + cronHealthAlert([['text' => 'Ticket SLA Monitor failed 3 minutes ago.', 'detail' => 'SQLSTATE[HY000]: General error: 2006 MySQL server has gone away']], + [['text' => 'Nightly Tasks is behind - it last ran 2 days ago.', 'detail' => '']], 1, 2); + }], 'bulk' => ['group' => 'Other', 'label' => 'Email to many contacts', 'to' => 'The contacts picked', 'when' => 'Email on the contacts or clients list', 'run' => function ($s) { emailBulkSend([['email' => $s['alex_email'], 'name' => $s['alex_name'], 'footnote' => 'You received this as a contact for ' . $s['client'] . ' with ' . (billingCompany()['company_name'] ?? '') . '.']], diff --git a/health.php b/health.php new file mode 100644 index 000000000..24e42248b --- /dev/null +++ b/health.php @@ -0,0 +1,56 @@ + answers 200 "OK" while cron is running and every + * job that's on is keeping up, and 503 with a line for each problem otherwise + * (no error text). Point a monitor (Uptime Kuma, UptimeRobot...) at it every + * 5 minutes: it's the only thing that can tell anyone cron itself has stopped. + * The key can also come in an X-Health-Key header, which keeps it out of the + * web server's log. The address is on Maintenance > Cron, where it can be + * renewed; a wrong key, or none made yet, is a 403. + */ + +header('Content-Type: text/plain; charset=utf-8'); +header('Cache-Control: no-store'); +header('X-Robots-Tag: noindex'); + +if (!file_exists(__DIR__ . '/config.php')) { + http_response_code(503); + exit("PROBLEM\n- ITFlow is not set up\n"); +} + +require_once __DIR__ . '/config.php'; +require_once __DIR__ . '/includes/inc_set_timezone.php'; +require_once __DIR__ . '/functions.php'; + +try { + $health_key = cronHealthReady() ? (string) (cronHealthSettings()['config_cron_health_key'] ?? '') : ''; +} catch (Throwable $e) { + http_response_code(503); + exit("PROBLEM\n- The health check could not read the database\n"); +} + +$health_given = (string) ($_SERVER['HTTP_X_HEALTH_KEY'] ?? ($_GET['key'] ?? '')); +if ($health_key === '' || !hash_equals($health_key, $health_given)) { + http_response_code(403); + exit("Forbidden\n"); +} + +try { + $health_problems = cronHealthProblems(); +} catch (Throwable $e) { + http_response_code(503); + exit("PROBLEM\n- The health check could not read the database\n"); +} + +if (!$health_problems) { + exit("OK\n"); +} + +http_response_code(503); +echo "PROBLEM\n"; +foreach ($health_problems as $health_problem) { + echo "- {$health_problem['label']}: {$health_problem['kind']}\n"; +} diff --git a/includes/cron_jobs.php b/includes/cron_jobs.php index 23cdf6aa1..141e9d6e6 100644 --- a/includes/cron_jobs.php +++ b/includes/cron_jobs.php @@ -203,6 +203,15 @@ function cronJobRegistry(): array 'schedule' => 'Daily', 'daily_at' => '02:30', ], + [ + 'name' => 'cron_watch', + 'label' => 'Cron Watch', + 'icon' => 'fa-heartbeat', + 'script' => 'cron_watch.php', + 'description' => 'Emails the admins when a job here fails, gets stuck or falls behind, once a day while it lasts, and when it\'s fixed - set on the Alerts card.', + 'schedule' => 'Interval', + 'interval_minutes' => 5, + ], [ 'name' => 'app_update', 'label' => 'Application Update', diff --git a/includes/top_nav.php b/includes/top_nav.php index 372763f14..2469579d1 100644 --- a/includes/top_nav.php +++ b/includes/top_nav.php @@ -66,6 +66,34 @@ + + + + + +