From 7e2971b6106f6327a60bc45b3405dd58d59fa02b Mon Sep 17 00:00:00 2001 From: johnnyq Date: Tue, 20 Oct 2026 11:34:23 -0400 Subject: [PATCH] Mail queue stage 1: rebuild the mail queue list with Waiting, Sending, Retrying, Failed, Sent and Cancelled tabs and counts, a status line per email, a banner when sending is off, cron is off or not running, the job is disabled or mail is waiting too long, and Retry now, Cancel and Delete per email and in bulk applied only where possible, add a Cancelled status, record the failure time for bad addresses, share the retry limits between the page and the worker, and fix the theme favicon reset failing on its audit entry --- CHANGELOG.md | 7 + admin/mail_queue.php | 385 ++++++++++++++++++++-------------- admin/post/mail_queue.php | 130 +++++------- admin/post/settings_theme.php | 2 +- cron/mail_queue.php | 15 +- cron/nightly_tasks.php | 2 +- functions.php | 1 + functions/mail_queue.php | 193 +++++++++++++++++ 8 files changed, 482 insertions(+), 253 deletions(-) create mode 100644 functions/mail_queue.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e14457a1..72d1abf5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1029,6 +1029,10 @@ If you are coming from 26.08 or earlier, the database update still has to be run - Tickets, Projects, Quotes, Invoices and billing, and Defaults are in it too. Each prefix and next number shows the number the next one will get as it is typed, and the last one made; a number already used is said at once, and can't be saved. Tickets is in sections - numbering, email to ticket, new tickets, time and billing, closing, and the tickets page - with what happens to an email from an unknown sender when they are left out. Invoices and billing holds numbering for invoices and recurring invoices, the footer and tax ID, late fees (with a warning when reminders are off, as no fee is added then), who hears of a payment made online, reminders and statements, billing time, billable expenses, and trips and mileage. Defaults is grouped into opening ITFlow, new clients and accounts; the start page can be any page of ITFlow typed in, and an account archived since it was picked stays picked. - Notifications, Security, Client portal, Single sign-on, Mail and Logs complete it. Notifications is no longer a table of mostly empty columns: alerts for your team, and the emails clients get by themselves, each saying when it goes - with a warning while mail sending is off, and reminders shown as they are, set in Invoices and billing. Security shows the address agents sign in at as the login key is typed, and requiring a key with none set is refused rather than quietly switched off. Single sign-on gives the redirect address to register in Microsoft Entra, with a copy button, and no longer puts the client secret into the page - left empty, it stays as saved. Mail keeps its tabs, each with the save bar. - Settings are in the search window (Ctrl K) for admins: type a setting - late fee, login key, smtp - and pick it to go straight to it. +- **Mail Queue** (rebuilt) + - Tabs for where each email is - Waiting, Sending, Retrying, Failed, Sent and Cancelled - with how many in each, following the search. Each email says where it stands: scheduled for when, tried 2 of 4 and when it is tried next, gave up, not tried again (a bad address), sent when, or cancelled. + - What is stopping mail going is at the top: sending not set up, cron switched off or not running, the Mail Queue job switched off, or email waiting longer than it should - with the job's last error. When nothing is, it says when the job last ran. + - Retry now starts an email over - a full set of tries, at the next run - for a failed or cancelled one. Cancel is its own status now, rather than Failed. Retry now, Cancel and Delete are on each email's menu and in Bulk Action, done only to the ones they can be, saying how many and why not the rest. ### Bug Fixes @@ -1180,6 +1184,8 @@ If you are coming from 26.08 or earlier, the database update still has to be run - Settings > Single sign-on: the Microsoft Entra client secret was put into the page, in the password field's value. - Settings > Security: requiring a login key with none set saved as not required, saying it had been saved. - Settings > Notifications, and recurring invoices: the note that recurring invoices are made as drafts pointed to the billing settings, but the switch is in Notifications. +- Mail Queue: cancelling an email marked it Failed, and an email to a bad address or with no from address showed Failed with no time. +- Mail Queue: Resend was only offered once all four tries were used up, and the next failure quietly gave a resent email three more. - `setup_cli.php`: a name with an apostrophe in it stopped the install part-way, a password with a space at either end couldn't sign in, and it wrote config.php before knowing the tables could be made - over a database that already had tables in it, too. ### Developer Updates @@ -1744,6 +1750,7 @@ If you are coming from 26.08 or earlier, the database update still has to be run - Settings: `functions/settings.php` - `settingsPages()` (the menu, the overview and each page's header), `settingsFields()` (every setting: its page, section, label, kind and what else it is searched by - Find a setting and the audit log read it), and the frame: `settingsPageStart()`, `settingsSectionStart()`/`settingsSectionEnd()`, `settingsInput()`, `settingsSelect()`, `settingsSwitch()`, `settingsTextarea()`, `settingsField()` and `settingsSaveBar()`. Handlers read with `settingsCurrent()`, refuse with `settingsRefuse($page, $errors)` (the form comes back once, from `$_SESSION['settings_kept']`), and save with `settingsSave()`, or `settingsWrite()` per table and `settingsSaid()`, which write only what changed and log it. New `admin/settings.php` and `js/settings.js` (loaded on admin pages: `form[data-settings-form]`, `#setting=`, Find a setting, picture previews, the theme preview and the Localization sample). The Settings menu is built from `settingsPages()`. Form fields renamed to their columns on Company details (`company_*`), Localization (`company_locale`, `company_currency`, `config_timezone`) and Theme (`config_theme`, `favicon`). The Preferences light/dark swatch is `.itflow-mode-swatch`. - Settings: Tickets, Projects, Quotes, Invoices and billing and Defaults use the frame. New in `functions/settings.php`: `settingsNumbering($kind)`, `settingsNumberInUse()`, `settingsNumberingCheck($kind, $post, $saved)` (prefix and next number checked against the highest in use with that prefix - skipped when both are as saved, so a clash already there doesn't block the rest of the page) and `settingsNumberingFields()` (with `[data-settings-next]` in `js/settings.js`, and `[data-settings-depends-on=name]` to fade fields while a switch is off); `settingsEmailProblem()`; `settingsStartPages()`, `settingsStartPageProblem()` and `settingsNetTerms()`. `settingsSectionStart()` takes the section's element id, so `#reminders`, `#billing-time`, `#billable-expenses` and `#mileage` still work; `settingsSelect()` takes `options_html`, `append` and `search => false`; `settingsSwitch()` takes `id`. `settingsSame()` treats NULL as 0 or empty and ignores Windows line ends; the log names accounts, calendars and categories. Values kept from a refused save are only used for five minutes. These handlers keep a setting that isn't sent (`$_POST + saved`), a switch not sent being off. Defaults' fields are renamed to their columns (`config_start_page`, `config_default_*`). The start page is escaped where it is printed. - Settings: Notifications, Security, Client portal, Single sign-on (`admin/identity_providers.php`), Mail and Logs use the frame. `settingsSearch($query, $limit)` scores settings as Find a setting does, leaving out pages for a module that is off; `agent/ajax.php?global_search` returns them as `settings` for admins and `js/global_search.js` shows them under Settings. Single sign-on's fields are renamed to their columns (`config_azure_client_id`, `config_azure_client_secret`). A field marked `quiet` in `settingsFields()` is logged as changed without its value (the login key). The Logs handler is unchanged; Mail's handlers too. +- Mail Queue: new `functions/mail_queue.php` - `MAIL_QUEUE_TRIES` (4), `MAIL_QUEUE_RETRY_MINUTES` (30) and `MAIL_QUEUE_KEEP_DAYS` (90), which `cron/mail_queue.php`'s retry pass and the nightly clean-up now use too; `mailQueueTabs()`, `mailQueueState($row)`, `mailQueueCan($row, $action)`, `mailQueueAct($action, $ids)` and `mailQueueHealth()`. `email_status` 4 is Cancelled - never selected by the worker. Retry now sets status 0, no attempts and no failure time. Handlers: `retry_mail`, `cancel_mail`, `delete_mail` (GET with the CSRF token) and `bulk_retry_emails`, `bulk_cancel_emails`, `bulk_delete_emails`; `send_failed_mail` still works. The worker sets `email_failed_at` on the bad sender, bad recipient and no-MX paths. - `js/app.js`: the four `tinymce.init()` calls go through `itflowTinyMce`, which does nothing where TinyMCE isn't loaded. ### Library Updates diff --git a/admin/mail_queue.php b/admin/mail_queue.php index 141c2c8b0..87dce8460 100644 --- a/admin/mail_queue.php +++ b/admin/mail_queue.php @@ -1,198 +1,257 @@ Mail Queue - every email ITFlow sends, from being queued to + * sent, and what came of it (functions/mail_queue.php says what each status + * is). The tabs pick Waiting, Sending, Retrying, Failed, Sent or Cancelled, + * each with how many; at the top, anything stopping the queue moving - + * sending switched off, cron not running, the job off, or email waiting + * longer than it should. An email can be retried from the start, cancelled + * or deleted, one at a time or several (admin/post/mail_queue.php), and + * opened to see it (modals/mail_queue/mail_queue_message_view.php). + */ + // Default Column Sortby Filter -$sort = "email_id"; +$sort = "email_queued_at"; $order = "DESC"; require_once "includes/inc_all_admin.php"; -$sql = mysqli_query( - $mysqli, - "SELECT SQL_CALC_FOUND_ROWS email_attempts, email_failed_at, email_from, email_from_name, email_id, email_queued_at, - email_recipient, email_recipient_name, email_sent_at, email_status, email_subject FROM email_queue - WHERE (email_id LIKE '%$q%' OR email_from LIKE '%$q%' OR email_from_name LIKE '%$q%' OR email_recipient LIKE '%$q%' OR email_recipient_name LIKE '%$q%' OR email_subject LIKE '%$q%') - AND DATE(email_queued_at) BETWEEN '$dtf' AND '$dtt' - ORDER BY $sort $order LIMIT $record_from, $record_to" -); +$page_title = 'Mail Queue'; + +$tabs = mailQueueTabs(); +$show = (string) ($_GET['show'] ?? ''); +$show = isset($tabs[$show]) ? $show : ''; + +// Only these sort - an unknown column would take the page down +$sortable_columns = [ + 'email_queued_at' => "email_queued_at $order", + 'email_recipient' => "email_recipient $order, email_queued_at DESC", + 'email_subject' => "email_subject $order, email_queued_at DESC", + 'email_status' => "FIELD(email_status, 2, 1, 0, 4, 3) $order, email_attempts DESC, email_queued_at DESC", +]; +if (!isset($sortable_columns[$sort])) { + $sort = 'email_queued_at'; +} + +$search = "(email_id LIKE '%$q%' OR email_from LIKE '%$q%' OR email_from_name LIKE '%$q%' OR email_recipient LIKE '%$q%' OR email_recipient_name LIKE '%$q%' OR email_subject LIKE '%$q%') + AND DATE(email_queued_at) BETWEEN '$dtf' AND '$dtt'"; + +$sql = mysqli_query($mysqli, "SELECT SQL_CALC_FOUND_ROWS email_id, email_status, email_attempts, email_from, email_from_name, email_recipient, email_recipient_name, + email_subject, email_queued_at, email_failed_at, email_sent_at, COALESCE(email_attachments, '') <> '' AS has_attachments, COALESCE(email_cal_str, '') <> '' AS has_invite + FROM email_queue + WHERE $search AND {$tabs[$show][2]} + ORDER BY {$sortable_columns[$sort]}, email_id DESC + LIMIT $record_from, $record_to"); $num_rows = mysqli_fetch_row(mysqli_query($mysqli, "SELECT FOUND_ROWS()")); +// How many in each tab, as searched +$count_sql = []; +foreach ($tabs as $key => [, , $where]) { + $count_sql[] = "COALESCE(SUM($where), 0)"; +} +$counts = array_combine(array_keys($tabs), array_map('intval', mysqli_fetch_row(mysqli_query($mysqli, "SELECT " . implode(', ', $count_sql) . " FROM email_queue WHERE $search")))); + +$health = mailQueueHealth(); + +$sort_link = function ($column, $label) use ($url_query_strings_sort, $sort, $disp, $order_icon) { + return '' + . $label . ($sort === $column ? " $order_icon" : '') . ''; +}; +$pill_url = function ($show) { + return '?' . http_build_query(array_filter(['q' => $_GET['q'] ?? '', 'show' => $show, 'canned_date' => $_GET['canned_date'] ?? '', 'dtf' => $_GET['dtf'] ?? '', 'dtt' => $_GET['dtt'] ?? ''])); +}; +$person = function ($name, $address) { + $name = trim((string) $name); + $address = (string) $address; + if ($address === '') { + return ($name !== '' ? '' . escapeHtml($name) . ' ' : '') . 'no address'; + } + if ($name === '' || strcasecmp($name, $address) === 0) { + return '' . escapeHtml($address) . ''; + } + return '' . escapeHtml($name) . ' ' . escapeHtml($address) . ''; +}; +$csrf = $_SESSION['csrf_token']; +$filtered_dates = isset($_GET['dtf']) && $_GET['dtf'] !== '' && $_GET['dtf'] !== '1970-01-01'; + ?> -
-
-

Email Queue

+
+
+

Mail Queue

+
+ Send a test
-
-
-
-
-
- - - -
+
+ + +
+ +
-
+ +
+ + +
+ + +
+
+
+ + +
-
- +
+
+
-
" id="advancedFilter"> -
-
-
- - - - - -
+
+
+
+
+ +
+ +
+ + +
- -
-
- - -
- - "> - - - - - - - - - - - - - Queued"; - } elseif($email_status == 1) { - $email_status_display = "
Sending
"; - } elseif($email_status == 2) { - $email_status_display = "
Failed
$email_failed_at"; - } else { - $email_status_display = "
Sent
$email_sent_at"; - } - - ?> - - - - - - - - - - - - - - - -
-
- -
- 		- - Queued - - - - From - - - - To - - - - Subject - - - - Status - - - - Attempts - - Action
- -
- -
- - 		$email_from_name" ?>$email_recipient_name" ?> - - - - - - 3) { ?> - - - - - - - - -
- - 'emails', - 'icon' => 'fa-inbox', - 'title' => 'Nothing in the mail queue', - 'text' => 'The email ITFlow sends waits here until it goes.', - ]; - ?> - + + +
+
+
+ +
+ + "> + + + + + + + + + + + (no subject)'; + $view = 'data-modal-size="lg" data-modal-url="modals/mail_queue/mail_queue_message_view.php?id=' . $email_id . '"'; + $from = $person($row['email_from_name'], $row['email_from']); + ?> + + + + + + + + + + +
+
+ +
+
+ +
+ +
+ + 		+ + +
+ + 		+
+ +
>
+
+ 		+ > + + +
From
+ 		+ +
#
+ 		+
+ +
+ >View + + Retry now + + + Cancel + + +
+ Delete + +
+
+
+
+
+ + 'emails', + 'icon' => 'fa-inbox', + 'title' => $show !== '' ? 'Nothing ' . strtolower($tabs[$show][0]) : 'Nothing in the mail queue', + 'text' => 'The email ITFlow sends waits here until it goes, and stays ' . MAIL_QUEUE_KEEP_DAYS . ' days after.', + ]; + ?> + +
+ Mail Queue (admin/mail_queue.php): retry an email from the + * start, cancel it or delete it - one (retry_mail, cancel_mail, delete_mail) + * or the ones ticked (bulk_retry_emails, bulk_cancel_emails, + * bulk_delete_emails). Each is only done where it can be (mailQueueAct(), + * functions/mail_queue.php) - a sent email can't be retried, as what it said + * was cleared on sending, and one going this minute is left alone - and the + * message says how many it was done to, and why any weren't. + */ + defined('FROM_POST_HANDLER') || die("Direct file access is not allowed"); +$mail_queue_actions = [ + 'retry' => ['retry_mail', 'bulk_retry_emails', 'retried', 'Retry', ['it goes at the next run', 'they go at the next run'], 'A sent one can\'t be - what it said was cleared on sending - nor one waiting or going now.'], + 'cancel' => ['cancel_mail', 'bulk_cancel_emails', 'cancelled', 'Cancel', ['it won\'t be sent', 'they won\'t be sent'], 'Only one still to be sent or tried again can be.'], + 'delete' => ['delete_mail', 'bulk_delete_emails', 'deleted', 'Delete', ['', ''], 'One going this minute can\'t be.'], +]; +// The Resend link before this page was rebuilt if (isset($_GET['send_failed_mail'])) { + $_GET['retry_mail'] = $_GET['send_failed_mail']; +} + +foreach ($mail_queue_actions as $mail_queue_action => [$one_key, $bulk_key, $done_word, $log_action, $done_note, $why_not]) { + $mail_queue_ids = null; + if (isset($_GET[$one_key])) { + $mail_queue_ids = [$_GET[$one_key]]; + } elseif (isset($_POST[$bulk_key])) { + $mail_queue_ids = is_array($_POST['email_ids'] ?? null) ? $_POST['email_ids'] : []; + } + if ($mail_queue_ids === null) { + continue; + } validateCSRFToken(); - $email_id = intval($_GET['send_failed_mail']); - - // Delivered mail has had its body cleared on send, so resending would deliver an empty message - $row = mysqli_fetch_assoc(mysqli_query($mysqli, "SELECT email_status FROM email_queue WHERE email_id = $email_id LIMIT 1")); - - if (!$row || intval($row['email_status']) === 3) { - flashAlert("That email has already been delivered and cannot be resent.", 'error'); + $asked = count(array_unique(array_filter(array_map('intval', $mail_queue_ids)))); + if (!$asked) { + flashAlert('None were ticked.', 'warning'); redirect(); } - mysqli_query($mysqli,"UPDATE email_queue SET email_status = 0, email_attempts = 3 WHERE email_id = $email_id"); + $done = mailQueueAct($mail_queue_action, $mail_queue_ids); + $count = count($done); - logAudit("Email", "Send", "$session_name attempted to force send email id: $email_id in the mail queue", 0, $email_id); - - flashAlert("Email Force Sent, give it a minute to resend"); - - redirect(); - -} - -if (isset($_GET['cancel_mail'])) { - - validateCSRFToken(); - - $email_id = intval($_GET['cancel_mail']); - - mysqli_query($mysqli,"UPDATE email_queue SET email_status = 2, email_attempts = 99, email_failed_at = NOW() WHERE email_id = $email_id"); - - logAudit("Email", "Send", "$session_name canceled send email id: $email_id in the mail queue", 0, $email_id); - - flashAlert("Email cancelled and marked as failed.", 'error'); - - redirect(); - -} - -if (isset($_POST['bulk_cancel_emails'])) { - - validateCSRFToken(); - - if (isset($_POST['email_ids'])) { - - $count = count($_POST['email_ids']); - - // Cycle through array and mark each email as failed - foreach ($_POST['email_ids'] as $email_id) { - - $email_id = intval($email_id); - mysqli_query($mysqli,"UPDATE email_queue SET email_status = 2, email_attempts = 99, email_failed_at = NOW() WHERE email_id = $email_id"); - - logAudit("Email", "Cancel", "$session_name cancelled email id: $email_id in the mail queue", 0, $email_id); - - } - - logAudit("Email", "Bulk Cancel", "$session_name cancelled $count email(s) in the mail queue"); - - flashAlert("Cancelled $count email(s)", 'error'); + if ($count) { + $ids = implode(', ', array_map(function ($id) { return "#$id"; }, array_slice($done, 0, 20))) . ($count > 20 ? ' and ' . ($count - 20) . ' more' : ''); + logAudit("Email", $log_action, "$session_name $done_word " . ($count === 1 ? 'email' : "$count emails") . " in the mail queue: $ids", 0, $count === 1 ? $done[0] : 0); + } + $email_word = function ($n) { + return $n === 1 ? 'email' : 'emails'; + }; + if ($count === $asked) { + $note = $done_note[$count === 1 ? 0 : 1]; + flashAlert(ucfirst($done_word) . " $count " . $email_word($count) . ($note !== '' ? " - $note" : ''), $mail_queue_action === 'delete' ? 'error' : 'success'); + } elseif ($count) { + flashAlert(ucfirst($done_word) . " $count of $asked " . $email_word($asked) . ". $why_not", 'warning'); + } else { + flashAlert(($asked === 1 ? 'That email' : 'None of those emails') . " couldn't be $done_word. $why_not", 'error'); } redirect(); - } -if (isset($_POST['bulk_delete_emails'])) { - - validateCSRFToken(); - - if (isset($_POST['email_ids'])) { - - $count = count($_POST['email_ids']); - - // Cycle through array and delete each email - foreach ($_POST['email_ids'] as $email_id) { - - $email_id = intval($email_id); - mysqli_query($mysqli,"DELETE FROM email_queue WHERE email_id = $email_id"); - - logAudit("Email", "Delete", "$session_name deleted email id: $email_id from the mail queue"); - - } - - logAudit("Email", "Bulk Delete", "$session_name deleted $count email(s) from the mail queue"); - - flashAlert("Deleted $count email(s)", 'error'); - - } - - redirect(); - -} +unset($mail_queue_actions, $mail_queue_action, $mail_queue_ids, $one_key, $bulk_key, $done_word, $log_action, $done_note, $why_not); diff --git a/admin/post/settings_theme.php b/admin/post/settings_theme.php index bdfbac922..26894148b 100644 --- a/admin/post/settings_theme.php +++ b/admin/post/settings_theme.php @@ -49,7 +49,7 @@ if (isset($_GET['reset_favicon'])) { unlink(dirname(__DIR__, 2) . '/uploads/favicon.ico'); } - logAudit("Settings", "Edit", "$session_name put ITFlow's own browser tab icon back"); + logAudit("Settings", "Edit", "$session_name put the ITFlow browser tab icon back"); flashAlert("The browser tab icon is ITFlow's own again"); diff --git a/cron/mail_queue.php b/cron/mail_queue.php index 82dca1377..0763808b6 100644 --- a/cron/mail_queue.php +++ b/cron/mail_queue.php @@ -360,7 +360,7 @@ if (mysqli_num_rows($sql_queue) > 0) { // Check sender if (!filter_var($email_from, FILTER_VALIDATE_EMAIL)) { $email_from_logging = escapeSql($rowq['email_from']); - mysqli_query($mysqli, "UPDATE email_queue SET email_status = 2, email_attempts = 99 WHERE email_id = $email_id"); + mysqli_query($mysqli, "UPDATE email_queue SET email_status = 2, email_attempts = 99, email_failed_at = NOW() WHERE email_id = $email_id"); logApp("Cron-Mail-Queue", "Error", "Failed to send email #$email_id due to invalid sender address: $email_from_logging - check configuration in settings."); appNotify("Mail", "Failed to send email #$email_id due to invalid sender address"); continue; @@ -375,7 +375,7 @@ if (mysqli_num_rows($sql_queue) > 0) { // Basic recipient syntax check if (!filter_var($email_recipient, FILTER_VALIDATE_EMAIL)) { - mysqli_query($mysqli, "UPDATE email_queue SET email_status = 2, email_attempts = 99 WHERE email_id = $email_id"); + mysqli_query($mysqli, "UPDATE email_queue SET email_status = 2, email_attempts = 99, email_failed_at = NOW() WHERE email_id = $email_id"); $email_to_logging = escapeSql($email_recipient); $email_subject_logging = escapeSql($rowq['email_subject']); logApp("Cron-Mail-Queue", "Error", "Failed to send email: $email_id to $email_to_logging due to invalid recipient address. Email subject was: $email_subject_logging"); @@ -386,7 +386,7 @@ if (mysqli_num_rows($sql_queue) > 0) { // More intelligent recipient MX check (if not disabled with --no-mx-validation) $domain = escapeSql(substr($email_recipient, strpos($email_recipient, '@') + 1)); if (!in_array('--no-mx-validation', $argv) && !checkdnsrr($domain, 'MX')) { - mysqli_query($mysqli, "UPDATE email_queue SET email_status = 2, email_attempts = 99 WHERE email_id = $email_id"); + mysqli_query($mysqli, "UPDATE email_queue SET email_status = 2, email_attempts = 99, email_failed_at = NOW() WHERE email_id = $email_id"); $email_to_logging = escapeSql($email_recipient); $email_subject_logging = escapeSql($rowq['email_subject']); logApp("Cron-Mail-Queue", "Error", "Failed to send email: $email_id to $email_to_logging due to invalid recipient domain (no MX). Email subject was: $email_subject_logging"); @@ -435,8 +435,9 @@ if (mysqli_num_rows($sql_queue) > 0) { } /** ======================================================================= - * RETRIES: status = 2 (Failed), attempts < 4, wait 30 min - * NOTE: Backoff is `email_failed_at <= NOW() - INTERVAL 30 MINUTE` + * RETRIES: status = 2 (Failed), attempts < MAIL_QUEUE_TRIES (4), wait + * MAIL_QUEUE_RETRY_MINUTES (30) - functions/mail_queue.php, which the Mail + * Queue page reads too, to say when the next try is * ======================================================================= */ $sql_failed_queue = mysqli_query( @@ -444,8 +445,8 @@ $sql_failed_queue = mysqli_query( "SELECT email_attachments, email_attempts, email_cal_str, email_content, email_from, email_from_name, email_id, email_recipient, email_recipient_name, email_subject FROM email_queue WHERE email_status = 2 - AND email_attempts < 4 - AND email_failed_at <= NOW() - INTERVAL 30 MINUTE" + AND email_attempts < " . MAIL_QUEUE_TRIES . " + AND email_failed_at <= NOW() - INTERVAL " . MAIL_QUEUE_RETRY_MINUTES . " MINUTE" ); if (mysqli_num_rows($sql_failed_queue) > 0) { diff --git a/cron/nightly_tasks.php b/cron/nightly_tasks.php index f89449f94..685e77ccd 100644 --- a/cron/nightly_tasks.php +++ b/cron/nightly_tasks.php @@ -163,7 +163,7 @@ mysqli_query($mysqli, "UPDATE users SET user_password_reset_token = NULL, user_p mysqli_query($mysqli, "DELETE FROM notifications WHERE notification_dismissed_at < CURDATE() - INTERVAL 90 DAY"); // Clean-up mail queue -mysqli_query($mysqli, "DELETE FROM email_queue WHERE email_queued_at < CURDATE() - INTERVAL 90 DAY"); +mysqli_query($mysqli, "DELETE FROM email_queue WHERE email_queued_at < CURDATE() - INTERVAL " . MAIL_QUEUE_KEEP_DAYS . " DAY"); // Clean-up the PDFs made to go with billing emails - sent or given up on by now billingEmailPdfCleanup(7); diff --git a/functions.php b/functions.php index e061be2f9..61b0c3337 100644 --- a/functions.php +++ b/functions.php @@ -83,3 +83,4 @@ require_once __DIR__ . '/functions/accounts.php'; require_once __DIR__ . '/functions/reconcile.php'; require_once __DIR__ . '/functions/portal.php'; require_once __DIR__ . '/functions/settings.php'; +require_once __DIR__ . '/functions/mail_queue.php'; diff --git a/functions/mail_queue.php b/functions/mail_queue.php new file mode 100644 index 000000000..9547f6c7d --- /dev/null +++ b/functions/mail_queue.php @@ -0,0 +1,193 @@ + Mail Queue shows it. + * + * Everything ITFlow sends is queued (emailQueue(), functions/email.php) and + * sent by the Mail Queue cron job (cron/mail_queue.php). An email's + * email_status is where it is: + * 0 waiting - or scheduled, while its email_queued_at is still ahead + * 1 being sent this minute + * 2 failed - tried again every MAIL_QUEUE_RETRY_MINUTES until it has had + * MAIL_QUEUE_TRIES; 99 tries means it isn't tried again (a bad address) + * 3 sent - its body, calendar file and attachments are cleared then, as + * they can carry share keys and temporary passwords + * 4 cancelled by an admin - never picked up again + * Sent, failed and cancelled email is deleted after 90 days (nightly). + */ + +const MAIL_QUEUE_TRIES = 4; +const MAIL_QUEUE_RETRY_MINUTES = 30; +const MAIL_QUEUE_KEEP_DAYS = 90; +const MAIL_QUEUE_CANCELLED = 4; + +/* The tabs of the list - key => [label, icon, SQL for what is in it]. */ +function mailQueueTabs() { + $tries = MAIL_QUEUE_TRIES; + return [ + '' => ['All', 'fa-inbox', '1 = 1'], + 'waiting' => ['Waiting', 'fa-hourglass-half', 'email_status = 0'], + 'sending' => ['Sending', 'fa-paper-plane', 'email_status = 1'], + 'retrying' => ['Retrying', 'fa-redo-alt', "email_status = 2 AND email_attempts < $tries"], + 'failed' => ['Failed', 'fa-exclamation-circle', "email_status = 2 AND email_attempts >= $tries"], + 'sent' => ['Sent', 'fa-check', 'email_status = 3'], + 'cancelled' => ['Cancelled', 'fa-ban', 'email_status = ' . MAIL_QUEUE_CANCELLED], + ]; +} + +/* + * Where an email stands, in words - key (as the tabs), label, the badge's + * classes, a line under it and the exact time for its title. $row has + * email_status, email_attempts and its times. + */ +function mailQueueState(array $row) { + $status = intval($row['email_status']); + $tries = intval($row['email_attempts']); + $when = function ($datetime) { + return $datetime ? date('M j, Y g:i A', strtotime($datetime)) : ''; + }; + $queued = (string) $row['email_queued_at']; + $failed = (string) ($row['email_failed_at'] ?? ''); + + if ($status === 0 && $queued !== '' && strtotime($queued) > time()) { + return ['key' => 'waiting', 'label' => 'Scheduled', 'badge' => 'text-bg-info', 'icon' => 'fa-calendar-alt', + 'detail' => 'Goes ' . date('M j, g:i A', strtotime($queued)), 'title' => 'Not before ' . $when($queued)]; + } + if ($status === 0) { + return ['key' => 'waiting', 'label' => 'Waiting', 'badge' => 'text-bg-primary', 'icon' => 'fa-hourglass-half', + 'detail' => 'Queued ' . timeAgo($queued), 'title' => 'Queued ' . $when($queued)]; + } + if ($status === 1) { + return ['key' => 'sending', 'label' => 'Sending', 'badge' => 'text-bg-warning', 'icon' => 'fa-paper-plane', + 'detail' => 'Going now', 'title' => '']; + } + if ($status === 2 && $tries < MAIL_QUEUE_TRIES) { + $next = $failed !== '' ? strtotime($failed) + MAIL_QUEUE_RETRY_MINUTES * 60 : time(); + return ['key' => 'retrying', 'label' => 'Retrying', 'badge' => 'text-bg-warning', 'icon' => 'fa-redo-alt', + 'detail' => 'Tried ' . max(1, $tries) . ' of ' . MAIL_QUEUE_TRIES . ' · next ' . ($next > time() ? 'about ' . date('g:i A', $next) : 'any minute'), + 'title' => $failed !== '' ? 'Last failed ' . $when($failed) : '']; + } + if ($status === 2) { + return ['key' => 'failed', 'label' => 'Failed', 'badge' => 'text-bg-danger', 'icon' => 'fa-exclamation-circle', + 'detail' => ($tries > MAIL_QUEUE_TRIES ? 'Not tried again' : 'Gave up after ' . MAIL_QUEUE_TRIES . ' tries') . ($failed !== '' ? ' · ' . timeAgo($failed) : ''), + 'title' => $failed !== '' ? 'Failed ' . $when($failed) : '']; + } + if ($status === 3) { + $sent = (string) ($row['email_sent_at'] ?? ''); + return ['key' => 'sent', 'label' => 'Sent', 'badge' => 'text-bg-success', 'icon' => 'fa-check', + 'detail' => $sent !== '' ? 'Sent ' . timeAgo($sent) : '', 'title' => $sent !== '' ? 'Sent ' . $when($sent) : '']; + } + return ['key' => 'cancelled', 'label' => 'Cancelled', 'badge' => 'text-bg-secondary', 'icon' => 'fa-ban', + 'detail' => $failed !== '' ? 'Cancelled ' . timeAgo($failed) : '', 'title' => $failed !== '' ? 'Cancelled ' . $when($failed) : '']; +} + +/* + * What can be done with an email: retry (anything that didn't go - sent mail + * has been cleared, and one going now is left alone), cancel (one still to + * be sent or tried again) and delete (anything not going this minute). + */ +function mailQueueCan(array $row, $action) { + $status = intval($row['email_status']); + $tries = intval($row['email_attempts']); + switch ($action) { + case 'retry': + return $status === 2 || $status === MAIL_QUEUE_CANCELLED; + case 'cancel': + return $status === 0 || ($status === 2 && $tries < MAIL_QUEUE_TRIES); + case 'delete': + return $status !== 1; + } + return false; +} + +/* + * Retry, cancel or delete emails - each only where it can be done + * (mailQueueCan()), checked again in the UPDATE so one the worker took + * meanwhile is left alone. The ids it was done to. + */ +function mailQueueAct($action, array $email_ids) { + global $mysqli; + $ids = array_values(array_unique(array_filter(array_map('intval', $email_ids)))); + if (!$ids) { + return []; + } + $list = implode(',', $ids); + $tries = MAIL_QUEUE_TRIES; + $cancelled = MAIL_QUEUE_CANCELLED; + $where = [ + 'retry' => "email_status IN (2, $cancelled)", + 'cancel' => "(email_status = 0 OR (email_status = 2 AND email_attempts < $tries))", + 'delete' => 'email_status <> 1', + ][$action] ?? null; + if ($where === null) { + return []; + } + $done = []; + $sql = mysqli_query($mysqli, "SELECT email_id FROM email_queue WHERE email_id IN ($list) AND $where"); + while ($row = mysqli_fetch_row($sql)) { + $done[] = intval($row[0]); + } + if (!$done) { + return []; + } + $done_list = implode(',', $done); + if ($action === 'retry') { + // From the start: a full set of tries, at the next run + mysqli_query($mysqli, "UPDATE email_queue SET email_status = 0, email_attempts = 0, email_failed_at = NULL, email_queued_at = LEAST(email_queued_at, NOW()) + WHERE email_id IN ($done_list) AND $where"); + } elseif ($action === 'cancel') { + mysqli_query($mysqli, "UPDATE email_queue SET email_status = $cancelled, email_failed_at = NOW() WHERE email_id IN ($done_list) AND $where"); + } else { + mysqli_query($mysqli, "DELETE FROM email_queue WHERE email_id IN ($done_list) AND $where"); + } + return $done; +} + +/* + * Whether the queue is moving - the first thing in the way, as level + * (danger/warning), title, text and where to put it right; or, when nothing + * is, level '' and a line on how it runs. $waiting is how many are due and + * have waited more than ten minutes. + */ +function mailQueueHealth() { + global $mysqli, $config_smtp_provider; + $settings = mysqli_fetch_assoc(mysqli_query($mysqli, "SELECT config_enable_cron, config_cron_last_dispatch_at FROM settings WHERE company_id = 1")) ?: []; + $job = mysqli_fetch_assoc(mysqli_query($mysqli, "SELECT cron_job_enabled, cron_job_last_run_at, cron_job_last_error, cron_job_last_error_at FROM cron_jobs WHERE cron_job_name = 'mail_queue'")) ?: null; + $stuck = mysqli_fetch_row(mysqli_query($mysqli, "SELECT COUNT(*), MIN(email_queued_at) FROM email_queue + WHERE email_status = 0 AND email_queued_at <= NOW() - INTERVAL 10 MINUTE")); + $waiting = intval($stuck[0] ?? 0); + $oldest = $stuck[1] ?? null; + $heard = $settings['config_cron_last_dispatch_at'] ?? null; + $ago = function ($datetime) { + if (!$datetime) { + return 'never'; + } + return abs(time() - strtotime($datetime)) < 60 ? 'less than a minute ago' : timeAgo($datetime); + }; + $held = $waiting ? " $waiting " . ($waiting === 1 ? 'email has' : 'emails have') . ' been waiting since ' . date('M j, g:i A', strtotime($oldest)) . '.' : ''; + + if (empty($config_smtp_provider)) { + return ['level' => 'danger', 'key' => 'smtp', 'title' => 'Mail sending is off', 'text' => 'Nothing in the queue goes until it is set up.' . $held, + 'url' => 'settings_mail.php?tab=sending', 'action' => 'Set up sending', 'waiting' => $waiting]; + } + if (empty($settings['config_enable_cron'])) { + return ['level' => 'danger', 'key' => 'cron_off', 'title' => 'Cron is switched off', 'text' => 'No job runs, so no email is sent.' . $held, + 'url' => 'cron.php', 'action' => 'Maintenance > Cron', 'waiting' => $waiting]; + } + if (!$heard || time() - strtotime($heard) >= 300) { + return ['level' => 'danger', 'key' => 'cron_stopped', 'title' => 'Cron isn\'t running', 'text' => 'ITFlow last heard from it ' . $ago($heard) . ', so no email is being sent.' . $held, + 'url' => 'cron.php', 'action' => 'See how to set it up', 'waiting' => $waiting]; + } + if ($job && !intval($job['cron_job_enabled'])) { + return ['level' => 'warning', 'key' => 'job_off', 'title' => 'The Mail Queue job is switched off', 'text' => 'Cron is running, but this job isn\'t.' . $held, + 'url' => 'cron.php', 'action' => 'Maintenance > Cron', 'waiting' => $waiting]; + } + if ($waiting) { + $error = $job && $job['cron_job_last_error'] && $job['cron_job_last_error_at'] && strtotime($job['cron_job_last_error_at']) > time() - 3600 + ? ' Its last error: ' . mb_strimwidth((string) $job['cron_job_last_error'], 0, 200, '...') : ''; + return ['level' => 'warning', 'key' => 'slow', 'title' => 'Email is waiting longer than it should', 'text' => trim($held) . ' The Mail Queue job last ran ' . $ago($job['cron_job_last_run_at'] ?? null) . '.' . $error, + 'url' => 'cron.php', 'action' => 'Maintenance > Cron', 'waiting' => $waiting]; + } + return ['level' => '', 'key' => 'ok', 'title' => '', 'text' => 'Sent by the Mail Queue job each minute - it last ran ' . $ago($job['cron_job_last_run_at'] ?? null) . '. A failed email is tried ' + . (MAIL_QUEUE_TRIES - 1) . ' more times, ' . MAIL_QUEUE_RETRY_MINUTES . ' minutes apart. Email is kept ' . MAIL_QUEUE_KEEP_DAYS . ' days; a sent one without what it said.', 'url' => '', 'action' => '', 'waiting' => 0]; +}