From 5ba8ffc0cf1579dd3fa89e20ed88db6223bf0fac Mon Sep 17 00:00:00 2001 From: johnnyq Date: Wed, 11 Nov 2026 14:13:11 -0500 Subject: [PATCH] Speed up the dashboard and the menu counts on installs with years of history: database 3.6.4 adds seven indexes, each only where it is missing and also in db.sql for new installs - tickets on ticket_resolved_at, ticket_closed_at and ticket_created_at, a five-column one led by ticket_billable that answers the unbilled count from the index alone, ticket_replies on ticket_reply_by with ticket_reply_created_at, invoices on invoice_status and expenses on expense_date - so open tickets, unpaid invoices, an agent time this week and unbilled tickets are found directly, where each was a read of the whole table, a dozen times over tickets on the dashboard and twice in the menu on every page; dashboardPeriodOptions finds the years with data with one index step a year per table in place of a UNION of YEAR over every row of five tables, and dashboardLatestIncome joins from payments with STRAIGHT_JOIN so the last five are read off payment_date, where every payment was sorted; measured on 280,000 tickets, 380,000 replies and 70,000 invoices the dashboard went from 5.1s to 0.7s and other pages lost about half a second, with every figure unchanged; also fix lists past 10,000 pages, where includes/filter_footer.php left the pager step unset and a division by zero ended the page at its page numbers; add a 26.11.7 changelog section --- CHANGELOG.md | 21 +++++++++ admin/database_updates/3.6.4.php | 78 ++++++++++++++++++++++++++++++++ db.sql | 15 ++++-- functions/dashboard.php | 42 ++++++++++++----- includes/filter_footer.php | 13 ++---- 5 files changed, 145 insertions(+), 24 deletions(-) create mode 100644 admin/database_updates/3.6.4.php diff --git a/CHANGELOG.md b/CHANGELOG.md index be3c33ae1..d68edd1b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ This file documents all notable changes made to ITFlow. +## [26.11.7] + +### Upgrading + +Update from Maintenance > Update — Queue Update hands the job to cron, which updates the files and applies this release's one database change (3.6.3 to 3.6.4) in the same pass. That change adds indexes to the tickets, ticket replies, invoices and expenses tables: a few seconds on most installs, and up to a few minutes on one with many years of tickets. Nothing else is required. + +### New Features & Updates + +- Speed: the dashboard, and the menu on every page, are much quicker on an ITFlow with years of history in it. Most of what they show is about what is still open - open tickets, unpaid invoices, unbilled time - and finding those few meant reading through every ticket and invoice ever made, several times over. They are now found directly. On a test install with 280,000 tickets, 380,000 replies and 70,000 invoices the dashboard went from about 5 seconds to under 1, a technician's dashboard from under 3 seconds to a tenth of one, and every other page opened about half a second sooner. A smaller install won't see much difference. Every figure shown is the same as before. + +### Bug Fixes + +- Lists: a list long enough to run past 10,000 pages - closed tickets at 10 a page on an install with over 100,000 of them, say - stopped drawing at its page numbers, so the rest of the page was missing and its menus and buttons did nothing. Lists of any length now finish, with their page numbers. + +### Developer Updates + +- Database 3.6.4 (`admin/database_updates/3.6.4.php`) adds seven indexes, each only where it isn't there already, and `db.sql` has them for new installs: `tickets` (`ticket_resolved_at`), (`ticket_closed_at`), (`ticket_created_at`) and (`ticket_billable`, `ticket_invoice_id`, `ticket_status`, `ticket_client_id`, `ticket_archived_at`); `ticket_replies` (`ticket_reply_by`, `ticket_reply_created_at`); `invoices` (`invoice_status`); `expenses` (`expense_date`). `ticket_resolved_at IS NULL` and `ticket_closed_at IS NULL` are a sliver of a mature install's tickets, where `*_archived_at IS NULL` is nearly all of them; the five-column one answers `billingUnbilledCount()` from the index alone; `invoice_status` is taken for Sent, Viewed and Partial and still scanned for Paid. The reasoning for each, and what was left out, is in the migration's comment. +- `dashboardPeriodOptions()` finds the years with data by asking each of payments, revenues, expenses and tickets for its first date from a year on - `SELECT MIN(col) ... WHERE col >= 'Y-01-01'`, one step into the index a year - in place of a `UNION` of `YEAR()` over every row of five tables; invoices get one `SELECT DISTINCT YEAR(invoice_date)`. The list it returns is the same. +- `dashboardLatestIncome()` joins with `STRAIGHT_JOIN` from `payments`, so the last five are read off `payment_date` backwards; the optimizer started from `clients` and sorted every payment. +- `includes/filter_footer.php` works `$pages_split` out for any number of pages (10, 100, 1,000, 10,000 and so on up). It was set only up to 10,000 pages, and past that `is_int($i / $pages_split)` threw a `DivisionByZeroError` in the middle of the pager. + ## [26.11.6] ### Upgrading diff --git a/admin/database_updates/3.6.4.php b/admin/database_updates/3.6.4.php new file mode 100644 index 000000000..0e04228da --- /dev/null +++ b/admin/database_updates/3.6.4.php @@ -0,0 +1,78 @@ + 0) { + continue; + } + + $itflow_index_column_list = '`' . implode('`, `', $itflow_index_columns) . '`'; + + mysqli_query($mysqli, "ALTER TABLE `$itflow_index_table` + ADD KEY `$itflow_index_name` ($itflow_index_column_list)"); + } + + unset($itflow_indexes, $itflow_index, $itflow_index_table, $itflow_index_name, + $itflow_index_columns, $itflow_index_exists, $itflow_index_column_list); diff --git a/db.sql b/db.sql index c85a18701..f75f13368 100644 --- a/db.sql +++ b/db.sql @@ -1751,7 +1751,8 @@ CREATE TABLE `expenses` ( KEY `expense_import_key` (`expense_import_key`), KEY `expense_reimbursement_id` (`expense_reimbursement_id`), KEY `expense_reconciliation_id` (`expense_reconciliation_id`), - KEY `expense_refund_of_id` (`expense_refund_of_id`) + KEY `expense_refund_of_id` (`expense_refund_of_id`), + KEY `expense_date` (`expense_date`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; /*!40101 SET character_set_client = @saved_cs_client */; @@ -1901,7 +1902,8 @@ CREATE TABLE `invoices` ( `invoice_quote_id` int(11) NOT NULL DEFAULT 0, `invoice_client_id` int(11) NOT NULL, PRIMARY KEY (`invoice_id`), - KEY `invoice_client_id` (`invoice_client_id`) + KEY `invoice_client_id` (`invoice_client_id`), + KEY `invoice_status` (`invoice_status`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; /*!40101 SET character_set_client = @saved_cs_client */; @@ -3920,7 +3922,8 @@ CREATE TABLE `ticket_replies` ( `ticket_reply_by` int(11) NOT NULL, `ticket_reply_ticket_id` int(11) NOT NULL, PRIMARY KEY (`ticket_reply_id`), - KEY `ticket_reply_ticket_id` (`ticket_reply_ticket_id`,`ticket_reply_archived_at`) + KEY `ticket_reply_ticket_id` (`ticket_reply_ticket_id`,`ticket_reply_archived_at`), + KEY `ticket_reply_by` (`ticket_reply_by`,`ticket_reply_created_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; /*!40101 SET character_set_client = @saved_cs_client */; @@ -4156,7 +4159,11 @@ CREATE TABLE `tickets` ( KEY `ticket_resolution_due_at` (`ticket_resolution_due_at`), KEY `ticket_client_id` (`ticket_client_id`,`ticket_archived_at`), KEY `ticket_project_id` (`ticket_project_id`), - KEY `ticket_template_id` (`ticket_template_id`) + KEY `ticket_template_id` (`ticket_template_id`), + KEY `ticket_resolved_at` (`ticket_resolved_at`), + KEY `ticket_closed_at` (`ticket_closed_at`), + KEY `ticket_created_at` (`ticket_created_at`), + KEY `ticket_billable` (`ticket_billable`,`ticket_invoice_id`,`ticket_status`,`ticket_client_id`,`ticket_archived_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/functions/dashboard.php b/functions/dashboard.php index 8f2df9dc7..d8c96a32b 100644 --- a/functions/dashboard.php +++ b/functions/dashboard.php @@ -36,20 +36,35 @@ function dashboardPeriodOptions() { 'ytd' => 'Year to date', ]; - $sql = mysqli_query($mysqli, " - SELECT YEAR(payment_date) AS y FROM payments - UNION SELECT YEAR(revenue_date) FROM revenues - UNION SELECT YEAR(expense_date) FROM expenses - UNION SELECT YEAR(invoice_date) FROM invoices - UNION SELECT YEAR(ticket_created_at) FROM tickets - ORDER BY y DESC - "); + // The years with anything in them. Each table is asked for its first date from a year on, then + // from the year after the one that gave - one step into the date's index a time (3.6.4 gave + // expenses and tickets theirs), so it costs the same with ten years of tickets as with ten + // tickets. As a YEAR() of every row it read all five tables through on every visit. + $this_year = intval(date('Y')); + $years = []; + foreach (['payments' => 'payment_date', 'revenues' => 'revenue_date', 'expenses' => 'expense_date', 'tickets' => 'ticket_created_at'] as $table => $column) { + $year = 1970; + while ($year < $this_year) { + $row = mysqli_fetch_row(mysqli_query($mysqli, "SELECT MIN($column) FROM $table WHERE $column >= '" . ($year + 1) . "-01-01'")); + $year = intval(substr((string) ($row[0] ?? ''), 0, 4)); + if ($year < 1971 || $year > $this_year) { + break; + } + $years[$year] = true; + } + } + // Invoices are few and narrow next to tickets, and invoice_date has no index - read once + $sql = mysqli_query($mysqli, "SELECT DISTINCT YEAR(invoice_date) AS y FROM invoices"); while ($row = mysqli_fetch_assoc($sql)) { $year = intval($row['y']); - if ($year > 1970 && $year <= intval(date('Y'))) { - $options["year-$year"] = (string) $year; + if ($year > 1970 && $year <= $this_year) { + $years[$year] = true; } } + krsort($years); + foreach (array_keys($years) as $year) { + $options["year-$year"] = (string) $year; + } return $options; } @@ -1034,11 +1049,14 @@ function dashboardAccountBalances() { function dashboardLatestIncome($limit = 5) { global $mysqli; + // In this order - payments, newest first, and stop at $limit. Left to itself the database started + // from the clients and sorted every payment there is to find the last few. + $sql = mysqli_query($mysqli, " SELECT payment_amount, payment_date, invoice_id, invoice_prefix, invoice_number, invoice_client_id, client_name FROM payments - JOIN invoices ON payment_invoice_id = invoice_id - JOIN clients ON invoice_client_id = client_id + STRAIGHT_JOIN invoices ON payment_invoice_id = invoice_id + STRAIGHT_JOIN clients ON invoice_client_id = client_id WHERE 1 = 1 " . clientScopeSql('invoice_client_id') . " ORDER BY payment_date DESC, payment_id DESC LIMIT " . intval($limit)); diff --git a/includes/filter_footer.php b/includes/filter_footer.php index 2944466f6..eb3272f23 100644 --- a/includes/filter_footer.php +++ b/includes/filter_footer.php @@ -72,14 +72,11 @@ if ($total_found_rows > 5) { 100)) { - $pages_split = 100; - } - if (($total_pages <= 10000) && ($total_pages > 1000)) { - $pages_split = 1000; + // Every 10th page is linked up to 100 pages, every 100th up to 1,000, and so on up - + // past 10,000 pages this had no value, and the division below ended the page + $pages_split = 10; + while ($pages_split * 10 < $total_pages) { + $pages_split *= 10; } if ($page > 1) { $prev_class = "";