From c24d13bfe8b3c3787f581596fcefce94603422a1 Mon Sep 17 00:00:00 2001 From: johnnyq Date: Mon, 2 Nov 2026 15:59:11 -0500 Subject: [PATCH] Speed up Transactions, the Transfers pill, Reconcile and the transactions export: database update 3.5.4 indexes transfers.transfer_expense_id so the ledger no longer checks every expense against every transfer, accountLedgerSql() takes a with_balance flag so the running balance is only worked out where it shows (one account in date order and its export), and the Transfers summary cards come from the same pass as the pill counts and totals --- admin/database_updates/3.5.4.php | 27 ++++++++++++++++++++ agent/post/transaction.php | 3 ++- agent/reconcile.php | 2 +- agent/transactions.php | 42 +++++++++++++++++--------------- db.sql | 3 ++- functions/accounts.php | 11 ++++++--- functions/reconcile.php | 4 +-- 7 files changed, 65 insertions(+), 27 deletions(-) create mode 100644 admin/database_updates/3.5.4.php diff --git a/admin/database_updates/3.5.4.php b/admin/database_updates/3.5.4.php new file mode 100644 index 000000000..a10960aef --- /dev/null +++ b/admin/database_updates/3.5.4.php @@ -0,0 +1,27 @@ + $dtf, 'dtt' => $dtt, 'kind' => $kind_filter, 'show' => $show]); $where = accountLedgerFiltersSql($filter_src, $summary, false, $all); $show_query = $show !== '' ? 'AND ' . $shows[$show]['where'] : ''; -$ledger = accountLedgerSql($account_filter); +// The balance after each line only where it shows - one account, in date order +$ledger = accountLedgerSql($account_filter, $balance_visible); $sql = mysqli_query($mysqli, "SELECT SQL_CALC_FOUND_ROWS ledger.* FROM ($ledger) AS ledger WHERE $where $show_query ORDER BY $order_sql LIMIT $record_from, $record_to"); $num_rows = mysqli_fetch_row(mysqli_query($mysqli, "SELECT FOUND_ROWS()")); @@ -81,21 +82,35 @@ while ($row = mysqli_fetch_assoc($sql)) { $rows[] = accountLedgerVisible($row); } -// The pills, and the money in and out of what is listed +// The pills, and the money in and out of what is listed - and, on every account's +// transfers, what was moved, what paid off cards and what payment processors paid out. +// One pass over the ledger, without the balance after each line. +$transfer_cards = $all && $show === 'transfers'; $count_sql = []; foreach ($shows as $key => $s) { $count_sql[] = $key === '' ? 'COUNT(*)' : "COALESCE(SUM({$s['where']}), 0)"; } -$row = mysqli_fetch_row(mysqli_query($mysqli, "SELECT " . implode(', ', $count_sql) . ", - COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.txn_amount > 0 THEN ledger.txn_amount END), 0), - COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.txn_amount < 0 THEN -ledger.txn_amount END), 0) - FROM ($ledger) AS ledger WHERE $where")); +$count_sql[] = "COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.txn_amount > 0 THEN ledger.txn_amount END), 0)"; +$count_sql[] = "COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.txn_amount < 0 THEN -ledger.txn_amount END), 0)"; +if ($transfer_cards) { + $count_sql[] = "COALESCE(SUM(CASE WHEN 1 $show_query THEN -ledger.txn_amount END), 0)"; + $count_sql[] = "COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.other_account_type = 3 THEN -ledger.txn_amount END), 0)"; + $count_sql[] = "COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.other_account_type = 3 THEN 1 END), 0)"; + $count_sql[] = "COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.account_type = 5 THEN -ledger.txn_amount END), 0)"; + $count_sql[] = "COALESCE(SUM(CASE WHEN 1 $show_query AND ledger.account_type = 5 THEN 1 END), 0)"; +} +$row = mysqli_fetch_row(mysqli_query($mysqli, "SELECT " . implode(', ', $count_sql) . " FROM (" . accountLedgerSql($account_filter, false) . ") AS ledger WHERE $where")); $counts = []; foreach (array_keys($shows) as $i => $key) { $counts[$key] = intval($row[$i]); } -$total_in = round(floatval($row[count($shows)]), 2); -$total_out = round(floatval($row[count($shows) + 1]), 2); +$n = count($shows); +$total_in = round(floatval($row[$n]), 2); +$total_out = round(floatval($row[$n + 1]), 2); +if ($transfer_cards) { + $moved = ['count' => $counts['transfers'], 'total' => round(floatval($row[$n + 2]), 2), 'cards' => round(floatval($row[$n + 3]), 2), 'card_count' => intval($row[$n + 4]), + 'payouts' => round(floatval($row[$n + 5]), 2), 'payout_count' => intval($row[$n + 6])]; +} $currency = $account ? (string) $account['account_currency_code'] : (string) $session_company_currency; $money = function ($amount, $code = null) use ($currency) { return escapeHtml(billingMoney($amount, $code ?? $currency)); @@ -117,17 +132,6 @@ foreach ($open_accounts as $open) { } $is_card = $account && $account['type'] === 3; -// Every account's transfers: what was moved, what paid off cards and what payment processors paid out -$transfer_cards = $all && $show === 'transfers'; -if ($transfer_cards) { - $row = mysqli_fetch_row(mysqli_query($mysqli, "SELECT COUNT(*), COALESCE(SUM(-ledger.txn_amount), 0), - COALESCE(SUM(CASE WHEN ledger.other_account_type = 3 THEN -ledger.txn_amount END), 0), COALESCE(SUM(CASE WHEN ledger.other_account_type = 3 THEN 1 END), 0), - COALESCE(SUM(CASE WHEN ledger.account_type = 5 THEN -ledger.txn_amount END), 0), COALESCE(SUM(CASE WHEN ledger.account_type = 5 THEN 1 END), 0) - FROM ($ledger) AS ledger - WHERE $where $show_query")); - $moved = ['count' => intval($row[0]), 'total' => round(floatval($row[1]), 2), 'cards' => round(floatval($row[2]), 2), 'card_count' => intval($row[3]), - 'payouts' => round(floatval($row[4]), 2), 'payout_count' => intval($row[5])]; -} ?> diff --git a/db.sql b/db.sql index 050defc6c..739104d56 100644 --- a/db.sql +++ b/db.sql @@ -4013,7 +4013,8 @@ CREATE TABLE `transfers` ( `transfer_expense_id` int(11) NOT NULL, `transfer_revenue_id` int(11) NOT NULL, PRIMARY KEY (`transfer_id`), - KEY `transfer_revenue_id` (`transfer_revenue_id`) + KEY `transfer_revenue_id` (`transfer_revenue_id`), + KEY `transfer_expense_id` (`transfer_expense_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/functions/accounts.php b/functions/accounts.php index cae1d3bab..c3c8e3f7a 100644 --- a/functions/accounts.php +++ b/functions/accounts.php @@ -358,8 +358,13 @@ function accountLedgerKinds() { * txn_method, txn_reference, txn_invoice_id, txn_currency, txn_imported and * txn_balance - the account's balance after the line, in date order. Select * from it as a derived table. + * + * The balance after each line means sorting the account's whole history, so + * it is only worked out with $with_balance - where it is shown. Without it, + * txn_balance is NULL: counts, totals and lists across every account don't + * need it. */ -function accountLedgerSql($account_id = 0) { +function accountLedgerSql($account_id = 0, $with_balance = true) { $account_id = intval($account_id); $on = function ($column) use ($account_id) { return $account_id ? "$column = $account_id" : "$column > 0"; @@ -408,8 +413,8 @@ function accountLedgerSql($account_id = 0) { return "SELECT ledger_lines.*, accounts.account_name, accounts.account_type, accounts.account_currency_code, category_name, category_color, category_icon, client_name, ledger_lines.txn_other_name AS other_account_name, other_account.account_type AS other_account_type, cleared.reconciliation_id AS txn_cleared_id, cleared.reconciliation_statement_date AS txn_cleared_date, cleared.reconciliation_opening AS txn_cleared_opening, - accounts.opening_balance + SUM(ledger_lines.txn_amount) OVER (PARTITION BY ledger_lines.txn_account_id ORDER BY ledger_lines.txn_date ASC, ledger_lines.txn_created_at ASC, - ledger_lines.txn_order ASC, ledger_lines.txn_id ASC) AS txn_balance + " . ($with_balance ? "accounts.opening_balance + SUM(ledger_lines.txn_amount) OVER (PARTITION BY ledger_lines.txn_account_id ORDER BY ledger_lines.txn_date ASC, ledger_lines.txn_created_at ASC, + ledger_lines.txn_order ASC, ledger_lines.txn_id ASC)" : "NULL") . " AS txn_balance FROM ($rows) AS ledger_lines JOIN accounts ON accounts.account_id = ledger_lines.txn_account_id LEFT JOIN accounts AS other_account ON ledger_lines.txn_kind IN ('transfer_in', 'transfer_out') AND other_account.account_id = ledger_lines.txn_other_account_id diff --git a/functions/reconcile.php b/functions/reconcile.php index f831c019c..7a1bcaace 100644 --- a/functions/reconcile.php +++ b/functions/reconcile.php @@ -151,7 +151,7 @@ function reconcileLines(array $row, &$later = 0) { global $mysqli; $id = intval($row['reconciliation_id']); $date = escapeSql($row['reconciliation_statement_date']); - $sql = mysqli_query($mysqli, "SELECT ledger.*, IF(ledger.txn_reconciliation_id = $id, 1, 0) AS ticked FROM (" . accountLedgerSql($row['reconciliation_account_id']) . ") AS ledger + $sql = mysqli_query($mysqli, "SELECT ledger.*, IF(ledger.txn_reconciliation_id = $id, 1, 0) AS ticked FROM (" . accountLedgerSql($row['reconciliation_account_id'], false) . ") AS ledger WHERE ledger.txn_reconciliation_id = $id OR (ledger.txn_reconciliation_id IS NULL AND ledger.txn_date <= '$date') ORDER BY ledger.txn_date ASC, ledger.txn_created_at ASC, ledger.txn_order ASC, ledger.txn_id ASC"); $lines = []; @@ -159,7 +159,7 @@ function reconcileLines(array $row, &$later = 0) { $line['ticked'] = intval($line['ticked']) === 1; $lines[] = $line; } - $later = intval(mysqli_fetch_row(mysqli_query($mysqli, "SELECT COUNT(*) FROM (" . accountLedgerSql($row['reconciliation_account_id']) . ") AS ledger + $later = intval(mysqli_fetch_row(mysqli_query($mysqli, "SELECT COUNT(*) FROM (" . accountLedgerSql($row['reconciliation_account_id'], false) . ") AS ledger WHERE ledger.txn_reconciliation_id IS NULL AND ledger.txn_date > '$date'"))[0]); return $lines; }