class="mb-3">
Show them
diff --git a/admin/post/debug.php b/admin/post/debug.php
new file mode 100644
index 000000000..539a272f3
--- /dev/null
+++ b/admin/post/debug.php
@@ -0,0 +1,43 @@
+ Debug
+ */
+
+defined('FROM_POST_HANDLER') || die("Direct file access is not allowed");
+
+if (isset($_POST['repair_database_collation'])) {
+
+ validateCSRFToken();
+
+ // Give every table whose collation isn't db.sql's the one db.sql has
+ // (functions/database_collation.php). Each table changed is rebuilt, and a very large
+ // one can outlast the browser or a proxy in front of it - so the work is not tied to
+ // the request, and what is left if it is cut off is left whole: a table is changed
+ // entirely or not at all, and Repair again carries on with the rest.
+ ignore_user_abort(true);
+ set_time_limit(0);
+
+ $repair = databaseCollationRepair();
+ $changed = count($repair['changed']);
+ $failed = count($repair['failed']);
+
+ foreach ($repair['failed'] as $table => $why) {
+ logApp('Database', 'warning', "Repair collations left the table $table with the collation it had: $why");
+ }
+
+ if ($changed) {
+ logAudit("Database", "Repair", "$session_name repaired the collation of " . ($changed === 1 ? '1 table' : "$changed tables") . ": " . implode(', ', $repair['changed']));
+ }
+
+ if ($failed) {
+ flashAlert(($changed ? ($changed === 1 ? '1 table' : "$changed tables") . ' repaired. ' : '')
+ . ($failed === 1 ? '1 table' : "$failed tables") . " could not be changed and " . ($failed === 1 ? 'is' : 'are') . " as before - App Logs has why.", 'error');
+ } elseif ($changed) {
+ flashAlert(($changed === 1 ? '1 table now has' : "$changed tables now have") . " the collation db.sql has for " . ($changed === 1 ? 'it' : 'them') . ".");
+ } else {
+ flashAlert("Every table already has the collation db.sql has for it.");
+ }
+
+ redirect("debug.php#debug-database");
+}
diff --git a/functions.php b/functions.php
index ca4e8fbc7..39c96734c 100644
--- a/functions.php
+++ b/functions.php
@@ -62,6 +62,7 @@ require_once __DIR__ . '/functions/tags.php';
require_once __DIR__ . '/functions/categories.php';
require_once __DIR__ . '/functions/custom_link.php';
require_once __DIR__ . '/functions/system_health.php';
+require_once __DIR__ . '/functions/database_collation.php';
require_once __DIR__ . '/functions/ticket_fields.php';
require_once __DIR__ . '/functions/ticket.php';
require_once __DIR__ . '/functions/ticket_status.php';
diff --git a/functions/database_collation.php b/functions/database_collation.php
new file mode 100644
index 000000000..3dcbd5ed6
--- /dev/null
+++ b/functions/database_collation.php
@@ -0,0 +1,193 @@
+ Debug, which
+ * reports where they differ (healthStructureDifferences()) and offers to put
+ * them right (admin/post/debug.php).
+ *
+ * Every table in db.sql is utf8mb4_general_ci, with two columns utf8mb4_bin
+ * on purpose (calendars.calendar_feed_key and user_calendar_feeds.feed_key,
+ * so that two keys differing only by case are two keys). A table can end up
+ * otherwise: made by a database update that left the collation to the server
+ * (2.1.6 to 2.4.1 did, and 3.6.5 put those right), restored from a dump made
+ * on another server, or changed by hand. Nothing shows until a query puts
+ * text from two collations side by side - a UNION, a comparison across two
+ * tables - and stops with "Illegal mix of collations".
+ *
+ * The repair is database update 3.6.5's, with db.sql saying what each column
+ * should be: one ALTER a table, each column changed with MODIFY from its own
+ * line of SHOW CREATE TABLE with only the collation replaced. Not CONVERT TO
+ * CHARACTER SET, which takes every text column in the table - the binary ones
+ * with the rest - and turns a utf8mb3 TEXT into a MEDIUMTEXT. Only tables and
+ * columns db.sql has are looked at; anything else in the database is left.
+ */
+
+/** A text column's type as SHOW CREATE TABLE and db.sql write it, for a regex. An enum's values are quoted, with a quote inside one doubled. */
+function databaseCollationTextType() {
+ return "(?:var)?char\\(\\d+\\)|(?:tiny|medium|long)?text|(?:enum|set)\\('(?:[^']|'')*'(?:,'(?:[^']|'')*')*\\)";
+}
+
+/*
+ * What db.sql gives each table - [table => ['collation' => its default,
+ * 'columns' => [text column => collation]]]. [] when db.sql isn't there.
+ */
+function databaseCollationExpected() {
+ static $expected = null;
+ if ($expected !== null) {
+ return $expected;
+ }
+ $expected = [];
+ $file = dirname(__DIR__) . '/db.sql';
+ if (!is_file($file)) {
+ return $expected;
+ }
+ if (preg_match_all('/CREATE TABLE\s+`([^`]+)`\s*\((.*?)\n\)([^;]*);/s', (string) file_get_contents($file), $creates, PREG_SET_ORDER)) {
+ foreach ($creates as $create) {
+ if (!preg_match('/\bCOLLATE=(\w+)/', $create[3], $default)) {
+ continue;
+ }
+ $columns = [];
+ foreach (explode("\n", $create[2]) as $line) {
+ if (preg_match('/^\s*`([^`]+)`\s+(?:' . databaseCollationTextType() . ')(?:\s+CHARACTER SET\s+\w+)?(?:\s+COLLATE\s+(\w+))?/i', $line, $column)) {
+ $columns[$column[1]] = !empty($column[2]) ? $column[2] : $default[1];
+ }
+ }
+ $expected[$create[1]] = ['collation' => $default[1], 'columns' => $columns];
+ }
+ }
+ return $expected;
+}
+
+/*
+ * Where the database's collations aren't db.sql's - [table => ['expected' =>
+ * the table's default in db.sql, 'found' => its default here ('' when that
+ * is right), 'columns' => [column => ['expected', 'found']]]], only the
+ * tables with something different. null when db.sql isn't there or the
+ * database can't be asked. A table or column the database doesn't have is
+ * healthStructureDifferences()'s to report, not this.
+ */
+function databaseCollationDifferences() {
+ global $mysqli;
+ $expected = databaseCollationExpected();
+ if (!$expected) {
+ return null;
+ }
+ $tables = [];
+ $columns = [];
+ try {
+ $result = mysqli_query($mysqli, "SELECT TABLE_NAME AS table_name, TABLE_COLLATION AS table_collation
+ FROM information_schema.TABLES
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE = 'BASE TABLE'");
+ while ($result && $row = mysqli_fetch_assoc($result)) {
+ $tables[$row['table_name']] = (string) $row['table_collation'];
+ }
+ $result = mysqli_query($mysqli, "SELECT TABLE_NAME AS table_name, COLUMN_NAME AS column_name, COLLATION_NAME AS collation_name
+ FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND COLLATION_NAME IS NOT NULL
+ ORDER BY TABLE_NAME, ORDINAL_POSITION");
+ while ($result && $row = mysqli_fetch_assoc($result)) {
+ $columns[$row['table_name']][$row['column_name']] = (string) $row['collation_name'];
+ }
+ } catch (Throwable $e) {
+ return null;
+ }
+ if (!$tables) {
+ return null;
+ }
+ $differences = [];
+ foreach ($expected as $table => $want) {
+ if (!isset($tables[$table])) {
+ continue;
+ }
+ $difference = ['expected' => $want['collation'], 'found' => $tables[$table] !== $want['collation'] ? $tables[$table] : '', 'columns' => []];
+ foreach ($want['columns'] as $column => $collation) {
+ $found = $columns[$table][$column] ?? null;
+ if ($found !== null && $found !== $collation) {
+ $difference['columns'][$column] = ['expected' => $collation, 'found' => $found];
+ }
+ }
+ if ($difference['found'] !== '' || $difference['columns']) {
+ $differences[$table] = $difference;
+ }
+ }
+ return $differences;
+}
+
+/*
+ * Give every table and text column in databaseCollationDifferences() the
+ * collation db.sql has for it. Returns ['changed' => [table, ...], 'failed'
+ * => [table => why]].
+ *
+ * A table's ALTER changes all of it or none of it, so one that fails is
+ * exactly as it was and the rest go on. What stops one is a unique text
+ * column holding two values the collation it has tells apart and the one it
+ * should have doesn't - 'cafe' and 'café', or a space at the end.
+ *
+ * records.record_updated_at has a zero date for its default, and MySQL 8 as
+ * it comes (strict, NO_ZERO_DATE) refuses any ALTER of a table with one. Zero
+ * dates are allowed on this connection while the tables are changed and the
+ * mode is put back after; strict mode itself stays on, so text that would
+ * not fit is an error and never cut short.
+ */
+function databaseCollationRepair() {
+ global $mysqli;
+ $changed = [];
+ $failed = [];
+ $differences = databaseCollationDifferences();
+ if (!$differences) {
+ return ['changed' => $changed, 'failed' => $failed];
+ }
+
+ try {
+ mysqli_query($mysqli, "SET @itflow_sql_mode = @@SESSION.sql_mode");
+ mysqli_query($mysqli, "SET SESSION sql_mode = REPLACE(REPLACE(@@SESSION.sql_mode, 'NO_ZERO_IN_DATE', ''), 'NO_ZERO_DATE', '')");
+ } catch (Throwable $e) {
+ // Carry on with the mode as it is - records may then be one that fails
+ }
+
+ foreach ($differences as $table => $difference) {
+ try {
+ if (!preg_match('/^[A-Za-z0-9_]+$/', $table) || !preg_match('/^[a-z0-9]+_[a-z0-9_]+$/', $difference['expected'])) {
+ throw new RuntimeException("its name or its collation in db.sql was not understood");
+ }
+ $changes = [];
+
+ if ($difference['columns']) {
+ $result = mysqli_query($mysqli, "SHOW CREATE TABLE `$table`");
+ $row = $result ? mysqli_fetch_row($result) : null;
+ if (!$row || empty($row[1])) {
+ throw new RuntimeException("its definition could not be read");
+ }
+ foreach ($difference['columns'] as $column => $collation) {
+ // `name` type [CHARACTER SET x] [COLLATE y] and the rest - NOT NULL, DEFAULT,
+ // COMMENT - which goes back exactly as the server wrote it
+ if (!preg_match('/^[A-Za-z0-9_]+$/', $column) || !preg_match('/^[a-z0-9]+_[a-z0-9_]+$/', $collation['expected'])
+ || !preg_match('/^\s*[`"]?' . $column . '[`"]?\s+(' . databaseCollationTextType() . ')(?:\s+CHARACTER SET\s+\w+)?(?:\s+COLLATE\s+\w+)?(.*?),?\s*$/mi', $row[1], $match)) {
+ throw new RuntimeException("the definition of its column $column was not understood");
+ }
+ $changes[] = "MODIFY `$column` {$match[1]} CHARACTER SET " . strtok($collation['expected'], '_') . " COLLATE {$collation['expected']}{$match[2]}";
+ }
+ }
+
+ if ($difference['found'] !== '') {
+ $changes[] = "DEFAULT CHARACTER SET " . strtok($difference['expected'], '_') . " COLLATE {$difference['expected']}";
+ }
+
+ if (!mysqli_query($mysqli, "ALTER TABLE `$table` " . implode(', ', $changes))) {
+ throw new RuntimeException(mysqli_error($mysqli));
+ }
+ $changed[] = $table;
+
+ } catch (Throwable $e) {
+ $failed[$table] = $e->getMessage();
+ }
+ }
+
+ try {
+ mysqli_query($mysqli, "SET SESSION sql_mode = IFNULL(@itflow_sql_mode, @@SESSION.sql_mode)");
+ } catch (Throwable $e) {
+ // The connection ends with the request
+ }
+
+ return ['changed' => $changed, 'failed' => $failed];
+}
diff --git a/functions/system_health.php b/functions/system_health.php
index d5a48b107..58d562185 100644
--- a/functions/system_health.php
+++ b/functions/system_health.php
@@ -148,7 +148,7 @@ function healthChecks() {
];
}
-/** ITFlow itself - its version, where its code came from, the database's version against the code's, and updates. */
+/** ITFlow itself - its version, where its code came from, the database's version against the code's, the collation of its tables, and updates. */
function healthItflowChecks() {
$items = [];
$branch = gitCurrentBranch();
@@ -167,6 +167,17 @@ function healthItflowChecks() {
$items[] = healthItem('Database version', 'ok', "$installed - up to date");
}
+ // Every table's collation against db.sql (functions/database_collation.php)
+ $collations = databaseCollationDifferences();
+ if ($collations) {
+ $count = count($collations);
+ $items[] = healthItem('Database collation', 'warning', "$count " . ($count === 1 ? 'table differs' : 'tables differ') . ' from db.sql',
+ 'Text in ' . ($count === 1 ? 'this table' : 'these tables') . ' is compared by different rules from the rest of the database, and a page that reads both can stop with "Illegal mix of collations". Repair collations, under Database below, puts ' . ($count === 1 ? 'it' : 'them') . ' right.',
+ '#debug-collation', 'Repair collations');
+ } elseif ($collations !== null) {
+ $items[] = healthItem('Database collation', 'ok', 'Every table matches db.sql');
+ }
+
// Updates - Maintenance > Update fetches them with git, run through exec and shell_exec
if ($commit === '') {
$items[] = healthItem('Updates', 'info', 'Not a git checkout - updated by hand', '', 'https://docs.itflow.org/updating', 'How to update');
@@ -531,8 +542,13 @@ function healthColumnType($type) {
/*
* How the database's tables and columns compare with db.sql - what a fresh
* install makes. Each difference ['what' => missing table, missing column,
- * extra table, extra column or type, 'table', 'column', 'expected', 'found'].
- * null when db.sql isn't there.
+ * extra table, extra column, type or collation, 'table', 'column',
+ * 'expected', 'found']. null when db.sql isn't there.
+ *
+ * Collations (databaseCollationDifferences()) come a row a table, not a row
+ * a column - a table that differs usually differs in all of them, and an
+ * install with every table off would list hundreds. A column db.sql gives a
+ * collation of its own, the binary ones, gets its own row.
*/
function healthStructureDifferences() {
global $database;
@@ -582,6 +598,23 @@ function healthStructureDifferences() {
$differences[] = ['what' => 'Extra table', 'table' => $table, 'column' => '', 'expected' => '', 'found' => ''];
}
}
+ foreach ((array) databaseCollationDifferences() as $table => $collation) {
+ $here = $collation['found'] !== '' ? [$collation['found']] : [];
+ $own = [];
+ foreach ($collation['columns'] as $column => $c) {
+ if ($c['expected'] === $collation['expected']) {
+ $here[] = $c['found'];
+ } else {
+ $own[$column] = $c;
+ }
+ }
+ if ($here) {
+ $differences[] = ['what' => 'Different collation', 'table' => $table, 'column' => '', 'expected' => $collation['expected'], 'found' => implode(', ', array_unique($here))];
+ }
+ foreach ($own as $column => $c) {
+ $differences[] = ['what' => 'Different collation', 'table' => $table, 'column' => $column, 'expected' => $c['expected'], 'found' => $c['found']];
+ }
+ }
return $differences;
}