Retry queued transactions after transient connection loss

[Minecraft0122]

Summary

• keep the current transaction at the head of Plan's single-threaded database queue when a transient/recoverable connection failure occurs
• retry with bounded exponential backoff from 1 to 30 seconds so later transactions cannot overtake failed work
• stop retrying promptly when the database is closing and preserve interruption status
• recognize JDBC transient/recoverable exceptions and SQLState class 08, while excluding fatal and explicitly non-transient failures
• replace recursive invalid-connection lookup in MySQLDB with a transient exception handled by the queue retry path

The buffer is intentionally in-memory for the lifetime of the running server process; this PR does not claim crash/restart persistence.

Tests

• verifies transient exceptions through JDBC type and SQLState cause chains
• verifies statement, fatal, and non-transient failures are not retried
• verifies an invalid MySQL connection is closed and rejected after one pool lookup rather than recursing
• verifies a temporarily failing transaction completes before the transaction queued behind it and the queue drains to zero
• targeted tests and :common:checkstyleMain / :common:checkstyleTest pass locally

This is the temporary-database-outage portion of the closed #4772, split into a single-purpose PR as requested.

Closes #4727

[AuroraLS3]

After testing this, it appears to not be functional fallback.

HikariCP does not recover connections and the retries fail forever

/*
 *  This file is part of Player Analytics (Plan).
 *
 *  Plan is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License v3 as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  Plan is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with Plan. If not, see <https://www.gnu.org/licenses/>.
 */
package com.djrapitops.plan.storage.database;

import com.djrapitops.plan.PlanSystem;
import com.djrapitops.plan.settings.config.PlanConfig;
import com.djrapitops.plan.settings.config.paths.DatabaseSettings;
import extension.FullSystemExtension;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;

import java.time.Duration;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import static org.junit.jupiter.api.Assertions.assertTrue;

/**
 * Tests against transient mysql connection errors
 *
 * @author AuroraLS3
 */
@Testcontainers(disabledWithoutDocker = true)
@ExtendWith(FullSystemExtension.class)
class MySQLTransientConnectionRecoveryTest {

    static MySQLContainer<?> mysql = new MySQLContainer<>(DockerImageName.parse("mysql:8.4.11"));

    @BeforeAll
    static void beforeAll(PlanConfig config, PlanSystem system) {
        IO.println("Starting mysql..");
        mysql.start();
        Awaitility.await()
                .atMost(Duration.ofSeconds(30))
                .until(() -> mysql.isRunning());

        config.set(DatabaseSettings.MYSQL_DATABASE, mysql.getDatabaseName());
        config.set(DatabaseSettings.MYSQL_USER, mysql.getUsername());
        config.set(DatabaseSettings.MYSQL_PASS, mysql.getPassword());
        config.set(DatabaseSettings.MYSQL_HOST, mysql.getHost());
        config.set(DatabaseSettings.MYSQL_PORT, String.valueOf(mysql.getMappedPort(3306)));
        config.set(DatabaseSettings.TYPE, "MySQL");

        IO.println("Enabling Plan..");
        system.enable();
    }

    @AfterAll
    static void afterAll(PlanSystem system) {
        if (system != null) system.disable();
        mysql.stop();
    }

    @Test
    void systemResistantToMySQLReboot(PlanSystem system) throws InterruptedException {
        IO.println("Stopping mysql..");
        mysql.stop();
        Thread.sleep(5000L);
        var wasTransactionExecuted = new AtomicBoolean(false);
        IO.println("Queue test transaction");
        system.getDatabaseSystem().getDatabase().executeInTransaction(db -> {
            IO.println("Test transaction was executed.");
            wasTransactionExecuted.set(true);
            return false;
        });

        Awaitility.await()
                .atMost(Duration.ofSeconds(30))
                .until(() -> !mysql.isRunning());
        IO.println("Starting mysql..");
        mysql.start();
        IO.println("Await completion..");
        Awaitility.await()
                .atMost(60, TimeUnit.SECONDS)
                .until(wasTransactionExecuted::get);
        assertTrue(system.isEnabled());
    }
}