From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Aikar <aikar@aikar.co> Date: Sun, 12 Apr 2020 15:50:48 -0400 Subject: [PATCH] Improved Watchdog Support Forced Watchdog Crash support and Improve Async Shutdown If the request to shut down the server is received while we are in a watchdog hang, immediately treat it as a crash and begin the shutdown process. Shutdown process is now improved to also shutdown cleanly when not using restart scripts either. If a server is deadlocked, a server owner can send SIGUP (or any other signal the JVM understands to shut down as it currently does) and the watchdog will no longer need to wait until the full timeout, allowing you to trigger a close process and try to shut the server down gracefully, saving player and world data. Previously there was no way to trigger this outside of waiting for a full watchdog timeout, which may be set to a really long time... Additionally, fix everything to do with shutting the server down asynchronously. Previously, nearly everything about the process was fragile and unsafe. Main might not have actually been frozen, and might still be manipulating state. Or, some reuest might ask main to do something in the shutdown but main is dead. Or worse, other things might start closing down items such as the Console or Thread Pool before we are fully shutdown. This change tries to resolve all of these issues by moving everything into the stop method and guaranteeing only one thread is stopping the server. We then issue Thread Death to the main thread of another thread initiates the stop process. We have to ensure Thread Death propagates correctly though to stop main completely. This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save. This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they are properly accounted for and wont trip watchdog on init. diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java index 6aaed8e8bf8c721fc834da5c76ac72a4c3e92458..4b002e8b75d117b726b0de274a76d3596fce015b 100644 --- a/src/main/java/com/destroystokyo/paper/Metrics.java +++ b/src/main/java/com/destroystokyo/paper/Metrics.java @@ -92,7 +92,12 @@ public class Metrics { * Starts the Scheduler which submits our data every 30 minutes. */ private void startSubmitting() { - final Runnable submitTask = this::submitData; + final Runnable submitTask = () -> { + if (MinecraftServer.getServer().hasStopped()) { + return; + } + submitData(); + }; // Many servers tend to restart at a fixed time at xx:00 which causes an uneven distribution of requests on the // bStats backend. To circumvent this problem, we introduce some randomness into the initial and second delay. diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java index a9a0248b1bd1ac454064e977b61f9b7d80962ff8..059b1a0bf048af6a28c322f35da3d3cbbe426546 100644 --- a/src/main/java/net/minecraft/CrashReport.java +++ b/src/main/java/net/minecraft/CrashReport.java @@ -233,6 +233,7 @@ public class CrashReport { } public static CrashReport forThrowable(Throwable cause, String title) { + if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper while (cause instanceof CompletionException && cause.getCause() != null) { cause = cause.getCause(); } diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 2deb639d404943ef5b028c4ede59cab99b31a40f..08f7f287af32597d8a39f429013adec9266020bf 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -297,7 +297,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>(); public int autosavePeriod; public Commands vanillaCommandDispatcher; - private boolean forceTicks; + public boolean forceTicks; // Paper // CraftBukkit end // Spigot start public static final int TPS = 20; @@ -308,6 +308,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa public final io.papermc.paper.configuration.PaperConfigurations paperConfigurations; public static long currentTickLong = 0L; // Paper + public volatile Thread shutdownThread; // Paper + public volatile boolean abnormalExit = false; // Paper + public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) { AtomicReference<S> atomicreference = new AtomicReference(); Thread thread = new io.papermc.paper.util.TickThread(() -> { // Paper - rewrite chunk system @@ -914,6 +917,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa // CraftBukkit start private boolean hasStopped = false; + public volatile boolean hasFullyShutdown = false; // Paper private final Object stopLock = new Object(); public final boolean hasStopped() { synchronized (this.stopLock) { @@ -928,6 +932,19 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa if (this.hasStopped) return; this.hasStopped = true; } + // Paper start - kill main thread, and kill it hard + shutdownThread = Thread.currentThread(); + org.spigotmc.WatchdogThread.doStop(); // Paper + if (!isSameThread()) { + MinecraftServer.LOGGER.info("Stopping main thread (Ignore any thread death message you see! - DO NOT REPORT THREAD DEATH TO PAPER)"); + while (this.getRunningThread().isAlive()) { + this.getRunningThread().stop(); + try { + Thread.sleep(1); + } catch (InterruptedException e) {} + } + } + // Paper end // CraftBukkit end if (this.metricsRecorder.isRecording()) { this.cancelRecordingMetrics(); @@ -981,7 +998,20 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa this.getProfileCache().save(false); // Paper } // Spigot end + // Paper start - move final shutdown items here + LOGGER.info("Flushing Chunk IO"); + // Paper end - move final shutdown items here io.papermc.paper.chunk.system.io.RegionFileIOThread.close(true); // Paper // Paper - rewrite chunk system + // Paper start - move final shutdown items here + LOGGER.info("Closing Thread Pool"); + Util.shutdownExecutors(); // Paper + LOGGER.info("Closing Server"); + try { + net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender + } catch (Exception e) { + } + this.onServerExit(); + // Paper end - move final shutdown items here } public String getLocalIp() { @@ -1076,6 +1106,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa protected void runServer() { try { + long serverStartTime = Util.getNanos(); // Paper if (!this.initServer()) { throw new IllegalStateException("Failed to initialize server"); } @@ -1085,6 +1116,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa this.status = this.buildServerStatus(); // Spigot start + // Paper start - move done tracking + LOGGER.info("Running delayed init tasks"); + this.server.getScheduler().mainThreadHeartbeat(this.tickCount); // run all 1 tick delay tasks during init, + // this is going to be the first thing the tick process does anyways, so move done and run it after + // everything is init before watchdog tick. + // anything at 3+ won't be caught here but also will trip watchdog.... + // tasks are default scheduled at -1 + delay, and first tick will tick at 1 + String doneTime = String.format(java.util.Locale.ROOT, "%.3fs", (double) (Util.getNanos() - serverStartTime) / 1.0E9D); + LOGGER.info("Done ({})! For help, type \"help\"", doneTime); + // Paper end + + org.spigotmc.WatchdogThread.tick(); // Paper org.spigotmc.WatchdogThread.hasStarted = true; // Paper Arrays.fill( this.recentTps, 20 ); long tickSection = Util.getNanos(), curTime, tickCount = 1; // Paper @@ -1163,6 +1206,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa JvmProfiler.INSTANCE.onServerTick(this.smoothedTickTimeMillis); } } catch (Throwable throwable) { + // Paper start + if (throwable instanceof ThreadDeath) { + MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable); + return; + } + // Paper end MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable); // Spigot Start if ( throwable.getCause() != null ) @@ -1193,14 +1242,14 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa this.services.profileCache().clearExecutor(); } - org.spigotmc.WatchdogThread.doStop(); // Spigot + //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop // CraftBukkit start - Restore terminal to original settings try { - net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender + //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop } catch (Exception ignored) { } // CraftBukkit end - this.onServerExit(); + //this.onServerExit(); // Paper - moved into stop } } @@ -1269,6 +1318,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa @Override public TickTask wrapRunnable(Runnable runnable) { + // Paper start - anything that does try to post to main during watchdog crash, run on watchdog + if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) { + runnable.run(); + runnable = () -> {}; + } + // Paper end return new TickTask(this.tickCount, runnable); } @@ -1547,6 +1602,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa try { crashreport = CrashReport.forThrowable(throwable, "Exception ticking world"); } catch (Throwable t) { + if (throwable instanceof ThreadDeath) { throw (ThreadDeath)throwable; } // Paper throw new RuntimeException("Error generating crash report", t); } // Spigot End @@ -2060,7 +2116,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa this.worldData.setDataConfiguration(worlddataconfiguration); this.resources.managers.updateRegistryTags(this.registryAccess()); - this.getPlayerList().saveAll(); + // Paper start + if (Thread.currentThread() != this.serverThread) { + return; + } + // this.getPlayerList().saveAll(); // Paper - we don't need to save everything, just advancements + for (ServerPlayer player : this.getPlayerList().getPlayers()) { + player.getAdvancements().save(); + } + // Paper end this.getPlayerList().reloadResources(); this.functionManager.replaceLibrary(this.resources.managers.getFunctionLibrary()); this.structureTemplateManager.onResourceManagerReload(this.resources.resourceManager); diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java index 109c7ff78d4c1f5496d294f52ecfd9df2070db1e..fe47a38137f7b7fa94c507e790eec4fb7303595f 100644 --- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java +++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java @@ -269,7 +269,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface long j = Util.getNanos() - i; String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D); - DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); + //DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); // Paper moved to after init if (dedicatedserverproperties.announcePlayerAchievements != null) { ((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, this); } @@ -397,7 +397,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface // this.remoteStatusListener.stop(); // Paper - don't wait for remote connections } - System.exit(0); // CraftBukkit + hasFullyShutdown = true; // Paper + System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper } @Override @@ -774,7 +775,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface @Override public void stopServer() { super.stopServer(); - Util.shutdownExecutors(); + //Util.shutdownExecutors(); // Paper - moved into super SkullBlockEntity.clear(); } diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java index 9036d9941b1597e14d3b25a5143d914248338800..f644c8b89e96d35fb9f2ac7941dcfcf9e8b66451 100644 --- a/src/main/java/net/minecraft/server/players/PlayerList.java +++ b/src/main/java/net/minecraft/server/players/PlayerList.java @@ -532,7 +532,7 @@ public abstract class PlayerList { this.cserver.getPluginManager().callEvent(playerQuitEvent); entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage()); - entityplayer.doTick(); // SPIGOT-924 + if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog) // CraftBukkit end // Paper start - Remove from collideRule team if needed diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java index f5829ae484d93b547a5437b85a9621346384a11b..83701fbfaa56a232593ee8f11a3afb8941238bfa 100644 --- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java +++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java @@ -152,6 +152,7 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler try { task.run(); } catch (Exception var3) { + if (var3.getCause() instanceof ThreadDeath) throw var3; // Paper LOGGER.error(LogUtils.FATAL_MARKER, "Error executing task on {}", this.name(), var3); } diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java index 35fd928c87403d98a80b6c13fa9df83496576c8f..179a7708a5f4674b31a87c49ca3d5c4c4dbd4b65 100644 --- a/src/main/java/net/minecraft/world/level/Level.java +++ b/src/main/java/net/minecraft/world/level/Level.java @@ -915,6 +915,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable { try { tickConsumer.accept(entity); } catch (Throwable throwable) { + if (throwable instanceof ThreadDeath) throw throwable; // Paper // Paper start - Prevent tile entity and entity crashes final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level().getWorld().getName(), entity.getX(), entity.getY(), entity.getZ()); MinecraftServer.LOGGER.error(msg, throwable); diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java index 12834995b2bab9e82a40feb01f038532961296cf..12af77215bfd6df3b6802a567ac3c013a4cdf06a 100644 --- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java +++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java @@ -1177,6 +1177,7 @@ public class LevelChunk extends ChunkAccess { gameprofilerfiller.pop(); } catch (Throwable throwable) { + if (throwable instanceof ThreadDeath) throw throwable; // Paper // Paper start - Prevent tile entity and entity crashes final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ()); net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable); diff --git a/src/main/java/org/bukkit/craftbukkit/Main.java b/src/main/java/org/bukkit/craftbukkit/Main.java index 17d74aa6d857e49b61f5aed30be2b17be871df4c..ff16522a4ae4e0cb32f4c6b919bcaffc4507dfe8 100644 --- a/src/main/java/org/bukkit/craftbukkit/Main.java +++ b/src/main/java/org/bukkit/craftbukkit/Main.java @@ -179,6 +179,36 @@ public class Main { OptionSet options = null; + // Paper start - preload logger classes to avoid plugins mixing versions + tryPreloadClass("org.apache.logging.log4j.core.Core"); + tryPreloadClass("org.apache.logging.log4j.core.appender.AsyncAppender"); + tryPreloadClass("org.apache.logging.log4j.core.Appender"); + tryPreloadClass("org.apache.logging.log4j.core.ContextDataInjector"); + tryPreloadClass("org.apache.logging.log4j.core.Filter"); + tryPreloadClass("org.apache.logging.log4j.core.ErrorHandler"); + tryPreloadClass("org.apache.logging.log4j.core.LogEvent"); + tryPreloadClass("org.apache.logging.log4j.core.Logger"); + tryPreloadClass("org.apache.logging.log4j.core.LoggerContext"); + tryPreloadClass("org.apache.logging.log4j.core.LogEventListener"); + tryPreloadClass("org.apache.logging.log4j.core.AbstractLogEvent"); + tryPreloadClass("org.apache.logging.log4j.message.AsynchronouslyFormattable"); + tryPreloadClass("org.apache.logging.log4j.message.FormattedMessage"); + tryPreloadClass("org.apache.logging.log4j.message.ParameterizedMessage"); + tryPreloadClass("org.apache.logging.log4j.message.Message"); + tryPreloadClass("org.apache.logging.log4j.message.MessageFactory"); + tryPreloadClass("org.apache.logging.log4j.message.TimestampMessage"); + tryPreloadClass("org.apache.logging.log4j.message.SimpleMessage"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLogger"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerContext"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncQueueFullPolicy"); + tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerDisruptor"); + tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEvent"); + tryPreloadClass("org.apache.logging.log4j.core.async.DisruptorUtil"); + tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEventHandler"); + tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy"); + tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedClassInfo"); + tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedStackTraceElement"); + // Paper end try { options = parser.parse(args); } catch (joptsimple.OptionException ex) { @@ -289,8 +319,64 @@ public class Main { } catch (Throwable t) { t.printStackTrace(); } + // Paper start + // load some required classes to avoid errors during shutdown if jar is replaced + // also to guarantee our version loads over plugins + tryPreloadClass("com.destroystokyo.paper.util.SneakyThrow"); + tryPreloadClass("com.google.common.collect.Iterators$PeekingImpl"); + tryPreloadClass("com.google.common.collect.MapMakerInternalMap$Values"); + tryPreloadClass("com.google.common.collect.MapMakerInternalMap$ValueIterator"); + tryPreloadClass("com.google.common.collect.MapMakerInternalMap$WriteThroughEntry"); + tryPreloadClass("com.google.common.collect.Iterables"); + for (int i = 1; i <= 15; i++) { + tryPreloadClass("com.google.common.collect.Iterables$" + i, false); + } + tryPreloadClass("org.apache.commons.lang3.mutable.MutableBoolean"); + tryPreloadClass("org.apache.commons.lang3.mutable.MutableInt"); + tryPreloadClass("org.jline.terminal.impl.MouseSupport"); + tryPreloadClass("org.jline.terminal.impl.MouseSupport$1"); + tryPreloadClass("org.jline.terminal.Terminal$MouseTracking"); + tryPreloadClass("co.aikar.timings.TimingHistory"); + tryPreloadClass("co.aikar.timings.TimingHistory$MinuteReport"); + tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext"); + tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$11"); + tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$12"); + tryPreloadClass("io.netty.channel.AbstractChannel$AbstractUnsafe$8"); + tryPreloadClass("io.netty.util.concurrent.DefaultPromise"); + tryPreloadClass("io.netty.util.concurrent.DefaultPromise$1"); + tryPreloadClass("io.netty.util.internal.PromiseNotificationUtil"); + tryPreloadClass("io.netty.util.internal.SystemPropertyUtil"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$1"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$2"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$3"); + tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$4"); + tryPreloadClass("org.slf4j.helpers.MessageFormatter"); + tryPreloadClass("org.slf4j.helpers.FormattingTuple"); + tryPreloadClass("org.slf4j.helpers.BasicMarker"); + tryPreloadClass("org.slf4j.helpers.Util"); + tryPreloadClass("com.destroystokyo.paper.event.player.PlayerConnectionCloseEvent"); + tryPreloadClass("com.destroystokyo.paper.event.entity.EntityRemoveFromWorldEvent"); + // Minecraft, seen during saving + tryPreloadClass(net.minecraft.world.level.lighting.LayerLightEventListener.DummyLightLayerEventListener.class.getName()); + tryPreloadClass(net.minecraft.world.level.lighting.LayerLightEventListener.class.getName()); + tryPreloadClass(net.minecraft.util.ExceptionCollector.class.getName()); + // Paper end + } + } + + // Paper start + private static void tryPreloadClass(String className) { + tryPreloadClass(className, true); + } + private static void tryPreloadClass(String className, boolean printError) { + try { + Class.forName(className); + } catch (ClassNotFoundException e) { + if (printError) System.err.println("An expected class " + className + " was not found for preloading: " + e.getMessage()); } } + // Paper end private static List<String> asList(String... params) { return Arrays.asList(params); diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java index c6e8441e299f477ddb22c1ce2618710763978f1a..e8e93538dfd71de86515d9405f728db1631e949a 100644 --- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java +++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java @@ -12,11 +12,27 @@ public class ServerShutdownThread extends Thread { @Override public void run() { try { + // Paper start - try to shutdown on main + server.safeShutdown(false, false); + for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) { + Thread.sleep(100); + } + if (server.hasStopped()) { + while (!server.hasFullyShutdown) Thread.sleep(1000); + return; + } + // Looks stalled, close async org.spigotmc.AsyncCatcher.enabled = false; // Spigot + server.forceTicks = true; this.server.close(); + while (!server.hasFullyShutdown) Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + // Paper end } finally { + org.apache.logging.log4j.LogManager.shutdown(); // Paper try { - net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender + //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop } catch (Exception e) { } } diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java index 051b9e3a5d29a5840d596468e3ddd013bedc8da3..631457374fda492776f4af0515cee2e43a934d2c 100644 --- a/src/main/java/org/spigotmc/RestartCommand.java +++ b/src/main/java/org/spigotmc/RestartCommand.java @@ -138,7 +138,7 @@ public class RestartCommand extends Command // Paper end // Paper start - copied from above and modified to return if the hook registered - private static boolean addShutdownHook(String restartScript) + public static boolean addShutdownHook(String restartScript) { String[] split = restartScript.split( " " ); if ( split.length > 0 && new File( split[0] ).isFile() ) diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java index 230d55820778e84c1c8aa2b013ae0e5e35568ea1..5ca863aa1859922fa359eba32539229db40e5b98 100644 --- a/src/main/java/org/spigotmc/WatchdogThread.java +++ b/src/main/java/org/spigotmc/WatchdogThread.java @@ -11,6 +11,7 @@ import org.bukkit.Bukkit; public final class WatchdogThread extends io.papermc.paper.util.TickThread // Paper - rewrite chunk system { + public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper private static WatchdogThread instance; private long timeoutTime; private boolean restart; @@ -39,6 +40,7 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa { if ( WatchdogThread.instance == null ) { + if (timeoutTime <= 0) timeoutTime = 300; // Paper WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart ); WatchdogThread.instance.start(); } else @@ -70,12 +72,13 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa // Paper start Logger log = Bukkit.getServer().getLogger(); long currentTime = WatchdogThread.monotonicMillis(); - if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable + MinecraftServer server = MinecraftServer.getServer(); + if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable { - boolean isLongTimeout = currentTime > lastTick + timeoutTime; + boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000); // Don't spam early warning dumps if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; - if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... + if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... lastEarlyWarning = currentTime; if (isLongTimeout) { // Paper end @@ -137,9 +140,24 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa if ( isLongTimeout ) { - if ( this.restart && !MinecraftServer.getServer().hasStopped() ) + if ( !server.hasStopped() ) { - RestartCommand.restart(); + AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us + server.forceTicks = true; + if (restart) { + RestartCommand.addShutdownHook( SpigotConfig.restartScript ); + } + // try one last chance to safe shutdown on main incase it 'comes back' + server.abnormalExit = true; + server.safeShutdown(false, restart); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (!server.hasStopped()) { + server.close(); + } } break; } // Paper end diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index ea4e2161c0bd43884055cc6b8d70b2139f70e720..266b4e6fb3988b5848021c83fdc68e342c70b188 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<Configuration status="WARN" packages="com.mojang.util"> +<Configuration status="WARN" packages="com.mojang.util" shutdownHook="disable"> <Appenders> <Queue name="ServerGuiConsole"> <PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg%n" />