mirrors/papermc
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Apply watchdoge patches

Browse source
This commit is contained in:
Jason Penilla 2024-10-25 18:13:48 -07:00
parent be2edeac2b
commit b14d336442
No known key found for this signature in database
GPG key ID: 0E75A301420E48F8
42 changed files with 459 additions and 463 deletions
Download patch file Download diff file Expand all files Collapse all files

475
patches/unapplied/server/1016-Improved-Watchdog-Support.patch
View file

@ -1,475 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Sun, 12 Apr 2020 15:50:48 -0400
Subject: [PATCH] Improved Watchdog Support
Forced Watchdog Crash support and Improve Async Shutdown
If the request to shut down the server is received while we are in
a watchdog hang, immediately treat it as a crash and begin the shutdown
process. Shutdown process is now improved to also shutdown cleanly when
not using restart scripts either.
If a server is deadlocked, a server owner can send SIGUP (or any other signal
the JVM understands to shut down as it currently does) and the watchdog
will no longer need to wait until the full timeout, allowing you to trigger
a close process and try to shut the server down gracefully, saving player and
world data.
Previously there was no way to trigger this outside of waiting for a full watchdog
timeout, which may be set to a really long time...
Additionally, fix everything to do with shutting the server down asynchronously.
Previously, nearly everything about the process was fragile and unsafe. Main might
not have actually been frozen, and might still be manipulating state.
Or, some reuest might ask main to do something in the shutdown but main is dead.
Or worse, other things might start closing down items such as the Console or Thread Pool
before we are fully shutdown.
This change tries to resolve all of these issues by moving everything into the stop
method and guaranteeing only one thread is stopping the server.
We then issue Thread Death to the main thread of another thread initiates the stop process.
We have to ensure Thread Death propagates correctly though to stop main completely.
This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save.
This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they
are properly accounted for and wont trip watchdog on init.
diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java
index 6aaed8e8bf8c721fc834da5c76ac72a4c3e92458..4b002e8b75d117b726b0de274a76d3596fce015b 100644
--- a/src/main/java/com/destroystokyo/paper/Metrics.java
+++ b/src/main/java/com/destroystokyo/paper/Metrics.java
@@ -92,7 +92,12 @@ public class Metrics {
* Starts the Scheduler which submits our data every 30 minutes.
*/
private void startSubmitting() {
- final Runnable submitTask = this::submitData;
+ final Runnable submitTask = () -> {
+ if (MinecraftServer.getServer().hasStopped()) {
+ return;
+ }
+ submitData();
+ };
// Many servers tend to restart at a fixed time at xx:00 which causes an uneven distribution of requests on the
// bStats backend. To circumvent this problem, we introduce some randomness into the initial and second delay.
diff --git a/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java b/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java
new file mode 100644
index 0000000000000000000000000000000000000000..183e141d0c13190c6905dc4510d891992afef878
--- /dev/null
+++ b/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java
@@ -0,0 +1,26 @@
+package io.papermc.paper.util;
+
+public class LogManagerShutdownThread extends Thread {
+
+ static LogManagerShutdownThread INSTANCE = new LogManagerShutdownThread();
+ public static final void hook() {
+ if (INSTANCE == null) {
+ throw new IllegalStateException("Cannot re-hook after being unhooked");
+ }
+ Runtime.getRuntime().addShutdownHook(INSTANCE);
+ }
+
+ public static final void unhook() {
+ Runtime.getRuntime().removeShutdownHook(INSTANCE);
+ INSTANCE = null;
+ }
+
+ private LogManagerShutdownThread() {
+ super("Log4j2 Shutdown Thread");
+ }
+
+ @Override
+ public void run() {
+ org.apache.logging.log4j.LogManager.shutdown();
+ }
+}
diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java
index 589a8bf75be6ccc59f1e5dd5d8d9afed41c4772d..b24265573fdef5d9a964bcd76146f34542c420cf 100644
--- a/src/main/java/net/minecraft/CrashReport.java
+++ b/src/main/java/net/minecraft/CrashReport.java
@@ -237,6 +237,7 @@ public class CrashReport {
}
public static CrashReport forThrowable(Throwable cause, String title) {
+ if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper
while (cause instanceof CompletionException && cause.getCause() != null) {
cause = cause.getCause();
}
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
index 244a19ecd0234fa1d7a6ecfea20751595688605d..581bd217304e0f9e0b2113c335694805dfb4e2a1 100644
--- a/src/main/java/net/minecraft/server/Main.java
+++ b/src/main/java/net/minecraft/server/Main.java
@@ -77,6 +77,7 @@ public class Main {
@DontObfuscate
public static void main(final OptionSet optionset) { // CraftBukkit - replaces main(String[] astring)
+ io.papermc.paper.util.LogManagerShutdownThread.hook(); // Paper
SharedConstants.tryDetectVersion();
/* CraftBukkit start - Replace everything
OptionParser optionparser = new OptionParser();
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index c13b4f39c19dd6a62ae39688bde5a4739b391e59..d25d500a2206d4f7e821de3766e8a523df4370e3 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -307,7 +307,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>();
public int autosavePeriod;
// Paper - don't store the vanilla dispatcher
- private boolean forceTicks;
+ public boolean forceTicks; // Paper - Improved watchdog support
// CraftBukkit end
// Spigot start
public static final int TPS = 20;
@@ -319,6 +319,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
public final io.papermc.paper.configuration.PaperConfigurations paperConfigurations; // Paper - add paper configuration files
public boolean isIteratingOverLevels = false; // Paper - Throw exception on world create while being ticked
+ public volatile Thread shutdownThread; // Paper
+ public volatile boolean abnormalExit = false; // Paper
+
public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) {
AtomicReference<S> atomicreference = new AtomicReference();
Thread thread = new ca.spottedleaf.moonrise.common.util.TickThread(() -> { // Paper - rewrite chunk system
@@ -488,6 +491,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
}
*/
// Paper end
+ io.papermc.paper.util.LogManagerShutdownThread.unhook(); // Paper
Runtime.getRuntime().addShutdownHook(new org.bukkit.craftbukkit.util.ServerShutdownThread(this));
// CraftBukkit end
this.paperConfigurations = services.paperConfigurations(); // Paper - add paper configuration files
@@ -1004,6 +1008,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
// CraftBukkit start
private boolean hasStopped = false;
private boolean hasLoggedStop = false; // Paper - Debugging
+ public volatile boolean hasFullyShutdown = false; // Paper
private final Object stopLock = new Object();
public final boolean hasStopped() {
synchronized (this.stopLock) {
@@ -1019,6 +1024,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
this.hasStopped = true;
}
if (!hasLoggedStop && isDebugging()) io.papermc.paper.util.TraceUtil.dumpTraceForThread("Server stopped"); // Paper - Debugging
+ // Paper start - kill main thread, and kill it hard
+ shutdownThread = Thread.currentThread();
+ org.spigotmc.WatchdogThread.doStop(); // Paper
+ // Paper end
// CraftBukkit end
if (this.metricsRecorder.isRecording()) {
this.cancelRecordingMetrics();
@@ -1092,7 +1101,17 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
}
// Spigot end
+ // Paper end - Improved watchdog support; move final shutdown items here
ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread.deinit(); // Paper - rewrite chunk system
+ // Paper start - move final shutdown items here
+ Util.shutdownExecutors();
+ try {
+ net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+ } catch (final Exception ignored) {
+ }
+ io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown
+ this.onServerExit();
+ // Paper end - Improved watchdog support
}
public String getLocalIp() {
@@ -1187,6 +1206,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
protected void runServer() {
try {
+ long serverStartTime = Util.getNanos(); // Paper
if (!this.initServer()) {
throw new IllegalStateException("Failed to initialize server");
}
@@ -1196,6 +1216,17 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
this.status = this.buildServerStatus();
// Spigot start
+ // Paper start - Improved Watchdog Support
+ LOGGER.info("Running delayed init tasks");
+ this.server.getScheduler().mainThreadHeartbeat(this.tickCount); // run all 1 tick delay tasks during init,
+ // this is going to be the first thing the tick process does anyways, so move done and run it after
+ // everything is init before watchdog tick.
+ // anything at 3+ won't be caught here but also will trip watchdog....
+ // tasks are default scheduled at -1 + delay, and first tick will tick at 1
+ final long actualDoneTimeMs = System.currentTimeMillis() - org.bukkit.craftbukkit.Main.BOOT_TIME.toEpochMilli(); // Paper - Add total time
+ LOGGER.info("Done ({})! For help, type \"help\"", String.format(java.util.Locale.ROOT, "%.3fs", actualDoneTimeMs / 1000.00D)); // Paper - Add total time
+ org.spigotmc.WatchdogThread.tick();
+ // Paper end - Improved Watchdog Support
org.spigotmc.WatchdogThread.hasStarted = true; // Paper
Arrays.fill( this.recentTps, 20 );
// Paper start - further improve server tick loop
@@ -1290,6 +1321,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
JvmProfiler.INSTANCE.onServerTick(this.smoothedTickTimeMillis);
}
} catch (Throwable throwable) {
+ // Paper start
+ if (throwable instanceof ThreadDeath) {
+ MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable);
+ return;
+ }
+ // Paper end
MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable);
CrashReport crashreport = MinecraftServer.constructOrExtractCrashReport(throwable);
@@ -1314,15 +1351,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
this.services.profileCache().clearExecutor();
}
- org.spigotmc.WatchdogThread.doStop(); // Spigot
+ //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop
// CraftBukkit start - Restore terminal to original settings
try {
- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
} catch (Exception ignored) {
}
// CraftBukkit end
- io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown
- this.onServerExit();
+ //io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown
+ //this.onServerExit(); // Paper - moved into stop
}
}
@@ -1445,6 +1482,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
@Override
public TickTask wrapRunnable(Runnable runnable) {
+ // Paper start - anything that does try to post to main during watchdog crash, run on watchdog
+ if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) {
+ runnable.run();
+ runnable = () -> {};
+ }
+ // Paper end
return new TickTask(this.tickCount, runnable);
}
@@ -2264,7 +2307,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
this.worldData.setDataConfiguration(worlddataconfiguration);
this.resources.managers.updateRegistryTags();
this.potionBrewing = this.potionBrewing.reload(this.worldData.enabledFeatures()); // Paper - Custom Potion Mixes
- this.getPlayerList().saveAll();
+ // Paper start
+ if (Thread.currentThread() != this.serverThread) {
+ return;
+ }
+ // this.getPlayerList().saveAll(); // Paper - we don't need to save everything, just advancements // TODO Move this to a different patch
+ for (ServerPlayer player : this.getPlayerList().getPlayers()) {
+ player.getAdvancements().save();
+ }
+ // Paper end
this.getPlayerList().reloadResources();
this.functionManager.replaceLibrary(this.resources.managers.getFunctionLibrary());
this.structureTemplateManager.onResourceManagerReload(this.resources.resourceManager);
diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
index 7d2896918ff5fed37e5de5a22c37b0c7f32634a8..d43b98bdfcb00603737a309c0fb7793d42289b8c 100644
--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
@@ -328,7 +328,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
long j = Util.getNanos() - i;
String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D);
- DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s);
+ DedicatedServer.LOGGER.info("Done preparing level \"{}\" ({})", this.getLevelIdName(), s); // Paper - clarify startup log messages & add total time
if (dedicatedserverproperties.announcePlayerAchievements != null) {
((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, this.overworld()); // CraftBukkit - per-world
}
@@ -462,7 +462,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
// this.remoteStatusListener.stop(); // Paper - don't wait for remote connections
}
- System.exit(0); // CraftBukkit
+ hasFullyShutdown = true; // Paper
+ System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper
}
@Override
@@ -842,7 +843,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
@Override
public void stopServer() {
super.stopServer();
- Util.shutdownExecutors();
+ //Util.shutdownExecutors(); // Paper - moved into super
SkullBlockEntity.clear();
}
diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java
index ec058eb6a10500831f173dcb47576c32c7516318..0cfaf31273bbec7640eb3a2b16a8d2c816b0ed61 100644
--- a/src/main/java/net/minecraft/server/players/PlayerList.java
+++ b/src/main/java/net/minecraft/server/players/PlayerList.java
@@ -604,7 +604,7 @@ public abstract class PlayerList {
this.cserver.getPluginManager().callEvent(playerQuitEvent);
entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage());
- entityplayer.doTick(); // SPIGOT-924
+ if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog)
// CraftBukkit end
// Paper start - Configurable player collision; Remove from collideRule team if needed
diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
index ca94a1aaccdcc9f28b5f7936b871216a75ab762a..f22fb84c7e7929d6c80c44b13179cf385d8a43f9 100644
--- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
@@ -150,6 +150,7 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler
try {
task.run();
} catch (Exception var3) {
+ if (var3.getCause() instanceof ThreadDeath) throw var3; // Paper
LOGGER.error(LogUtils.FATAL_MARKER, "Error executing task on {}", this.name(), var3);
}
}
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
index 5c4eaa6bcf20b0fcec14bd5ef76ea6f29a8613a2..e2a0487089eb5a7bdc1433e4c75f69d8e9f9d5f9 100644
--- a/src/main/java/net/minecraft/world/level/Level.java
+++ b/src/main/java/net/minecraft/world/level/Level.java
@@ -1419,6 +1419,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable, ca.spottedl
try {
tickConsumer.accept(entity);
} catch (Throwable throwable) {
+ if (throwable instanceof ThreadDeath) throw throwable; // Paper
// Paper start - Prevent block entity and entity crashes
final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level().getWorld().getName(), entity.getX(), entity.getY(), entity.getZ());
MinecraftServer.LOGGER.error(msg, throwable);
diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
index b1a6dc25b04ab6a43e8d62378e14d88d1c60bbbe..7c11853c5090fbc4fa5b3e73a69acf166158fdec 100644
--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
@@ -1044,6 +1044,7 @@ public class LevelChunk extends ChunkAccess implements ca.spottedleaf.moonrise.p
gameprofilerfiller.pop();
} catch (Throwable throwable) {
+ if (throwable instanceof ThreadDeath) throw throwable; // Paper
// Paper start - Prevent block entity and entity crashes
final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ());
net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable);
diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
index c6e8441e299f477ddb22c1ce2618710763978f1a..e8e93538dfd71de86515d9405f728db1631e949a 100644
--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
@@ -12,11 +12,27 @@ public class ServerShutdownThread extends Thread {
@Override
public void run() {
try {
+ // Paper start - try to shutdown on main
+ server.safeShutdown(false, false);
+ for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) {
+ Thread.sleep(100);
+ }
+ if (server.hasStopped()) {
+ while (!server.hasFullyShutdown) Thread.sleep(1000);
+ return;
+ }
+ // Looks stalled, close async
org.spigotmc.AsyncCatcher.enabled = false; // Spigot
+ server.forceTicks = true;
this.server.close();
+ while (!server.hasFullyShutdown) Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ // Paper end
} finally {
+ org.apache.logging.log4j.LogManager.shutdown(); // Paper
try {
- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
} catch (Exception e) {
}
}
diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
index 39e56b95aaafbcd8ebe68fdefaace83702e9510d..3ba27955548a26367a87d6b87c3c61beb299dfb9 100644
--- a/src/main/java/org/spigotmc/RestartCommand.java
+++ b/src/main/java/org/spigotmc/RestartCommand.java
@@ -139,7 +139,7 @@ public class RestartCommand extends Command
// Paper end
// Paper start - copied from above and modified to return if the hook registered
- private static boolean addShutdownHook(String restartScript)
+ public static boolean addShutdownHook(String restartScript) // Paper
{
String[] split = restartScript.split( " " );
if ( split.length > 0 && new File( split[0] ).isFile() )
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
index 529df2a41dd93d6e1505053bd04032dbf0cdaa31..c9e17225bc52fe5e7b2dc0908db225a86c6e94d1 100644
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -11,6 +11,7 @@ import org.bukkit.Bukkit;
public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThread // Paper - rewrite chunk system
{
+ public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper - Improved watchdog support
private static WatchdogThread instance;
private long timeoutTime;
private boolean restart;
@@ -39,6 +40,7 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
{
if ( WatchdogThread.instance == null )
{
+ if (timeoutTime <= 0) timeoutTime = 300; // Paper
WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart );
WatchdogThread.instance.start();
} else
@@ -70,12 +72,13 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
// Paper start
Logger log = Bukkit.getServer().getLogger();
long currentTime = WatchdogThread.monotonicMillis();
- if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
+ MinecraftServer server = MinecraftServer.getServer();
+ if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable
{
- boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
// Don't spam early warning dumps
if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
- if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+ if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
lastEarlyWarning = currentTime;
if (isLongTimeout) {
// Paper end
@@ -136,9 +139,24 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
if ( isLongTimeout )
{
- if ( this.restart && !MinecraftServer.getServer().hasStopped() )
+ if ( !server.hasStopped() )
{
- RestartCommand.restart();
+ AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
+ server.forceTicks = true;
+ if (restart) {
+ RestartCommand.addShutdownHook( SpigotConfig.restartScript );
+ }
+ // try one last chance to safe shutdown on main incase it 'comes back'
+ server.abnormalExit = true;
+ server.safeShutdown(false, restart);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ if (!server.hasStopped()) {
+ server.close();
+ }
}
break;
} // Paper end
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
index 637d64da9938e51a97338b9253b43889585c67bb..d2a75850af9c6ad2aca66a5f994f1b587d73eac4 100644
--- a/src/main/resources/log4j2.xml
+++ b/src/main/resources/log4j2.xml
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="WARN">
+<Configuration status="WARN" shutdownHook="disable">
<Appenders>
<Queue name="ServerGuiConsole">
<PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg{nolookups}%n" />

296
patches/unapplied/server/1027-Detail-more-information-in-watchdog-dumps.patch
View file

@ -1,296 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Thu, 26 Mar 2020 21:59:32 -0700
Subject: [PATCH] Detail more information in watchdog dumps
- Dump position, world, velocity, and uuid for currently ticking entities
- Dump player name, player uuid, position, and world for packet handling
diff --git a/src/main/java/net/minecraft/network/Connection.java b/src/main/java/net/minecraft/network/Connection.java
index 90a2c61c42cba7e38f167eccdd7a951a947963c4..3e550f8e7cd4f4e16f499a8a2a4b95420270f07a 100644
--- a/src/main/java/net/minecraft/network/Connection.java
+++ b/src/main/java/net/minecraft/network/Connection.java
@@ -632,7 +632,13 @@ public class Connection extends SimpleChannelInboundHandler<Packet<?>> {
if (!(this.packetListener instanceof net.minecraft.server.network.ServerLoginPacketListenerImpl loginPacketListener)
|| loginPacketListener.state != net.minecraft.server.network.ServerLoginPacketListenerImpl.State.VERIFYING
|| Connection.joinAttemptsThisTick++ < MAX_PER_TICK) {
+ // Paper start - detailed watchdog information
+ net.minecraft.network.protocol.PacketUtils.packetProcessing.push(this.packetListener);
+ try {
tickablepacketlistener.tick();
+ } finally {
+ net.minecraft.network.protocol.PacketUtils.packetProcessing.pop();
+ } // Paper end - detailed watchdog information
} // Paper end - Buffer joins to world
}
diff --git a/src/main/java/net/minecraft/network/protocol/PacketUtils.java b/src/main/java/net/minecraft/network/protocol/PacketUtils.java
index d0d36a57ec4896bcb74970f8fb24d8f3e17db133..e2c24813f59c2fd075c740ac1842a38f20ed8554 100644
--- a/src/main/java/net/minecraft/network/protocol/PacketUtils.java
+++ b/src/main/java/net/minecraft/network/protocol/PacketUtils.java
@@ -20,6 +20,24 @@ public class PacketUtils {
private static final Logger LOGGER = LogUtils.getLogger();
+ // Paper start - detailed watchdog information
+ public static final java.util.concurrent.ConcurrentLinkedDeque<PacketListener> packetProcessing = new java.util.concurrent.ConcurrentLinkedDeque<>();
+ static final java.util.concurrent.atomic.AtomicLong totalMainThreadPacketsProcessed = new java.util.concurrent.atomic.AtomicLong();
+
+ public static long getTotalProcessedPackets() {
+ return totalMainThreadPacketsProcessed.get();
+ }
+
+ public static java.util.List<PacketListener> getCurrentPacketProcessors() {
+ java.util.List<PacketListener> ret = new java.util.ArrayList<>(4);
+ for (PacketListener listener : packetProcessing) {
+ ret.add(listener);
+ }
+
+ return ret;
+ }
+ // Paper end - detailed watchdog information
+
public PacketUtils() {}
public static <T extends PacketListener> void ensureRunningOnSameThread(Packet<T> packet, T listener, ServerLevel world) throws RunningOnDifferentThreadException {
@@ -29,6 +47,8 @@ public class PacketUtils {
public static <T extends PacketListener> void ensureRunningOnSameThread(Packet<T> packet, T listener, BlockableEventLoop<?> engine) throws RunningOnDifferentThreadException {
if (!engine.isSameThread()) {
engine.executeIfPossible(() -> {
+ packetProcessing.push(listener); // Paper - detailed watchdog information
+ try { // Paper - detailed watchdog information
if (listener instanceof ServerCommonPacketListenerImpl serverCommonPacketListener && serverCommonPacketListener.processedDisconnect) return; // CraftBukkit - Don't handle sync packets for kicked players
if (listener.shouldHandleMessage(packet)) {
co.aikar.timings.Timing timing = co.aikar.timings.MinecraftTimings.getPacketTiming(packet); // Paper - timings
@@ -48,6 +68,12 @@ public class PacketUtils {
} else {
PacketUtils.LOGGER.debug("Ignoring packet due to disconnection: {}", packet);
}
+ // Paper start - detailed watchdog information
+ } finally {
+ totalMainThreadPacketsProcessed.getAndIncrement();
+ packetProcessing.pop();
+ }
+ // Paper end - detailed watchdog information
});
throw RunningOnDifferentThreadException.RUNNING_ON_DIFFERENT_THREAD;
diff --git a/src/main/java/net/minecraft/server/level/ServerLevel.java b/src/main/java/net/minecraft/server/level/ServerLevel.java
index 9e1af229f52a8ac09833901ad53bd154fed34a4f..c5698b05cc3ea4cb1bc6789c30f2aab76748d491 100644
--- a/src/main/java/net/minecraft/server/level/ServerLevel.java
+++ b/src/main/java/net/minecraft/server/level/ServerLevel.java
@@ -1199,7 +1199,26 @@ public class ServerLevel extends Level implements WorldGenLevel, ca.spottedleaf.
}
+ // Paper start - log detailed entity tick information
+ // TODO replace with varhandle
+ static final java.util.concurrent.atomic.AtomicReference<Entity> currentlyTickingEntity = new java.util.concurrent.atomic.AtomicReference<>();
+
+ public static List<Entity> getCurrentlyTickingEntities() {
+ Entity ticking = currentlyTickingEntity.get();
+ List<Entity> ret = java.util.Arrays.asList(ticking == null ? new Entity[0] : new Entity[] { ticking });
+
+ return ret;
+ }
+ // Paper end - log detailed entity tick information
+
public void tickNonPassenger(Entity entity) {
+ // Paper start - log detailed entity tick information
+ ca.spottedleaf.moonrise.common.util.TickThread.ensureTickThread("Cannot tick an entity off-main");
+ try {
+ if (currentlyTickingEntity.get() == null) {
+ currentlyTickingEntity.lazySet(entity);
+ }
+ // Paper end - log detailed entity tick information
++TimingHistory.entityTicks; // Paper - timings
// Spigot start
co.aikar.timings.Timing timer; // Paper
@@ -1239,7 +1258,13 @@ public class ServerLevel extends Level implements WorldGenLevel, ca.spottedleaf.
this.tickPassenger(entity, entity1);
}
// } finally { timer.stopTiming(); } // Paper - timings - move up
-
+ // Paper start - log detailed entity tick information
+ } finally {
+ if (currentlyTickingEntity.get() == entity) {
+ currentlyTickingEntity.lazySet(null);
+ }
+ }
+ // Paper end - log detailed entity tick information
}
private void tickPassenger(Entity vehicle, Entity passenger) {
diff --git a/src/main/java/net/minecraft/world/entity/Entity.java b/src/main/java/net/minecraft/world/entity/Entity.java
index 2a49bbc2c7b06a065cd33575e6807594529bd1bc..bc45c74987974b4828201e06fc8b1f3fbc0af8b4 100644
--- a/src/main/java/net/minecraft/world/entity/Entity.java
+++ b/src/main/java/net/minecraft/world/entity/Entity.java
@@ -1086,8 +1086,43 @@ public abstract class Entity implements SyncedDataHolder, Nameable, EntityAccess
return this.onGround;
}
+ // Paper start - detailed watchdog information
+ public final Object posLock = new Object(); // Paper - log detailed entity tick information
+
+ private Vec3 moveVector;
+ private double moveStartX;
+ private double moveStartY;
+ private double moveStartZ;
+
+ public final Vec3 getMoveVector() {
+ return this.moveVector;
+ }
+
+ public final double getMoveStartX() {
+ return this.moveStartX;
+ }
+
+ public final double getMoveStartY() {
+ return this.moveStartY;
+ }
+
+ public final double getMoveStartZ() {
+ return this.moveStartZ;
+ }
+ // Paper end - detailed watchdog information
+
public void move(MoverType movementType, Vec3 movement) {
final Vec3 originalMovement = movement; // Paper - Expose pre-collision velocity
+ // Paper start - detailed watchdog information
+ ca.spottedleaf.moonrise.common.util.TickThread.ensureTickThread("Cannot move an entity off-main");
+ synchronized (this.posLock) {
+ this.moveStartX = this.getX();
+ this.moveStartY = this.getY();
+ this.moveStartZ = this.getZ();
+ this.moveVector = movement;
+ }
+ try {
+ // Paper end - detailed watchdog information
if (this.noPhysics) {
this.setPos(this.getX() + movement.x, this.getY() + movement.y, this.getZ() + movement.z);
} else {
@@ -1257,6 +1292,13 @@ public abstract class Entity implements SyncedDataHolder, Nameable, EntityAccess
this.level().getProfiler().pop();
}
}
+ // Paper start - detailed watchdog information
+ } finally {
+ synchronized (this.posLock) { // Paper
+ this.moveVector = null;
+ } // Paper
+ }
+ // Paper end - detailed watchdog information
}
private boolean isStateClimbable(BlockState state) {
@@ -4552,7 +4594,9 @@ public abstract class Entity implements SyncedDataHolder, Nameable, EntityAccess
}
public void setDeltaMovement(Vec3 velocity) {
+ synchronized (this.posLock) { // Paper
this.deltaMovement = velocity;
+ } // Paper
}
public void addDeltaMovement(Vec3 velocity) {
@@ -4658,7 +4702,9 @@ public abstract class Entity implements SyncedDataHolder, Nameable, EntityAccess
}
// Paper end - Fix MC-4
if (this.position.x != x || this.position.y != y || this.position.z != z) {
+ synchronized (this.posLock) { // Paper
this.position = new Vec3(x, y, z);
+ } // Paper
int i = Mth.floor(x);
int j = Mth.floor(y);
int k = Mth.floor(z);
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
index c9e17225bc52fe5e7b2dc0908db225a86c6e94d1..f7a4fee9bb25ff256dc2e5ea26bfbceca6a49167 100644
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -22,6 +22,78 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
private volatile long lastTick;
private volatile boolean stopping;
+ // Paper start - log detailed tick information
+ private void dumpEntity(net.minecraft.world.entity.Entity entity) {
+ Logger log = Bukkit.getServer().getLogger();
+ double posX, posY, posZ;
+ net.minecraft.world.phys.Vec3 mot;
+ double moveStartX, moveStartY, moveStartZ;
+ net.minecraft.world.phys.Vec3 moveVec;
+ synchronized (entity.posLock) {
+ posX = entity.getX();
+ posY = entity.getY();
+ posZ = entity.getZ();
+ mot = entity.getDeltaMovement();
+ moveStartX = entity.getMoveStartX();
+ moveStartY = entity.getMoveStartY();
+ moveStartZ = entity.getMoveStartZ();
+ moveVec = entity.getMoveVector();
+ }
+
+ String entityType = net.minecraft.world.entity.EntityType.getKey(entity.getType()).toString();
+ java.util.UUID entityUUID = entity.getUUID();
+ net.minecraft.world.level.Level world = entity.level();
+
+ log.log(Level.SEVERE, "Ticking entity: " + entityType + ", entity class: " + entity.getClass().getName());
+ log.log(Level.SEVERE, "Entity status: removed: " + entity.isRemoved() + ", valid: " + entity.valid + ", alive: " + entity.isAlive() + ", is passenger: " + entity.isPassenger());
+ log.log(Level.SEVERE, "Entity UUID: " + entityUUID);
+ log.log(Level.SEVERE, "Position: world: '" + (world == null ? "unknown world?" : world.getWorld().getName()) + "' at location (" + posX + ", " + posY + ", " + posZ + ")");
+ log.log(Level.SEVERE, "Velocity: " + (mot == null ? "unknown velocity" : mot.toString()) + " (in blocks per tick)");
+ log.log(Level.SEVERE, "Entity AABB: " + entity.getBoundingBox());
+ if (moveVec != null) {
+ log.log(Level.SEVERE, "Move call information: ");
+ log.log(Level.SEVERE, "Start position: (" + moveStartX + ", " + moveStartY + ", " + moveStartZ + ")");
+ log.log(Level.SEVERE, "Move vector: " + moveVec.toString());
+ }
+ }
+
+ private void dumpTickingInfo() {
+ Logger log = Bukkit.getServer().getLogger();
+
+ // ticking entities
+ for (net.minecraft.world.entity.Entity entity : net.minecraft.server.level.ServerLevel.getCurrentlyTickingEntities()) {
+ this.dumpEntity(entity);
+ net.minecraft.world.entity.Entity vehicle = entity.getVehicle();
+ if (vehicle != null) {
+ log.log(Level.SEVERE, "Detailing vehicle for above entity:");
+ this.dumpEntity(vehicle);
+ }
+ }
+
+ // packet processors
+ for (net.minecraft.network.PacketListener packetListener : net.minecraft.network.protocol.PacketUtils.getCurrentPacketProcessors()) {
+ if (packetListener instanceof net.minecraft.server.network.ServerGamePacketListenerImpl) {
+ net.minecraft.server.level.ServerPlayer player = ((net.minecraft.server.network.ServerGamePacketListenerImpl)packetListener).player;
+ long totalPackets = net.minecraft.network.protocol.PacketUtils.getTotalProcessedPackets();
+ if (player == null) {
+ log.log(Level.SEVERE, "Handling packet for player connection or ticking player connection (null player): " + packetListener);
+ log.log(Level.SEVERE, "Total packets processed on the main thread for all players: " + totalPackets);
+ } else {
+ this.dumpEntity(player);
+ net.minecraft.world.entity.Entity vehicle = player.getVehicle();
+ if (vehicle != null) {
+ log.log(Level.SEVERE, "Detailing vehicle for above entity:");
+ this.dumpEntity(vehicle);
+ }
+ log.log(Level.SEVERE, "Total packets processed on the main thread for all players: " + totalPackets);
+ }
+ } else {
+ log.log(Level.SEVERE, "Handling packet for connection: " + packetListener);
+ }
+ }
+ }
+ // Paper end - log detailed tick information
+
private WatchdogThread(long timeoutTime, boolean restart)
{
super( "Paper Watchdog Thread" );
@@ -119,6 +191,7 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler.dumpAllChunkLoadInfo(MinecraftServer.getServer(), isLongTimeout); // Paper - rewrite chunk system
+ this.dumpTickingInfo(); // Paper - log detailed tick information
WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
log.log( Level.SEVERE, "------------------------------" );
//