bc127ea819
Upstream has released updates that appear to apply and compile correctly. This update has not been tested by PaperMC and as with ANY update, please do your own testing Bukkit Changes: eec4aab0 SPIGOT-6657: Add getPlayer to SheepDyeWoolEvent 205213c6 SPIGOT-6656: CauldronLevelChangeEvent is not fired correctly when dripstone fills the cauldron CraftBukkit Changes: b8c522d5 SPIGOT-6657: Add getPlayer to SheepDyeWoolEvent f04a77dc SPIGOT-6656: CauldronLevelChangeEvent is not fired correctly when dripstone fills the cauldron d1dbcebc SPIGOT-6653: Canceling snow bucket placement removes snow from bucket 4f34a67b #891: Fix scheduler task ID overflow and duplication issues Spigot Changes: d03d7f12 BUILDTOOLS-604: Rebuild patches
182 lines
10 KiB
Diff
182 lines
10 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: miclebrick <miclebrick@outlook.com>
|
|
Date: Wed, 8 Aug 2018 15:30:52 -0400
|
|
Subject: [PATCH] Add Early Warning Feature to WatchDog
|
|
|
|
Detect when the server has been hung for a long duration, and start printing
|
|
thread dumps at an interval until the point of crash.
|
|
|
|
This will help diagnose what was going on in that time before the crash.
|
|
|
|
diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java
|
|
index bd508025b771424c942fd856c31d520b6f548082..62621562137cba4804f0465c58d25ca2786328e5 100644
|
|
--- a/src/main/java/com/destroystokyo/paper/PaperConfig.java
|
|
+++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java
|
|
@@ -25,6 +25,7 @@ import org.bukkit.configuration.file.YamlConfiguration;
|
|
import co.aikar.timings.Timings;
|
|
import co.aikar.timings.TimingsManager;
|
|
import org.spigotmc.SpigotConfig;
|
|
+import org.spigotmc.WatchdogThread;
|
|
|
|
public class PaperConfig {
|
|
|
|
@@ -297,6 +298,14 @@ public class PaperConfig {
|
|
}
|
|
}
|
|
|
|
+ public static int watchdogPrintEarlyWarningEvery = 5000;
|
|
+ public static int watchdogPrintEarlyWarningDelay = 10000;
|
|
+ private static void watchdogEarlyWarning() {
|
|
+ watchdogPrintEarlyWarningEvery = getInt("settings.watchdog.early-warning-every", 5000);
|
|
+ watchdogPrintEarlyWarningDelay = getInt("settings.watchdog.early-warning-delay", 10000);
|
|
+ WatchdogThread.doStart(SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
|
|
+ }
|
|
+
|
|
public static int tabSpamIncrement = 1;
|
|
public static int tabSpamLimit = 500;
|
|
private static void tabSpamLimiters() {
|
|
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
index 2bbed2eb8f992764deac2cd5e08471aa0f967606..d0d524bfe7ae90f6d2edefa4063bf009cddee17b 100644
|
|
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
@@ -1084,6 +1084,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
this.updateStatusIcon(this.status);
|
|
|
|
// Spigot start
|
|
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
|
|
Arrays.fill( recentTps, 20 );
|
|
long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
|
|
lastTick = start - TICK_TIME; // Paper
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
index 096caeab9813d59e4b086c379e100d8b22713194..6338f32acb10fdb8a6cf8004e78096db33ce1aec 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
@@ -806,6 +806,7 @@ public final class CraftServer implements Server {
|
|
|
|
@Override
|
|
public void reload() {
|
|
+ org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
|
|
this.reloadCount++;
|
|
this.configuration = YamlConfiguration.loadConfiguration(this.getConfigFile());
|
|
this.commandsConfiguration = YamlConfiguration.loadConfiguration(this.getCommandsConfigFile());
|
|
@@ -921,6 +922,7 @@ public final class CraftServer implements Server {
|
|
this.enablePlugins(PluginLoadOrder.STARTUP);
|
|
this.enablePlugins(PluginLoadOrder.POSTWORLD);
|
|
this.getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
|
|
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
|
|
}
|
|
|
|
@Override
|
|
diff --git a/src/main/java/org/spigotmc/SpigotConfig.java b/src/main/java/org/spigotmc/SpigotConfig.java
|
|
index 314fa148fe783a0558ba00b068e0bf69a91577e1..66a3395a7a0cc8eb1b3cd24761bed9d8c3349f4c 100644
|
|
--- a/src/main/java/org/spigotmc/SpigotConfig.java
|
|
+++ b/src/main/java/org/spigotmc/SpigotConfig.java
|
|
@@ -229,7 +229,7 @@ public class SpigotConfig
|
|
SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript );
|
|
SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) );
|
|
SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) );
|
|
- WatchdogThread.doStart( timeoutTime, restartOnCrash );
|
|
+ //WatchdogThread.doStart( timeoutTime, restartOnCrash ); // Paper - moved to PaperConfig
|
|
}
|
|
|
|
public static boolean bungee;
|
|
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
|
|
index 6e1fa4f0616ccfd258acd1b4f5b08fc0ad4c9529..a01a266a25e3267c94c20f8597b4b596efe20faa 100644
|
|
--- a/src/main/java/org/spigotmc/WatchdogThread.java
|
|
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
|
|
@@ -5,6 +5,7 @@ import java.lang.management.MonitorInfo;
|
|
import java.lang.management.ThreadInfo;
|
|
import java.util.logging.Level;
|
|
import java.util.logging.Logger;
|
|
+import com.destroystokyo.paper.PaperConfig;
|
|
import net.minecraft.server.MinecraftServer;
|
|
import org.bukkit.Bukkit;
|
|
|
|
@@ -14,6 +15,10 @@ public class WatchdogThread extends Thread
|
|
private static WatchdogThread instance;
|
|
private long timeoutTime;
|
|
private boolean restart;
|
|
+ private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
|
|
+ private final long earlyWarningDelay; // Paper
|
|
+ public static volatile boolean hasStarted; // Paper
|
|
+ private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
|
|
private volatile long lastTick;
|
|
private volatile boolean stopping;
|
|
|
|
@@ -22,6 +27,8 @@ public class WatchdogThread extends Thread
|
|
super( "Paper Watchdog Thread" );
|
|
this.timeoutTime = timeoutTime;
|
|
this.restart = restart;
|
|
+ earlyWarningEvery = Math.min(PaperConfig.watchdogPrintEarlyWarningEvery, timeoutTime); // Paper
|
|
+ earlyWarningDelay = Math.min(PaperConfig.watchdogPrintEarlyWarningDelay, timeoutTime); // Paper
|
|
}
|
|
|
|
private static long monotonicMillis()
|
|
@@ -61,9 +68,18 @@ public class WatchdogThread extends Thread
|
|
while ( !this.stopping )
|
|
{
|
|
//
|
|
- if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
|
|
+ // Paper start
|
|
+ Logger log = Bukkit.getServer().getLogger();
|
|
+ long currentTime = WatchdogThread.monotonicMillis();
|
|
+ if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
|
|
{
|
|
- Logger log = Bukkit.getServer().getLogger();
|
|
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime;
|
|
+ // Don't spam early warning dumps
|
|
+ if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
|
|
+ if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
|
|
+ lastEarlyWarning = currentTime;
|
|
+ if (isLongTimeout) {
|
|
+ // Paper end
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
|
|
log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
|
|
@@ -93,29 +109,45 @@ public class WatchdogThread extends Thread
|
|
}
|
|
}
|
|
// Paper end
|
|
+ } else
|
|
+ {
|
|
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---");
|
|
+ log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
|
|
+ }
|
|
+ // Paper end - Different message for short timeout
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
|
|
WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
//
|
|
+ // Paper start - Only print full dump on long timeouts
|
|
+ if ( isLongTimeout )
|
|
+ {
|
|
log.log( Level.SEVERE, "Entire Thread Dump:" );
|
|
ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
|
|
for ( ThreadInfo thread : threads )
|
|
{
|
|
WatchdogThread.dumpThread( thread, log );
|
|
}
|
|
+ } else {
|
|
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
|
|
+ }
|
|
+
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
|
|
+ if ( isLongTimeout )
|
|
+ {
|
|
if ( this.restart && !MinecraftServer.getServer().hasStopped() )
|
|
{
|
|
RestartCommand.restart();
|
|
}
|
|
break;
|
|
+ } // Paper end
|
|
}
|
|
|
|
try
|
|
{
|
|
- sleep( 10000 );
|
|
+ sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
|
|
} catch ( InterruptedException ex )
|
|
{
|
|
interrupt();
|