mirrors/papermc
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Houston, we got a patch (#2731)

Browse source 
* Houston, we got a patch

* is this the end of the beginning or the beginning of the end
This commit is contained in:
MiniDigger 2019-12-12 17:20:43 +01:00 • committed by Shane Freeder
parent 806e192e26
commit 44d032f1e9
128 changed files with 1116 additions and 1172 deletions
Download patch file Download diff file Expand all files Collapse all files

510
removed/1.15/0354-Allow-Saving-of-Oversized-Chunks.patch Normal file
View file

@ -0,0 +1,510 @@
From 34e2a8960d67bda57cbaafad5086bc2f520f327a Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Fri, 15 Feb 2019 01:08:19 -0500
Subject: [PATCH] Allow Saving of Oversized Chunks
The Minecraft World Region File format has a hard cap of 1MB per chunk.
This is due to the fact that the header of the file format only allocates
a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector.
This limit can be reached fairly easily with books, resulting in the chunk being unable
to save to the world. Worse off, is that nothing printed when this occured, and silently
performed a chunk rollback on next load.
This leads to security risk with duplication and is being actively exploited.
This patch catches the too large scenario, falls back and moves any large Entity
or Tile Entity into a new compound, and this compound is saved into a different file.
On Chunk Load, we check for oversized status, and if so, we load the extra file and
merge the Entities and Tile Entities from the oversized chunk back into the level to
then be loaded as normal.
Once a chunk is returned back to normal size, the oversized flag will clear, and no
extra data file will exist.
This fix maintains compatability with all existing Anvil Region Format tools as it
does not alter the save format. They will just not know about the extra entities.
This fix also maintains compatability if someone switches server jars to one without
this fix, as the data will remain in the oversized file. Once the server returns
to a jar with this fix, the data will be restored.
diff --git a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
index 9fd8a75da..d49afd622 100644
--- a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
+++ b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
@@ -69,6 +69,7 @@ public class NBTCompressedStreamTools {
}
+ public static NBTTagCompound readNBT(DataInputStream datainputstream) throws IOException { return a(datainputstream); } // Paper - OBFHELPER
public static NBTTagCompound a(DataInputStream datainputstream) throws IOException {
return a((DataInput) datainputstream, NBTReadLimiter.a);
}
@@ -89,6 +90,7 @@ public class NBTCompressedStreamTools {
}
}
+ public static void writeNBT(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException { a(nbttagcompound, dataoutput); } // Paper - OBFHELPER
public static void a(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException {
a((NBTBase) nbttagcompound, dataoutput);
}
diff --git a/src/main/java/net/minecraft/server/NBTTagList.java b/src/main/java/net/minecraft/server/NBTTagList.java
index b7c94fe23..80eea5dfb 100644
--- a/src/main/java/net/minecraft/server/NBTTagList.java
+++ b/src/main/java/net/minecraft/server/NBTTagList.java
@@ -11,7 +11,7 @@ import java.util.Objects;
public class NBTTagList extends NBTList<NBTBase> {
- private List<NBTBase> list = Lists.newArrayList();
+ List<NBTBase> list = Lists.newArrayList(); // Paper - private -> package
private byte type = 0;
public NBTTagList() {}
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
index e90ef45ee..ccc3d6c7a 100644
--- a/src/main/java/net/minecraft/server/RegionFile.java
+++ b/src/main/java/net/minecraft/server/RegionFile.java
@@ -23,7 +23,7 @@ public class RegionFile implements AutoCloseable {
// Minecraft is limited to 256 sections per chunk. So 1MB. This can easily be overriden.
// So we extend this to use the REAL size when the count is maxed by seeking to that section and reading the length.
private static final boolean ENABLE_EXTENDED_SAVE = Boolean.parseBoolean(System.getProperty("net.minecraft.server.RegionFile.enableExtendedSave", "true"));
- private final File file;
+ final File file; // Paper - private -> package
// Spigot end
private static final byte[] a = new byte[4096];
private final RandomAccessFile b; private RandomAccessFile getDataFile() { return this.b; } // Paper - OBFHELPER // PAIL dataFile
@@ -33,6 +33,7 @@ public class RegionFile implements AutoCloseable {
public RegionFile(File file) throws IOException {
this.b = new RandomAccessFile(file, "rw");
+ this.file = file; // Spigot // Paper - We need this earlier
if (this.b.length() < 8192L) { // Paper - headers should be 8192
this.b.write(RegionFile.a);
this.b.write(RegionFile.a);
@@ -66,6 +67,7 @@ public class RegionFile implements AutoCloseable {
}
((java.nio.Buffer) header).clear();
java.nio.IntBuffer headerAsInts = header.asIntBuffer();
+ initOversizedState();
// Paper End
int k;
@@ -83,7 +85,7 @@ public class RegionFile implements AutoCloseable {
this.b.seek(j * 4 + 4); // Go back to where we were
}
}
- if (k > 0 && (k >> 8) > 1 && (k >> 8) + (k & 255) <= this.e.size()) { // Paper >= 1 as 0/1 are the headers, and negative isnt valid
+ if (k > 0 && (k >> 8) > 1 && (k >> 8) + (length) <= this.e.size()) { // Paper >= 1 as 0/1 are the headers, and negative isnt valid
for (int l = 0; l < (length); ++l) {
// Spigot end
this.e.set((k >> 8) + l, false);
@@ -102,11 +104,11 @@ public class RegionFile implements AutoCloseable {
if (this.offsets[j] != 0) this.timestamps[j] = k; // Paper - don't set timestamp if it got 0'd above due to corruption
}
- this.file = file; // Spigot
+ // Paper - we need this earlier
}
@Nullable
- public synchronized DataInputStream a(ChunkCoordIntPair chunkcoordintpair) {
+ public synchronized DataInputStream getReadStream(ChunkCoordIntPair chunkcoordintpair) { return this.a(chunkcoordintpair); } public synchronized DataInputStream a(ChunkCoordIntPair chunkcoordintpair) { // Paper - OBFHELPER
try {
int i = this.getOffset(chunkcoordintpair);
@@ -182,8 +184,8 @@ public class RegionFile implements AutoCloseable {
}
}
- public DataOutputStream c(ChunkCoordIntPair chunkcoordintpair) {
- return new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new RegionFile.ChunkBuffer(chunkcoordintpair))));
+ public DataOutputStream getWriteStream(ChunkCoordIntPair chunkcoordintpair) { return this.c(chunkcoordintpair); } public DataOutputStream c(ChunkCoordIntPair chunkcoordintpair) { // Paper - OBFHELPER
+ return new DataOutputStream(new RegionFile.ChunkBuffer(chunkcoordintpair)); // Paper - remove middleware, move deflate to .close() for dynamic levels
}
protected synchronized void a(ChunkCoordIntPair chunkcoordintpair, byte[] abyte, int i) {
@@ -202,8 +204,9 @@ public class RegionFile implements AutoCloseable {
if (i1 >= 256) {
// Spigot start
- if (!ENABLE_EXTENDED_SAVE) throw new RuntimeException(String.format("Too big to save, %d > 1048576", i));
+ if (!USE_SPIGOT_OVERSIZED_METHOD && !RegionFileCache.isOverzealous()) throw new ChunkTooLargeException(chunkcoordintpair.x, chunkcoordintpair.z, l); // Paper - throw error instead
org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.WARNING,"Large Chunk Detected: ({0}) Size: {1} {2}", new Object[]{chunkcoordintpair, i1, this.file});
+ if (!ENABLE_EXTENDED_SAVE) throw new RuntimeException(String.format("Too big to save, %d > 1048576", i)); // Paper - move after our check
// Spigot end
}
@@ -395,6 +398,109 @@ public class RegionFile implements AutoCloseable {
logger.error("Error backing up corrupt file" + file.getAbsolutePath(), e);
}
}
+
+ private final byte[] oversized = new byte[1024];
+ private int oversizedCount = 0;
+
+ private synchronized void initOversizedState() throws IOException {
+ File metaFile = getOversizedMetaFile();
+ if (metaFile.exists()) {
+ final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath());
+ System.arraycopy(read, 0, oversized, 0, oversized.length);
+ for (byte temp : oversized) {
+ oversizedCount += temp;
+ }
+ }
+ }
+
+ private static int getChunkIndex(int x, int z) {
+ return (x & 31) + (z & 31) * 32;
+ }
+ synchronized boolean isOversized(int x, int z) {
+ return this.oversized[getChunkIndex(x, z)] == 1;
+ }
+ synchronized void setOversized(int x, int z, boolean oversized) throws IOException {
+ final int offset = getChunkIndex(x, z);
+ boolean previous = this.oversized[offset] == 1;
+ this.oversized[offset] = (byte) (oversized ? 1 : 0);
+ if (!previous && oversized) {
+ oversizedCount++;
+ } else if (!oversized && previous) {
+ oversizedCount--;
+ }
+ if (previous && !oversized) {
+ File oversizedFile = getOversizedFile(x, z);
+ if (oversizedFile.exists()) {
+ oversizedFile.delete();
+ }
+ }
+ if (oversizedCount > 0) {
+ if (previous != oversized) {
+ writeOversizedMeta();
+ }
+ } else if (previous) {
+ File oversizedMetaFile = getOversizedMetaFile();
+ if (oversizedMetaFile.exists()) {
+ oversizedMetaFile.delete();
+ }
+ }
+ }
+
+ private void writeOversizedMeta() throws IOException {
+ java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized);
+ }
+
+ private File getOversizedMetaFile() {
+ return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + ".oversized.nbt");
+ }
+
+ private File getOversizedFile(int x, int z) {
+ return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt");
+ }
+
+ void writeOversizedData(int x, int z, NBTTagCompound oversizedData) throws IOException {
+ File file = getOversizedFile(x, z);
+ try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new java.io.FileOutputStream(file), new java.util.zip.Deflater(java.util.zip.Deflater.BEST_COMPRESSION), 32 * 1024), 32 * 1024))) {
+ NBTCompressedStreamTools.writeNBT(oversizedData, out);
+ }
+ this.setOversized(x, z, true);
+
+ }
+
+ synchronized NBTTagCompound getOversizedData(int x, int z) throws IOException {
+ File file = getOversizedFile(x, z);
+ try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) {
+ return NBTCompressedStreamTools.readNBT(out);
+ }
+
+ }
+
+ private static final boolean USE_SPIGOT_OVERSIZED_METHOD = Boolean.getBoolean("Paper.useSpigotExtendedSaveMethod"); // Paper
+ static {
+ if (USE_SPIGOT_OVERSIZED_METHOD) {
+ org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.SEVERE, "====================================");
+ org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.SEVERE, "Using Spigot Oversized Chunk save method. Warning this will result in extremely fragmented chunks, as well as making the entire region file unable to be to used in any other software but Forge or Spigot (not usable in Vanilla or CraftBukkit). Paper's method is highly recommended.");
+ org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.SEVERE, "====================================");
+ }
+ }
+ public class ChunkTooLargeException extends RuntimeException {
+ public ChunkTooLargeException(int x, int z, int sectors) {
+ super("Chunk " + x + "," + z + " of " + RegionFile.this.file.toString() + " is too large (" + sectors + "/255)");
+ }
+ }
+ private static class DirectByteArrayOutputStream extends ByteArrayOutputStream {
+ public DirectByteArrayOutputStream() {
+ super();
+ }
+
+ public DirectByteArrayOutputStream(int size) {
+ super(size);
+ }
+
+ public byte[] getBuffer() {
+ return this.buf;
+ }
+ }
// Paper end
class ChunkBuffer extends ByteArrayOutputStream {
@@ -406,8 +512,35 @@ public class RegionFile implements AutoCloseable {
this.b = chunkcoordintpair;
}
- public void close() {
- RegionFile.this.a(this.b, this.buf, this.count);
+ public void close() throws IOException {
+ // Paper start - apply dynamic compression
+ int origLength = this.count;
+ byte[] buf = this.buf;
+ DirectByteArrayOutputStream out = compressData(buf, origLength);
+ byte[] bytes = out.getBuffer();
+ int length = out.size();
+
+ RegionFile.this.a(this.b, bytes, length); // Paper - change to bytes/length
}
}
+
+ private static final byte[] compressionBuffer = new byte[1024 * 64]; // 64k fits most standard chunks input size even, ideally 1 pass through zlib
+ private static final java.util.zip.Deflater deflater = new java.util.zip.Deflater();
+ // since file IO is single threaded, no benefit to using per-region file buffers/synchronization, we can change that later if it becomes viable.
+ private static DirectByteArrayOutputStream compressData(byte[] buf, int length) throws IOException {
+ synchronized (deflater) {
+ deflater.setInput(buf, 0, length);
+ deflater.finish();
+
+ DirectByteArrayOutputStream out = new DirectByteArrayOutputStream(length);
+ while (!deflater.finished()) {
+ out.write(compressionBuffer, 0, deflater.deflate(compressionBuffer));
+ }
+ out.close();
+ deflater.reset();
+ return out;
+ }
+ }
+ // Paper end
+
}
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
index 871881165..c53518a47 100644
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
@@ -47,6 +47,7 @@ public abstract class RegionFileCache implements AutoCloseable {
// Paper start
}
+ public RegionFile getRegionFile(ChunkCoordIntPair chunkcoordintpair, boolean existingOnly) throws IOException { return this.a(chunkcoordintpair, existingOnly); } // Paper - OBFHELPER
private RegionFile a(ChunkCoordIntPair chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit
long i = ChunkCoordIntPair.pair(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ());
RegionFile regionfile = (RegionFile) this.cache.getAndMoveToFirst(i);
@@ -79,12 +80,151 @@ public abstract class RegionFileCache implements AutoCloseable {
public synchronized boolean hasRegionFile(File file, int i, int j) {
return cache.containsKey(ChunkCoordIntPair.pair(i, j));
}
+ // Paper start
+ private static void printOversizedLog(String msg, File file, int x, int z) {
+ org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed.");
+ }
+
+ private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8;
+ private static final int OVERZEALOUS_TOTAL_THRESHOLD = 1024 * 64;
+ private static final int OVERZEALOUS_THRESHOLD = 1024;
+ private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD;
+ private static void resetFilterThresholds() {
+ SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD));
+ }
+ static {
+ resetFilterThresholds();
+ }
+
+ static boolean isOverzealous() {
+ return SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD;
+ }
+
+ private void writeRegion(ChunkCoordIntPair chunk, NBTTagCompound nbttagcompound) throws IOException {
+ RegionFile regionfile = getRegionFile(chunk, false);
+
+ int chunkX = chunk.x;
+ int chunkZ = chunk.z;
+
+ DataOutputStream out = regionfile.getWriteStream(chunk);
+ try {
+ NBTCompressedStreamTools.writeNBT(nbttagcompound, out);
+ out.close();
+ regionfile.setOversized(chunkX, chunkZ, false);
+ } catch (RegionFile.ChunkTooLargeException ignored) {
+ printOversizedLog("ChunkTooLarge! Someone is trying to duplicate.", regionfile.file, chunkX, chunkZ);
+ // Clone as we are now modifying it, don't want to corrupt the pending save state
+ nbttagcompound = nbttagcompound.clone();
+ // Filter out TileEntities and Entities
+ NBTTagCompound oversizedData = filterChunkData(nbttagcompound);
+ //noinspection SynchronizationOnLocalVariableOrMethodParameter
+ synchronized (regionfile) {
+ out = regionfile.getWriteStream(chunk);
+ NBTCompressedStreamTools.writeNBT(nbttagcompound, out);
+ try {
+ out.close();
+ // 2048 is below the min allowed, so it means we enter overzealous mode below
+ if (SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD) {
+ resetFilterThresholds();
+ }
+ } catch (RegionFile.ChunkTooLargeException e) {
+ printOversizedLog("ChunkTooLarge even after reduction. Trying in overzealous mode.", regionfile.file, chunkX, chunkZ);
+ // Eek, major fail. We have retry logic, so reduce threshholds and fall back
+ SIZE_THRESHOLD = OVERZEALOUS_THRESHOLD;
+ throw e;
+ }
+
+ regionfile.writeOversizedData(chunkX, chunkZ, oversizedData);
+ }
+ }
+ }
+
+ private static NBTTagCompound filterChunkData(NBTTagCompound chunk) {
+ NBTTagCompound oversizedLevel = new NBTTagCompound();
+ NBTTagCompound level = chunk.getCompound("Level");
+ filterChunkList(level, oversizedLevel, "Entities");
+ filterChunkList(level, oversizedLevel, "TileEntities");
+ NBTTagCompound oversized = new NBTTagCompound();
+ oversized.set("Level", oversizedLevel);
+ return oversized;
+ }
+
+ private static void filterChunkList(NBTTagCompound level, NBTTagCompound extra, String key) {
+ NBTTagList list = level.getList(key, 10);
+ NBTTagList newList = extra.getList(key, 10);
+ int totalSize = 0;
+ for (java.util.Iterator<NBTBase> iterator = list.list.iterator(); iterator.hasNext();) {
+ NBTBase object = iterator.next();
+ int nbtSize = getNBTSize(object);
+ if (nbtSize > SIZE_THRESHOLD || (SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD && totalSize > OVERZEALOUS_TOTAL_THRESHOLD)) {
+ newList.add(object);
+ iterator.remove();
+ } else {
+ totalSize += nbtSize;
+ }
+ }
+ level.set(key, list);
+ extra.set(key, newList);
+ }
+
+
+ private static NBTTagCompound readOversizedChunk(RegionFile regionfile, ChunkCoordIntPair chunkCoordinate) throws IOException {
+ synchronized (regionfile) {
+ try (DataInputStream datainputstream = regionfile.getReadStream(chunkCoordinate)) {
+ NBTTagCompound oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z);
+ NBTTagCompound chunk = NBTCompressedStreamTools.readNBT(datainputstream);
+ if (oversizedData == null) {
+ return chunk;
+ }
+ NBTTagCompound oversizedLevel = oversizedData.getCompound("Level");
+ NBTTagCompound level = chunk.getCompound("Level");
+
+ mergeChunkList(level, oversizedLevel, "Entities");
+ mergeChunkList(level, oversizedLevel, "TileEntities");
+
+ chunk.set("Level", level);
+
+ return chunk;
+ } catch (Throwable throwable) {
+ throwable.printStackTrace();
+ throw throwable;
+ }
+ }
+ }
+
+ private static void mergeChunkList(NBTTagCompound level, NBTTagCompound oversizedLevel, String key) {
+ NBTTagList levelList = level.getList(key, 10);
+ NBTTagList oversizedList = oversizedLevel.getList(key, 10);
+
+ if (!oversizedList.isEmpty()) {
+ levelList.addAll(oversizedList);
+ level.set(key, levelList);
+ }
+ }
+
+ private static int getNBTSize(NBTBase nbtBase) {
+ DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream());
+ try {
+ nbtBase.write(test);
+ return test.size();
+ } catch (IOException e) {
+ e.printStackTrace();
+ return 0;
+ }
+ }
+
// Paper End
@Nullable
public NBTTagCompound read(ChunkCoordIntPair chunkcoordintpair) throws IOException {
RegionFile regionfile = this.a(chunkcoordintpair, false); // CraftBukkit
DataInputStream datainputstream = regionfile.a(chunkcoordintpair);
+ // Paper start
+ if (regionfile.isOversized(chunkcoordintpair.x, chunkcoordintpair.z)) {
+ printOversizedLog("Loading Oversized Chunk!", regionfile.file, chunkcoordintpair.x, chunkcoordintpair.z);
+ return readOversizedChunk(regionfile, chunkcoordintpair);
+ }
+ // Paper end
Throwable throwable = null;
NBTTagCompound nbttagcompound;
@@ -119,29 +259,32 @@ public abstract class RegionFileCache implements AutoCloseable {
protected void write(ChunkCoordIntPair chunkcoordintpair, NBTTagCompound nbttagcompound) throws IOException {
int attempts = 0; Exception laste = null; while (attempts++ < 5) { try { // Paper
- RegionFile regionfile = this.a(chunkcoordintpair, false); // CraftBukkit
- DataOutputStream dataoutputstream = regionfile.c(chunkcoordintpair);
- Throwable throwable = null;
-
- try {
- NBTCompressedStreamTools.a(nbttagcompound, (DataOutput) dataoutputstream);
- } catch (Throwable throwable1) {
- throwable = throwable1;
- throw throwable1;
- } finally {
- if (dataoutputstream != null) {
- if (throwable != null) {
- try {
- dataoutputstream.close();
- } catch (Throwable throwable2) {
- throwable.addSuppressed(throwable2);
- }
- } else {
- dataoutputstream.close();
- }
- }
-
- }
+ // Paper start
+ this.writeRegion(chunkcoordintpair, nbttagcompound);
+// RegionFile regionfile = this.a(chunkcoordintpair, false); // CraftBukkit
+// DataOutputStream dataoutputstream = regionfile.c(chunkcoordintpair);
+// Throwable throwable = null;
+//
+// try {
+// NBTCompressedStreamTools.a(nbttagcompound, (DataOutput) dataoutputstream);
+// } catch (Throwable throwable1) {
+// throwable = throwable1;
+// throw throwable1;
+// } finally {
+// if (dataoutputstream != null) {
+// if (throwable != null) {
+// try {
+// dataoutputstream.close();
+// } catch (Throwable throwable2) {
+// throwable.addSuppressed(throwable2);
+// }
+// } else {
+// dataoutputstream.close();
+// }
+// }
+//
+// }
+ // Paper end
// Paper start
return;
--
2.22.0

63
removed/1.15/0376-Handle-bad-chunks-more-gracefully.patch Normal file
View file

@ -0,0 +1,63 @@
From b03aa2b2428cf6e504fe4f069f208679e666676f Mon Sep 17 00:00:00 2001
From: Shane Freeder <theboyetronic@gmail.com>
Date: Mon, 15 Apr 2019 02:24:52 +0100
Subject: [PATCH] Handle bad chunks more gracefully
Prior to this change the server would crash when attempting to load a
chunk from a region with bad data.
After this change the server will defer back to vanilla behavior. At
this time, that means attempting to generate a chunk in its place
(and occasionally just not generating anything and leaving small
holes in the world (This statement might not be accurate as of 1.13.x)).
Should Mojang choose to alter this behavior in the future, this change
will simply defer to whatever that new behavior is.
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
index c53518a47..6f34d8aea 100644
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
@@ -171,8 +171,21 @@ public abstract class RegionFileCache implements AutoCloseable {
private static NBTTagCompound readOversizedChunk(RegionFile regionfile, ChunkCoordIntPair chunkCoordinate) throws IOException {
synchronized (regionfile) {
try (DataInputStream datainputstream = regionfile.getReadStream(chunkCoordinate)) {
- NBTTagCompound oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z);
- NBTTagCompound chunk = NBTCompressedStreamTools.readNBT(datainputstream);
+ // Paper start - Handle bad chunks more gracefully - also handle similarly with oversized data
+ NBTTagCompound oversizedData = null;
+
+ try {
+ oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z);
+ } catch (Exception ex) {}
+
+ NBTTagCompound chunk;
+
+ try {
+ chunk = NBTCompressedStreamTools.readNBT(datainputstream);
+ } catch (final Exception ex) {
+ return null;
+ }
+ // Paper end
if (oversizedData == null) {
return chunk;
}
@@ -231,8 +244,13 @@ public abstract class RegionFileCache implements AutoCloseable {
try {
if (datainputstream != null) {
- nbttagcompound = NBTCompressedStreamTools.a(datainputstream);
- return nbttagcompound;
+ // Paper start - Handle bad chunks more gracefully
+ try {
+ return NBTCompressedStreamTools.a(datainputstream);
+ } catch (Exception ex) {
+ return null;
+ }
+ // Paper end
}
nbttagcompound = null;
--
2.23.0

379
removed/1.15/0393-Fix-World-isChunkGenerated-calls.patch Normal file
View file

@ -0,0 +1,379 @@
From 14f4011c2f16754e3f39826237f4822c3b6446b1 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Sat, 15 Jun 2019 08:54:33 -0700
Subject: [PATCH] Fix World#isChunkGenerated calls
Optimize World#loadChunk() too
This patch also adds a chunk status cache on region files (note that
its only purpose is to cache the status on DISK)
diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java
index 8689e0f9f..56761afdf 100644
--- a/src/main/java/net/minecraft/server/ChunkProviderServer.java
+++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java
@@ -28,7 +28,7 @@ public class ChunkProviderServer extends IChunkProvider {
private final WorldServer world;
private final Thread serverThread;
private final LightEngineThreaded lightEngine;
- private final ChunkProviderServer.a serverThreadQueue;
+ public final ChunkProviderServer.a serverThreadQueue; // Paper private -> public
public final PlayerChunkMap playerChunkMap;
private final WorldPersistentData worldPersistentData;
private long lastTickTime;
@@ -109,6 +109,21 @@ public class ChunkProviderServer extends IChunkProvider {
return playerChunk.getFullChunk();
}
+
+ @Nullable
+ public IChunkAccess getChunkAtImmediately(int x, int z) {
+ long k = ChunkCoordIntPair.pair(x, z);
+
+ // Note: Bypass cache to make this MT-Safe
+
+ PlayerChunk playerChunk = this.getChunk(k);
+ if (playerChunk == null) {
+ return null;
+ }
+
+ return playerChunk.getAvailableChunkNow();
+
+ }
// Paper end
@Nullable
diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
index e778c2e85..73f93e494 100644
--- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java
+++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
@@ -410,6 +410,17 @@ public class ChunkRegionLoader {
return nbttagcompound;
}
+ // Paper start
+ public static ChunkStatus getStatus(NBTTagCompound compound) {
+ if (compound == null) {
+ return null;
+ }
+
+ // Note: Copied from below
+ return ChunkStatus.getStatus(compound.getCompound("Level").getString("Status"));
+ }
+ // Paper end
+
public static ChunkStatus.Type a(@Nullable NBTTagCompound nbttagcompound) {
if (nbttagcompound != null) {
ChunkStatus chunkstatus = ChunkStatus.a(nbttagcompound.getCompound("Level").getString("Status"));
diff --git a/src/main/java/net/minecraft/server/ChunkStatus.java b/src/main/java/net/minecraft/server/ChunkStatus.java
index dd1822d6f..e324989b4 100644
--- a/src/main/java/net/minecraft/server/ChunkStatus.java
+++ b/src/main/java/net/minecraft/server/ChunkStatus.java
@@ -176,6 +176,7 @@ public class ChunkStatus {
return this.s;
}
+ public ChunkStatus getPreviousStatus() { return this.e(); } // Paper - OBFHELPER
public ChunkStatus e() {
return this.u;
}
@@ -196,6 +197,17 @@ public class ChunkStatus {
return this.y;
}
+ // Paper start
+ public static ChunkStatus getStatus(String name) {
+ try {
+ // We need this otherwise we return EMPTY for invalid names
+ MinecraftKey key = new MinecraftKey(name);
+ return IRegistry.CHUNK_STATUS.getOptional(key).orElse(null);
+ } catch (Exception ex) {
+ return null; // invalid name
+ }
+ }
+ // Paper end
public static ChunkStatus a(String s) {
return (ChunkStatus) IRegistry.CHUNK_STATUS.get(MinecraftKey.a(s));
}
diff --git a/src/main/java/net/minecraft/server/PlayerChunk.java b/src/main/java/net/minecraft/server/PlayerChunk.java
index 14a176d61..98590e233 100644
--- a/src/main/java/net/minecraft/server/PlayerChunk.java
+++ b/src/main/java/net/minecraft/server/PlayerChunk.java
@@ -70,6 +70,19 @@ public class PlayerChunk {
Either<IChunkAccess, PlayerChunk.Failure> either = (Either<IChunkAccess, PlayerChunk.Failure>) statusFuture.getNow(null);
return either == null ? null : (Chunk) either.left().orElse(null);
}
+
+ public IChunkAccess getAvailableChunkNow() {
+ // TODO can we just getStatusFuture(EMPTY)?
+ for (ChunkStatus curr = ChunkStatus.FULL, next = curr.getPreviousStatus(); curr != next; curr = next, next = next.getPreviousStatus()) {
+ CompletableFuture<Either<IChunkAccess, PlayerChunk.Failure>> future = this.getStatusFutureUnchecked(curr);
+ Either<IChunkAccess, PlayerChunk.Failure> either = future.getNow(null);
+ if (either == null || !either.left().isPresent()) {
+ continue;
+ }
+ return either.left().get();
+ }
+ return null;
+ }
// Paper end
public CompletableFuture<Either<IChunkAccess, PlayerChunk.Failure>> getStatusFutureUnchecked(ChunkStatus chunkstatus) {
diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/java/net/minecraft/server/PlayerChunkMap.java
index 6f1e48ba4..eb49e9021 100644
--- a/src/main/java/net/minecraft/server/PlayerChunkMap.java
+++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java
@@ -897,11 +897,61 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
}
@Nullable
- private NBTTagCompound readChunkData(ChunkCoordIntPair chunkcoordintpair) throws IOException {
+ public NBTTagCompound readChunkData(ChunkCoordIntPair chunkcoordintpair) throws IOException { // Paper - private -> public
NBTTagCompound nbttagcompound = this.read(chunkcoordintpair);
- return nbttagcompound == null ? null : this.getChunkData(this.world.getWorldProvider().getDimensionManager(), this.m, nbttagcompound, chunkcoordintpair, world); // CraftBukkit
+ // Paper start - Cache chunk status on disk
+ if (nbttagcompound == null) {
+ return null;
+ }
+
+ nbttagcompound = this.getChunkData(this.world.getWorldProvider().getDimensionManager(), this.m, nbttagcompound, chunkcoordintpair, world); // CraftBukkit
+ if (nbttagcompound == null) {
+ return null;
+ }
+
+ this.updateChunkStatusOnDisk(chunkcoordintpair, nbttagcompound);
+
+ return nbttagcompound;
+ // Paper end
+ }
+
+ // Paper start - chunk status cache "api"
+ public ChunkStatus getChunkStatusOnDiskIfCached(ChunkCoordIntPair chunkPos) {
+ RegionFile regionFile = this.getRegionFileIfLoaded(chunkPos);
+
+ return regionFile == null ? null : regionFile.getStatusIfCached(chunkPos.x, chunkPos.z);
+ }
+
+ public ChunkStatus getChunkStatusOnDisk(ChunkCoordIntPair chunkPos) throws IOException {
+ RegionFile regionFile = this.getRegionFile(chunkPos, false);
+
+ if (!regionFile.chunkExists(chunkPos)) {
+ return null;
+ }
+
+ ChunkStatus status = regionFile.getStatusIfCached(chunkPos.x, chunkPos.z);
+
+ if (status != null) {
+ return status;
+ }
+
+ this.readChunkData(chunkPos);
+
+ return regionFile.getStatusIfCached(chunkPos.x, chunkPos.z);
+ }
+
+ public void updateChunkStatusOnDisk(ChunkCoordIntPair chunkPos, @Nullable NBTTagCompound compound) throws IOException {
+ RegionFile regionFile = this.getRegionFile(chunkPos, false);
+
+ regionFile.setStatus(chunkPos.x, chunkPos.z, ChunkRegionLoader.getStatus(compound));
+ }
+
+ public IChunkAccess getUnloadingChunk(int chunkX, int chunkZ) {
+ PlayerChunk chunkHolder = this.pendingUnload.get(ChunkCoordIntPair.pair(chunkX, chunkZ));
+ return chunkHolder == null ? null : chunkHolder.getAvailableChunkNow();
}
+ // Paper end
boolean isOutsideOfRange(ChunkCoordIntPair chunkcoordintpair) {
// Spigot start
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
index ccc3d6c7a..b487e8060 100644
--- a/src/main/java/net/minecraft/server/RegionFile.java
+++ b/src/main/java/net/minecraft/server/RegionFile.java
@@ -31,6 +31,30 @@ public class RegionFile implements AutoCloseable {
private final int[] d = new int[1024]; private final int[] timestamps = d; // Paper - OBFHELPER
private final List<Boolean> e; // PAIL freeSectors
+ // Paper start - Cache chunk status
+ private final ChunkStatus[] statuses = new ChunkStatus[32 * 32];
+
+ private boolean closed;
+
+ // invoked on write/read
+ public void setStatus(int x, int z, ChunkStatus status) {
+ if (this.closed) {
+ // We've used an invalid region file.
+ throw new IllegalStateException("RegionFile is closed");
+ }
+ this.statuses[this.getChunkLocation(new ChunkCoordIntPair(x, z))] = status;
+ }
+
+ public ChunkStatus getStatusIfCached(int x, int z) {
+ if (this.closed) {
+ // We've used an invalid region file.
+ throw new IllegalStateException("RegionFile is closed");
+ }
+ final int location = this.getChunkLocation(new ChunkCoordIntPair(x, z));
+ return this.statuses[location];
+ }
+ // Paper end
+
public RegionFile(File file) throws IOException {
this.b = new RandomAccessFile(file, "rw");
this.file = file; // Spigot // Paper - We need this earlier
@@ -291,6 +315,7 @@ public class RegionFile implements AutoCloseable {
return this.c[this.f(chunkcoordintpair)];
}
+ public final boolean chunkExists(ChunkCoordIntPair chunkPos) { return this.d(chunkPos); } // Paper - OBFHELPER
public boolean d(ChunkCoordIntPair chunkcoordintpair) {
return this.getOffset(chunkcoordintpair) != 0;
}
@@ -304,6 +329,7 @@ public class RegionFile implements AutoCloseable {
this.c[j] = i; // Spigot - move this to after the write
}
+ private final int getChunkLocation(ChunkCoordIntPair chunkcoordintpair) { return this.f(chunkcoordintpair); } // Paper - OBFHELPER
private int f(ChunkCoordIntPair chunkcoordintpair) {
return chunkcoordintpair.j() + chunkcoordintpair.k() * 32;
}
@@ -318,6 +344,7 @@ public class RegionFile implements AutoCloseable {
}
public void close() throws IOException {
+ this.closed = true; // Paper
this.b.close();
}
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
index 6f34d8aea..d2b328945 100644
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
@@ -47,6 +47,12 @@ public abstract class RegionFileCache implements AutoCloseable {
// Paper start
}
+ // Paper start
+ public RegionFile getRegionFileIfLoaded(ChunkCoordIntPair chunkcoordintpair) {
+ return this.cache.getAndMoveToFirst(ChunkCoordIntPair.pair(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()));
+ }
+ // Paper end
+
public RegionFile getRegionFile(ChunkCoordIntPair chunkcoordintpair, boolean existingOnly) throws IOException { return this.a(chunkcoordintpair, existingOnly); } // Paper - OBFHELPER
private RegionFile a(ChunkCoordIntPair chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit
long i = ChunkCoordIntPair.pair(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ());
@@ -110,6 +116,7 @@ public abstract class RegionFileCache implements AutoCloseable {
try {
NBTCompressedStreamTools.writeNBT(nbttagcompound, out);
out.close();
+ regionfile.setStatus(chunk.x, chunk.z, ChunkRegionLoader.getStatus(nbttagcompound)); // Paper - cache status on disk
regionfile.setOversized(chunkX, chunkZ, false);
} catch (RegionFile.ChunkTooLargeException ignored) {
printOversizedLog("ChunkTooLarge! Someone is trying to duplicate.", regionfile.file, chunkX, chunkZ);
@@ -127,6 +134,7 @@ public abstract class RegionFileCache implements AutoCloseable {
if (SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD) {
resetFilterThresholds();
}
+ regionfile.setStatus(chunk.x, chunk.z, ChunkRegionLoader.getStatus(nbttagcompound)); // Paper - cache status on disk
} catch (RegionFile.ChunkTooLargeException e) {
printOversizedLog("ChunkTooLarge even after reduction. Trying in overzealous mode.", regionfile.file, chunkX, chunkZ);
// Eek, major fail. We have retry logic, so reduce threshholds and fall back
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
index e42bd2638..2227de3bf 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
@@ -18,6 +18,7 @@ import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
@@ -408,8 +409,22 @@ public class CraftWorld implements World {
@Override
public boolean isChunkGenerated(int x, int z) {
+ // Paper start - Fix this method
+ if (!Bukkit.isPrimaryThread()) {
+ return CompletableFuture.supplyAsync(() -> {
+ return CraftWorld.this.isChunkGenerated(x, z);
+ }, world.getChunkProvider().serverThreadQueue).join();
+ }
+ IChunkAccess chunk = world.getChunkProvider().getChunkAtImmediately(x, z);
+ if (chunk == null) {
+ chunk = world.getChunkProvider().playerChunkMap.getUnloadingChunk(x, z);
+ }
+ if (chunk != null) {
+ return chunk instanceof ProtoChunkExtension || chunk instanceof net.minecraft.server.Chunk;
+ }
try {
- return world.getChunkProvider().getChunkAtIfCachedImmediately(x, z) != null || world.getChunkProvider().playerChunkMap.chunkExists(new ChunkCoordIntPair(x, z)); // Paper
+ return world.getChunkProvider().playerChunkMap.getChunkStatusOnDisk(new ChunkCoordIntPair(x, z)) == ChunkStatus.FULL;
+ // Paper end
} catch (IOException ex) {
throw new RuntimeException(ex);
}
@@ -521,20 +536,49 @@ public class CraftWorld implements World {
@Override
public boolean loadChunk(int x, int z, boolean generate) {
org.spigotmc.AsyncCatcher.catchOp("chunk load"); // Spigot
- IChunkAccess chunk = world.getChunkProvider().getChunkAt(x, z, generate || isChunkGenerated(x, z) ? ChunkStatus.FULL : ChunkStatus.EMPTY, true); // Paper
+ // Paper start - Optimize this method
+ ChunkCoordIntPair chunkPos = new ChunkCoordIntPair(x, z);
- // If generate = false, but the chunk already exists, we will get this back.
- if (chunk instanceof ProtoChunkExtension) {
- // We then cycle through again to get the full chunk immediately, rather than after the ticket addition
- chunk = world.getChunkProvider().getChunkAt(x, z, ChunkStatus.FULL, true);
- }
+ if (!generate) {
- if (chunk instanceof net.minecraft.server.Chunk) {
- world.getChunkProvider().addTicket(TicketType.PLUGIN, new ChunkCoordIntPair(x, z), 1, Unit.INSTANCE);
- return true;
+ IChunkAccess immediate = world.getChunkProvider().getChunkAtImmediately(x, z);
+ if (immediate == null) {
+ immediate = world.getChunkProvider().playerChunkMap.getUnloadingChunk(x, z);
+ }
+ if (immediate != null) {
+ if (!(immediate instanceof ProtoChunkExtension) && !(immediate instanceof net.minecraft.server.Chunk)) {
+ return false; // not full status
+ }
+ world.getChunkProvider().addTicket(TicketType.PLUGIN, chunkPos, 1, Unit.INSTANCE);
+ world.getChunkAt(x, z); // make sure we're at ticket level 32 or lower
+ return true;
+ }
+
+ net.minecraft.server.RegionFile file;
+ try {
+ file = world.getChunkProvider().playerChunkMap.getRegionFile(chunkPos, false);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+
+ ChunkStatus status = file.getStatusIfCached(x, z);
+ if (!file.chunkExists(chunkPos) || (status != null && status != ChunkStatus.FULL)) {
+ return false;
+ }
+
+ IChunkAccess chunk = world.getChunkProvider().getChunkAt(x, z, ChunkStatus.EMPTY, true);
+ if (!(chunk instanceof ProtoChunkExtension) && !(chunk instanceof net.minecraft.server.Chunk)) {
+ return false;
+ }
+
+ // fall through to load
+ // we do this so we do not re-read the chunk data on disk
}
- return false;
+ world.getChunkProvider().addTicket(TicketType.PLUGIN, chunkPos, 1, Unit.INSTANCE);
+ world.getChunkProvider().getChunkAt(x, z, ChunkStatus.FULL, true);
+ return true;
+ // Paper end
}
@Override
--
2.24.0

83
removed/1.15/0399-Use-getChunkIfLoadedImmediately-in-places.patch Normal file
View file

@ -0,0 +1,83 @@
From 59cfc5788449e1f54560c9c2ff59b4ff2efd6b5d Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Mon, 8 Jul 2019 00:13:36 -0700
Subject: [PATCH] Use getChunkIfLoadedImmediately in places
This prevents us from hitting chunk loads for chunks at or less-than
ticket level 33 (yes getChunkIfLoaded will actually perform a chunk
load in that case).
diff --git a/src/main/java/net/minecraft/server/Entity.java b/src/main/java/net/minecraft/server/Entity.java
index 55c73ffca..e8def7f81 100644
--- a/src/main/java/net/minecraft/server/Entity.java
+++ b/src/main/java/net/minecraft/server/Entity.java
@@ -199,7 +199,7 @@ public abstract class Entity implements INamableTileEntity, ICommandListener, Ke
}
public boolean isChunkLoaded() {
- return world.isChunkLoaded((int) Math.floor(this.locX) >> 4, (int) Math.floor(this.locZ) >> 4);
+ return world.getChunkIfLoadedImmediately((int) Math.floor(this.locX) >> 4, (int) Math.floor(this.locZ) >> 4) != null; // Paper
}
// CraftBukkit end
diff --git a/src/main/java/net/minecraft/server/PlayerConnection.java b/src/main/java/net/minecraft/server/PlayerConnection.java
index 4a16d6c14..ae17cdf23 100644
--- a/src/main/java/net/minecraft/server/PlayerConnection.java
+++ b/src/main/java/net/minecraft/server/PlayerConnection.java
@@ -981,7 +981,7 @@ public class PlayerConnection implements PacketListenerPlayIn {
speed = player.abilities.walkSpeed * 10f;
}
// Paper start - Prevent moving into unloaded chunks
- if (player.world.paperConfig.preventMovingIntoUnloadedChunks && (this.player.locX != toX || this.player.locZ != toZ) && !worldserver.isChunkLoaded((int) Math.floor(toX) >> 4, (int) Math.floor(toZ) >> 4)) {
+ if (player.world.paperConfig.preventMovingIntoUnloadedChunks && (this.player.locX != toX || this.player.locZ != toZ) && worldserver.getChunkIfLoadedImmediately((int) Math.floor(toX) >> 4, (int) Math.floor(toZ) >> 4) == null) { // Paper - use getIfLoadedImmediately
this.internalTeleport(this.player.locX, this.player.locY, this.player.locZ, this.player.yaw, this.player.pitch, Collections.emptySet());
return;
}
diff --git a/src/main/java/net/minecraft/server/World.java b/src/main/java/net/minecraft/server/World.java
index ab98c7b79..b81b37445 100644
--- a/src/main/java/net/minecraft/server/World.java
+++ b/src/main/java/net/minecraft/server/World.java
@@ -131,7 +131,7 @@ public abstract class World implements IIBlockAccess, GeneratorAccess, AutoClose
}
public Chunk getChunkIfLoaded(int x, int z) {
- return ((ChunkProviderServer) this.chunkProvider).getChunkAt(x, z, false);
+ return ((ChunkProviderServer) this.chunkProvider).getChunkAtIfLoadedImmediately(x, z); // Paper
}
protected World(WorldData worlddata, DimensionManager dimensionmanager, BiFunction<World, WorldProvider, IChunkProvider> bifunction, GameProfilerFiller gameprofilerfiller, boolean flag, org.bukkit.generator.ChunkGenerator gen, org.bukkit.World.Environment env) {
@@ -274,12 +274,12 @@ public abstract class World implements IIBlockAccess, GeneratorAccess, AutoClose
}
public boolean isLoaded(BlockPosition blockposition) {
- return getChunkIfLoaded(blockposition.getX() >> 4, blockposition.getZ() >> 4) != null; // Paper
+ return getChunkIfLoadedImmediately(blockposition.getX() >> 4, blockposition.getZ() >> 4) != null; // Paper
}
// Paper start
public boolean isLoadedAndInBounds(BlockPosition blockposition) {
- return getWorldBorder().isInBounds(blockposition) && getChunkIfLoaded(blockposition.getX() >> 4, blockposition.getZ() >> 4) != null;
+ return getWorldBorder().isInBounds(blockposition) && getChunkIfLoadedImmediately(blockposition.getX() >> 4, blockposition.getZ() >> 4) != null;
}
public Chunk getChunkIfLoaded(BlockPosition blockposition) {
return getChunkIfLoaded(blockposition.getX() >> 4, blockposition.getZ() >> 4);
diff --git a/src/main/java/org/spigotmc/ActivationRange.java b/src/main/java/org/spigotmc/ActivationRange.java
index f86404f83..92601c581 100644
--- a/src/main/java/org/spigotmc/ActivationRange.java
+++ b/src/main/java/org/spigotmc/ActivationRange.java
@@ -143,9 +143,10 @@ public class ActivationRange
{
for ( int j1 = k; j1 <= l; ++j1 )
{
- if ( world.getWorld().isChunkLoaded( i1, j1 ) )
+ Chunk chunk = (Chunk) world.getChunkIfLoadedImmediately( i1, j1 );
+ if ( chunk != null )
{
- activateChunkEntities( world.getChunkAt( i1, j1 ) );
+ activateChunkEntities( chunk );
}
}
}
--
2.23.0

23
removed/1.15/0403-Preserve-old-flush-on-save-flag-for-reliable-regionf.patch Normal file
View file

@ -0,0 +1,23 @@
From 98e10786918880c728f3afc751290cdf5eb14cf5 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Mon, 5 Aug 2019 08:24:01 -0700
Subject: [PATCH] Preserve old flush on save flag for reliable regionfiles
Originally this patch was in paper
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
index b487e8060..a8c8ace46 100644
--- a/src/main/java/net/minecraft/server/RegionFile.java
+++ b/src/main/java/net/minecraft/server/RegionFile.java
@@ -349,7 +349,7 @@ public class RegionFile implements AutoCloseable {
}
// Spigot start - Make region files reliable
- private static final boolean FLUSH_ON_SAVE = Boolean.getBoolean("spigot.flush-on-save");
+ private static final boolean FLUSH_ON_SAVE = Boolean.getBoolean("spigot.flush-on-save") || Boolean.getBoolean("paper.flush-on-save"); // Paper - preserve old flag
private void syncRegionFile() throws IOException {
if (!FLUSH_ON_SAVE) {
return;
--
2.23.0

3917
removed/1.15/0409-Asynchronous-chunk-IO-and-loading.patch Normal file
View file

File diff suppressed because it is too large Load diff

349
removed/1.15/0410-Reduce-sync-loads.patch Normal file
View file

@ -0,0 +1,349 @@
From f90a04ddc82483f855e77ec91c3de3f9e2befdf6 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Fri, 19 Jul 2019 03:29:14 -0700
Subject: [PATCH] Reduce sync loads
This reduces calls to getChunkAt which would load chunks.
This patch also adds a tool to find calls which are doing this, however
it must be enabled by setting the startup flag -Dpaper.debug-sync-loads=true
To get a debug log for sync loads, the command is /paper syncloadinfo
diff --git a/src/main/java/com/destroystokyo/paper/PaperCommand.java b/src/main/java/com/destroystokyo/paper/PaperCommand.java
index 09efbf725..132397b3f 100644
--- a/src/main/java/com/destroystokyo/paper/PaperCommand.java
+++ b/src/main/java/com/destroystokyo/paper/PaperCommand.java
@@ -1,9 +1,13 @@
package com.destroystokyo.paper;
+import com.destroystokyo.paper.io.SyncLoadFinder;
import com.google.common.base.Functions;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.gson.JsonObject;
+import com.google.gson.internal.Streams;
+import com.google.gson.stream.JsonWriter;
import net.minecraft.server.*;
import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.commons.lang3.tuple.Pair;
@@ -18,6 +22,9 @@ import org.bukkit.craftbukkit.CraftWorld;
import org.bukkit.entity.Player;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.io.StringWriter;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
@@ -130,6 +137,9 @@ public class PaperCommand extends Command {
case "chunkinfo":
doChunkInfo(sender, args);
break;
+ case "syncloadinfo":
+ this.doSyncLoadInfo(sender, args);
+ break;
case "ver":
case "version":
Command ver = org.bukkit.Bukkit.getServer().getCommandMap().getCommand("version");
@@ -146,6 +156,40 @@ public class PaperCommand extends Command {
return true;
}
+ private void doSyncLoadInfo(CommandSender sender, String[] args) {
+ if (!SyncLoadFinder.ENABLED) {
+ sender.sendMessage(ChatColor.RED + "This command requires the server startup flag '-Dpaper.debug-sync-loads=true' to be set.");
+ return;
+ }
+ File file = new File(new File(new File("."), "debug"),
+ "sync-load-info" + DateTimeFormatter.ofPattern("yyyy-MM-dd_HH.mm.ss").format(LocalDateTime.now()) + ".txt");
+ file.getParentFile().mkdirs();
+ sender.sendMessage(ChatColor.GREEN + "Writing sync load info to " + file.toString());
+
+
+ try {
+ final JsonObject data = SyncLoadFinder.serialize();
+
+ StringWriter stringWriter = new StringWriter();
+ JsonWriter jsonWriter = new JsonWriter(stringWriter);
+ jsonWriter.setIndent(" ");
+ jsonWriter.setLenient(false);
+ Streams.write(data, jsonWriter);
+
+ String fileData = stringWriter.toString();
+
+ try (
+ PrintStream out = new PrintStream(new FileOutputStream(file), false, "UTF-8")
+ ) {
+ out.print(fileData);
+ }
+ sender.sendMessage(ChatColor.GREEN + "Successfully written sync load information!");
+ } catch (Throwable thr) {
+ sender.sendMessage(ChatColor.RED + "Failed to write sync load information");
+ thr.printStackTrace();
+ }
+ }
+
private void doChunkInfo(CommandSender sender, String[] args) {
List<org.bukkit.World> worlds;
if (args.length < 2 || args[1].equals("*")) {
diff --git a/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java b/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java
new file mode 100644
index 000000000..59aec1032
--- /dev/null
+++ b/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java
@@ -0,0 +1,172 @@
+package com.destroystokyo.paper.io;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.mojang.datafixers.util.Pair;
+import it.unimi.dsi.fastutil.longs.Long2IntMap;
+import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
+import net.minecraft.server.World;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
+
+public class SyncLoadFinder {
+
+ public static final boolean ENABLED = Boolean.getBoolean("paper.debug-sync-loads");
+
+ private static final WeakHashMap<World, Object2ObjectOpenHashMap<ThrowableWithEquals, SyncLoadInformation>> SYNC_LOADS = new WeakHashMap<>();
+
+ private static final class SyncLoadInformation {
+
+ public int times;
+
+ public final Long2IntOpenHashMap coordinateTimes = new Long2IntOpenHashMap();
+ }
+
+ public static void logSyncLoad(final World world, final int chunkX, final int chunkZ) {
+ if (!ENABLED) {
+ return;
+ }
+
+ final ThrowableWithEquals stacktrace = new ThrowableWithEquals(Thread.currentThread().getStackTrace());
+
+ SYNC_LOADS.compute(world, (final World keyInMap, Object2ObjectOpenHashMap<ThrowableWithEquals, SyncLoadInformation> map) -> {
+ if (map == null) {
+ map = new Object2ObjectOpenHashMap<>();
+ }
+
+ map.compute(stacktrace, (ThrowableWithEquals keyInMap0, SyncLoadInformation valueInMap) -> {
+ if (valueInMap == null) {
+ valueInMap = new SyncLoadInformation();
+ }
+
+ ++valueInMap.times;
+
+ valueInMap.coordinateTimes.compute(IOUtil.getCoordinateKey(chunkX, chunkZ), (Long keyInMap1, Integer valueInMap1) -> {
+ return valueInMap1 == null ? Integer.valueOf(1) : Integer.valueOf(valueInMap1.intValue() + 1);
+ });
+
+ return valueInMap;
+ });
+
+ return map;
+ });
+ }
+
+ public static JsonObject serialize() {
+ final JsonObject ret = new JsonObject();
+
+ final JsonArray worldsData = new JsonArray();
+
+ for (final Map.Entry<World, Object2ObjectOpenHashMap<ThrowableWithEquals, SyncLoadInformation>> entry : SYNC_LOADS.entrySet()) {
+ final World world = entry.getKey();
+
+ final JsonObject worldData = new JsonObject();
+
+ worldData.addProperty("name", world.getWorld().getName());
+
+ final List<Pair<ThrowableWithEquals, SyncLoadInformation>> data = new ArrayList<>();
+
+ entry.getValue().forEach((ThrowableWithEquals stacktrace, SyncLoadInformation times) -> {
+ data.add(new Pair<>(stacktrace, times));
+ });
+
+ data.sort((Pair<ThrowableWithEquals, SyncLoadInformation> pair1, Pair<ThrowableWithEquals, SyncLoadInformation> pair2) -> {
+ return Integer.compare(pair2.getSecond().times, pair1.getSecond().times); // reverse order
+ });
+
+ final JsonArray stacktraces = new JsonArray();
+
+ for (Pair<ThrowableWithEquals, SyncLoadInformation> pair : data) {
+ final JsonObject stacktrace = new JsonObject();
+
+ stacktrace.addProperty("times", pair.getSecond().times);
+
+ final JsonArray traces = new JsonArray();
+
+ for (StackTraceElement element : pair.getFirst().stacktrace) {
+ traces.add(String.valueOf(element));
+ }
+
+ stacktrace.add("stacktrace", traces);
+
+ final JsonArray coordinates = new JsonArray();
+
+ for (Long2IntMap.Entry coordinate : pair.getSecond().coordinateTimes.long2IntEntrySet()) {
+ final long key = coordinate.getLongKey();
+ final int times = coordinate.getIntValue();
+ coordinates.add("(" + IOUtil.getCoordinateX(key) + "," + IOUtil.getCoordinateZ(key) + "): " + times);
+ }
+
+ stacktrace.add("coordinates", coordinates);
+
+ stacktraces.add(stacktrace);
+ }
+
+
+ worldData.add("stacktraces", stacktraces);
+ worldsData.add(worldData);
+ }
+
+ ret.add("worlds", worldsData);
+
+ return ret;
+ }
+
+ static final class ThrowableWithEquals {
+
+ private final StackTraceElement[] stacktrace;
+ private final int hash;
+
+ public ThrowableWithEquals(final StackTraceElement[] stacktrace) {
+ this.stacktrace = stacktrace;
+ this.hash = ThrowableWithEquals.hash(stacktrace);
+ }
+
+ public static int hash(final StackTraceElement[] stacktrace) {
+ int hash = 0;
+
+ for (int i = 0; i < stacktrace.length; ++i) {
+ hash *= 31;
+ hash += stacktrace[i].hashCode();
+ }
+
+ return hash;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.hash;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == null || obj.getClass() != this.getClass()) {
+ return false;
+ }
+
+ final ThrowableWithEquals other = (ThrowableWithEquals)obj;
+ final StackTraceElement[] otherStackTrace = other.stacktrace;
+
+ if (this.stacktrace.length != otherStackTrace.length) {
+ return false;
+ }
+
+ if (this == obj) {
+ return true;
+ }
+
+ for (int i = 0; i < this.stacktrace.length; ++i) {
+ if (!this.stacktrace[i].equals(otherStackTrace[i])) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }
+}
diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java
index 277c2245d..8d7971ad8 100644
--- a/src/main/java/net/minecraft/server/ChunkProviderServer.java
+++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java
@@ -280,6 +280,7 @@ public class ChunkProviderServer extends IChunkProvider {
this.world.asyncChunkTaskManager.raisePriority(x, z, com.destroystokyo.paper.io.PrioritizedTaskQueue.HIGHEST_PRIORITY);
com.destroystokyo.paper.io.chunk.ChunkTaskManager.pushChunkWait(this.world, x, z);
// Paper end
+ com.destroystokyo.paper.io.SyncLoadFinder.logSyncLoad(this.world, x, z); // Paper - sync load info
this.world.timings.chunkAwait.startTiming(); // Paper
this.serverThreadQueue.awaitTasks(completablefuture::isDone);
com.destroystokyo.paper.io.chunk.ChunkTaskManager.popChunkWait(); // Paper - async chunk debug
diff --git a/src/main/java/net/minecraft/server/World.java b/src/main/java/net/minecraft/server/World.java
index b81b37445..9d29fc8ca 100644
--- a/src/main/java/net/minecraft/server/World.java
+++ b/src/main/java/net/minecraft/server/World.java
@@ -1195,14 +1195,14 @@ public abstract class World implements IIBlockAccess, GeneratorAccess, AutoClose
}
public boolean n(BlockPosition blockposition) {
- return isOutsideWorld(blockposition) ? false : this.chunkProvider.b(blockposition.getX() >> 4, blockposition.getZ() >> 4);
+ return isOutsideWorld(blockposition) ? false : this.isLoaded(blockposition); // Paper - reduce sync loads
}
public boolean a(BlockPosition blockposition, Entity entity) {
if (isOutsideWorld(blockposition)) {
return false;
} else {
- IChunkAccess ichunkaccess = this.getChunkAt(blockposition.getX() >> 4, blockposition.getZ() >> 4, ChunkStatus.FULL, false);
+ IChunkAccess ichunkaccess = this.getChunkIfLoadedImmediately(blockposition.getX() >> 4, blockposition.getZ() >> 4); // Paper - reduce sync loads
return ichunkaccess == null ? false : ichunkaccess.getType(blockposition).a((IBlockAccess) this, blockposition, entity);
}
@@ -1249,7 +1249,7 @@ public abstract class World implements IIBlockAccess, GeneratorAccess, AutoClose
for (int i1 = i; i1 <= j; ++i1) {
for (int j1 = k; j1 <= l; ++j1) {
- Chunk chunk = this.getChunkProvider().getChunkAt(i1, j1, false);
+ Chunk chunk = (Chunk)this.getChunkIfLoadedImmediately(i1, j1); // Paper
if (chunk != null) {
chunk.a(entity, axisalignedbb, list, predicate);
@@ -1269,7 +1269,7 @@ public abstract class World implements IIBlockAccess, GeneratorAccess, AutoClose
for (int i1 = i; i1 < j; ++i1) {
for (int j1 = k; j1 < l; ++j1) {
- Chunk chunk = this.getChunkProvider().getChunkAt(i1, j1, false);
+ Chunk chunk = (Chunk)this.getChunkIfLoadedImmediately(i1, j1); // Paper
if (chunk != null) {
chunk.a(entitytypes, axisalignedbb, list, predicate);
@@ -1291,7 +1291,7 @@ public abstract class World implements IIBlockAccess, GeneratorAccess, AutoClose
for (int i1 = i; i1 < j; ++i1) {
for (int j1 = k; j1 < l; ++j1) {
- Chunk chunk = ichunkprovider.getChunkAt(i1, j1, false);
+ Chunk chunk = (Chunk)this.getChunkIfLoadedImmediately(i1, j1); // Paper
if (chunk != null) {
chunk.a(oclass, axisalignedbb, list, predicate);
diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java
index 3a6745dcc..b7878201b 100644
--- a/src/main/java/net/minecraft/server/WorldServer.java
+++ b/src/main/java/net/minecraft/server/WorldServer.java
@@ -151,6 +151,12 @@ public class WorldServer extends World {
};
public final com.destroystokyo.paper.io.chunk.ChunkTaskManager asyncChunkTaskManager;
// Paper end
+ // Paper start
+ @Override
+ public boolean isChunkLoaded(int x, int z) {
+ return this.getChunkProvider().getChunkAtIfLoadedImmediately(x, z) != null;
+ }
+ // Paper end
// Add env and gen to constructor
public WorldServer(MinecraftServer minecraftserver, Executor executor, WorldNBTStorage worldnbtstorage, WorldData worlddata, DimensionManager dimensionmanager, GameProfilerFiller gameprofilerfiller, WorldLoadListener worldloadlistener, org.bukkit.World.Environment env, org.bukkit.generator.ChunkGenerator gen) {
--
2.23.0

30
removed/1.15/0412-Improve-POI-data-saving-logic.patch Normal file
View file

@ -0,0 +1,30 @@
From cf3689f611fad7d903831b63086deefad3cd8e92 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Mon, 19 Aug 2019 06:33:17 -0700
Subject: [PATCH] Improve POI data saving logic
- Do not unload data if world saving is disabled
- Aggressively target unloading
diff --git a/src/main/java/net/minecraft/server/VillagePlace.java b/src/main/java/net/minecraft/server/VillagePlace.java
index 0e98b7803..fb99b4306 100644
--- a/src/main/java/net/minecraft/server/VillagePlace.java
+++ b/src/main/java/net/minecraft/server/VillagePlace.java
@@ -132,9 +132,12 @@ public class VillagePlace extends RegionFileSection<VillagePlaceSection> {
// Paper start - async chunk io
if (this.world == null) {
super.a(booleansupplier);
- } else {
+ } else if (!this.world.isSavingDisabled()) { // Paper - only save if saving is enabled
//super.a(booleansupplier); // re-implement below
- while (!((RegionFileSection)this).d.isEmpty() && booleansupplier.getAsBoolean()) {
+ // Paper start - target unloading aggressively
+ int queueTarget = Math.min(this.d.size() - 100, (int)(this.d.size() * 0.96));
+ while (!((RegionFileSection)this).d.isEmpty() && (this.d.size() > queueTarget || booleansupplier.getAsBoolean())) {
+ // Paper end
ChunkCoordIntPair chunkcoordintpair = SectionPosition.a(((RegionFileSection)this).d.firstLong()).u();
NBTTagCompound data;
--
2.23.0