mirrors/papermc
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
papermc/patches/server/0397-Optimize-PlayerChunkMap-memory-use-for-visibleChunks.patch
Jake Potrebic a6004d4f7f
fixes issue with chunk saving 
fixes #5854
2021-06-16 13:35:10 -07:00

299 lines
17 KiB
Diff
Raw Blame History

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Wed, 8 Apr 2020 03:06:30 -0400
Subject: [PATCH] Optimize PlayerChunkMap memory use for visibleChunks
No longer clones visible chunks which is causing massive memory
allocation issues, likely the source of Humongous Objects on large servers.
Instead we just synchronize, clear and rebuild, reusing the same object buffers
as before with only 2 small objects created (FastIterator/MapEntry)
This should result in siginificant memory use reduction and improved GC behavior.
diff --git a/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
new file mode 100644
index 0000000000000000000000000000000000000000..f6ff4d8132a95895680f5bc81f8f873e78f0bbdb
--- /dev/null
+++ b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
@@ -0,0 +1,39 @@
+package com.destroystokyo.paper.util.map;
+
+import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
+
+public class Long2ObjectLinkedOpenHashMapFastCopy<V> extends Long2ObjectLinkedOpenHashMap<V> {
+
+ public void copyFrom(Long2ObjectLinkedOpenHashMapFastCopy<V> map) {
+ if (key.length != map.key.length) {
+ key = null;
+ key = new long[map.key.length];
+ }
+ if (value.length != map.value.length) {
+ value = null;
+ //noinspection unchecked
+ value = (V[]) new Object[map.value.length];
+ }
+ if (link.length != map.link.length) {
+ link = null;
+ link = new long[map.link.length];
+ }
+ System.arraycopy(map.key, 0, this.key, 0, map.key.length);
+ System.arraycopy(map.value, 0, this.value, 0, map.value.length);
+ System.arraycopy(map.link, 0, this.link, 0, map.link.length);
+ this.size = map.size;
+ this.mask = map.mask;
+ this.first = map.first;
+ this.last = map.last;
+ this.n = map.n;
+ this.maxFill = map.maxFill;
+ this.containsNullKey = map.containsNullKey;
+ }
+
+ @Override
+ public Long2ObjectLinkedOpenHashMapFastCopy<V> clone() {
+ Long2ObjectLinkedOpenHashMapFastCopy<V> clone = (Long2ObjectLinkedOpenHashMapFastCopy<V>) super.clone();
+ clone.copyFrom(this);
+ return clone;
+ }
+}
diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java
index 9c88426ab1275ee5fb6e28be8b213533dc4ab859..87c9a5c1b43f6010898d72136b5eb9973299b723 100644
--- a/src/main/java/net/minecraft/server/MCUtil.java
+++ b/src/main/java/net/minecraft/server/MCUtil.java
@@ -614,7 +614,7 @@ public final class MCUtil {
ServerLevel world = ((org.bukkit.craftbukkit.CraftWorld)bukkitWorld).getHandle();
ChunkMap chunkMap = world.getChunkSource().chunkMap;
- Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = chunkMap.visibleChunkMap;
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = chunkMap.getVisibleChunks();
DistanceManager chunkMapDistance = chunkMap.distanceManager;
List<ChunkHolder> allChunks = new ArrayList<>(visibleChunks.values());
List<ServerPlayer> players = world.players;
diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java
index acd7318af28022b196d7397601d3511d7f199821..6947bc74216704d1848387e2b2a8bb57d5105c0f 100644
--- a/src/main/java/net/minecraft/server/level/ChunkMap.java
+++ b/src/main/java/net/minecraft/server/level/ChunkMap.java
@@ -112,9 +112,36 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
private static final int MIN_VIEW_DISTANCE = 3;
public static final int MAX_VIEW_DISTANCE = 33;
public static final int MAX_CHUNK_DISTANCE = 33 + ChunkStatus.maxDistance();
+ // Paper start - faster copying
+ public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<>(); // Paper - faster copying
+ public final Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap = new ProtectedVisibleChunksMap(); // Paper - faster copying
+
+ private class ProtectedVisibleChunksMap extends com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> {
+ @Override
+ public ChunkHolder put(long k, ChunkHolder playerChunk) {
+ throw new UnsupportedOperationException("Updating visible Chunks");
+ }
+
+ @Override
+ public ChunkHolder remove(long k) {
+ throw new UnsupportedOperationException("Removing visible Chunks");
+ }
+
+ @Override
+ public ChunkHolder get(long k) {
+ return ChunkMap.this.getVisibleChunkIfPresent(k);
+ }
+
+ public ChunkHolder safeGet(long k) {
+ return super.get(k);
+ }
+ }
+ // Paper end
+ public final com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> pendingVisibleChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>(); // Paper - this is used if the visible chunks is updated while iterating only
+ public transient com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> visibleChunksClone; // Paper - used for async access of visible chunks, clone and cache only when needed
public static final int FORCED_TICKET_LEVEL = 31;
- public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new Long2ObjectLinkedOpenHashMap();
- public volatile Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap;
+ // public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new Long2ObjectLinkedOpenHashMap(); // Paper - moved up
+ // public volatile Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap; // Paper - moved up
private final Long2ObjectLinkedOpenHashMap<ChunkHolder> pendingUnloads;
public final LongSet entitiesInLevel;
public final ServerLevel level;
@@ -237,7 +264,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
public ChunkMap(ServerLevel world, LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, StructureManager structureManager, Executor executor, BlockableEventLoop<Runnable> mainThreadExecutor, LightChunkGetter chunkProvider, ChunkGenerator chunkGenerator, ChunkProgressListener worldGenerationProgressListener, ChunkStatusUpdateListener chunkStatusChangeListener, Supplier<DimensionDataStorage> persistentStateManagerFactory, int viewDistance, boolean dsync) {
super(new File(session.getDimensionPath(world.dimension()), "region"), dataFixer, dsync);
- this.visibleChunkMap = this.updatingChunkMap.clone();
+ //this.visibleChunks = this.updatingChunks.clone(); // Paper - no more cloning
this.pendingUnloads = new Long2ObjectLinkedOpenHashMap();
this.entitiesInLevel = new LongOpenHashSet();
this.toDrop = new LongOpenHashSet();
@@ -376,9 +403,52 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
return (ChunkHolder) this.updatingChunkMap.get(pos);
}
+ // Paper start - remove cloning of visible chunks unless accessed as a collection async
+ private static final boolean DEBUG_ASYNC_VISIBLE_CHUNKS = Boolean.getBoolean("paper.debug-async-visible-chunks");
+ private boolean isIterating = false;
+ private boolean hasPendingVisibleUpdate = false;
+ public void forEachVisibleChunk(java.util.function.Consumer<ChunkHolder> consumer) {
+ org.spigotmc.AsyncCatcher.catchOp("forEachVisibleChunk");
+ boolean prev = isIterating;
+ isIterating = true;
+ try {
+ for (ChunkHolder value : this.visibleChunkMap.values()) {
+ consumer.accept(value);
+ }
+ } finally {
+ this.isIterating = prev;
+ if (!this.isIterating && this.hasPendingVisibleUpdate) {
+ ((ProtectedVisibleChunksMap)this.visibleChunkMap).copyFrom(this.pendingVisibleChunks);
+ this.pendingVisibleChunks.clear();
+ this.hasPendingVisibleUpdate = false;
+ }
+ }
+ }
+ public Long2ObjectLinkedOpenHashMap<ChunkHolder> getVisibleChunks() {
+ if (Thread.currentThread() == this.level.thread) {
+ return this.visibleChunkMap;
+ } else {
+ synchronized (this.visibleChunkMap) {
+ if (DEBUG_ASYNC_VISIBLE_CHUNKS) new Throwable("Async getVisibleChunks").printStackTrace();
+ if (this.visibleChunksClone == null) {
+ this.visibleChunksClone = this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.clone() : ((ProtectedVisibleChunksMap)this.visibleChunkMap).clone();
+ }
+ return this.visibleChunksClone;
+ }
+ }
+ }
+ // Paper end
+
@Nullable
public ChunkHolder getVisibleChunkIfPresent(long pos) {
- return (ChunkHolder) this.visibleChunkMap.get(pos);
+ // Paper start - mt safe get
+ if (Thread.currentThread() != this.level.thread) {
+ synchronized (this.visibleChunkMap) {
+ return (ChunkHolder) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(pos) : ((ProtectedVisibleChunksMap)this.visibleChunkMap).safeGet(pos));
+ }
+ }
+ return (ChunkHolder) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(pos) : ((ProtectedVisibleChunksMap)this.visibleChunkMap).safeGet(pos));
+ // Paper end
}
protected IntSupplier getChunkQueueLevel(long pos) {
@@ -535,8 +605,9 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
}
protected void saveAllChunks(boolean flush) {
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = this.getVisibleChunks(); // Paper remove clone of visible Chunks unless saving off main thread (watchdog kill)
if (flush) {
- List<ChunkHolder> list = (List) this.visibleChunkMap.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).peek(ChunkHolder::refreshAccessibility).collect(Collectors.toList());
+ List<ChunkHolder> list = (List) visibleChunks.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).peek(ChunkHolder::refreshAccessibility).collect(Collectors.toList()); // Paper - remove cloning of visible chunks
MutableBoolean mutableboolean = new MutableBoolean();
do {
@@ -567,7 +638,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
// this.i(); // Paper - nuke IOWorker
ChunkMap.LOGGER.info("ThreadedAnvilChunkStorage ({}): All chunks are saved", this.storageFolder.getName());
} else {
- this.visibleChunkMap.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).forEach((playerchunk) -> {
+ visibleChunks.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).forEach((playerchunk) -> {
ChunkAccess ichunkaccess = (ChunkAccess) playerchunk.getChunkToSave().getNow(null); // CraftBukkit - decompile error
if (ichunkaccess instanceof ImposterProtoChunk || ichunkaccess instanceof LevelChunk) {
@@ -725,7 +796,20 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
if (!this.modified) {
return false;
} else {
- this.visibleChunkMap = this.updatingChunkMap.clone();
+ // Paper start - stop cloning visibleChunks
+ synchronized (this.visibleChunkMap) {
+ if (isIterating) {
+ hasPendingVisibleUpdate = true;
+ this.pendingVisibleChunks.copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>)this.updatingChunkMap);
+ } else {
+ hasPendingVisibleUpdate = false;
+ this.pendingVisibleChunks.clear();
+ ((ProtectedVisibleChunksMap)this.visibleChunkMap).copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>)this.updatingChunkMap);
+ this.visibleChunksClone = null;
+ }
+ }
+ // Paper end
+
this.modified = false;
return true;
}
@@ -1172,12 +1256,12 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
}
protected Iterable<ChunkHolder> getChunks() {
- return Iterables.unmodifiableIterable(this.visibleChunkMap.values());
+ return Iterables.unmodifiableIterable(this.getVisibleChunks().values()); // Paper
}
void dumpChunks(Writer writer) throws IOException {
CsvOutput csvwriter = CsvOutput.builder().addColumn("x").addColumn("z").addColumn("level").addColumn("in_memory").addColumn("status").addColumn("full_status").addColumn("accessible_ready").addColumn("ticking_ready").addColumn("entity_ticking_ready").addColumn("ticket").addColumn("spawning").addColumn("block_entity_count").build(writer);
- ObjectBidirectionalIterator objectbidirectionaliterator = this.visibleChunkMap.long2ObjectEntrySet().iterator();
+ ObjectBidirectionalIterator objectbidirectionaliterator = this.getVisibleChunks().long2ObjectEntrySet().iterator(); // Paper
while (objectbidirectionaliterator.hasNext()) {
Entry<ChunkHolder> entry = (Entry) objectbidirectionaliterator.next();
diff --git a/src/main/java/net/minecraft/server/level/ServerChunkCache.java b/src/main/java/net/minecraft/server/level/ServerChunkCache.java
index b0ff4a46807994e3afe4c8dc3810ecdd43b68025..d8cd652f76232b43b104b11247682288e99d9521 100644
--- a/src/main/java/net/minecraft/server/level/ServerChunkCache.java
+++ b/src/main/java/net/minecraft/server/level/ServerChunkCache.java
@@ -767,7 +767,7 @@ public class ServerChunkCache extends ChunkSource {
};
// Paper end
this.level.timings.chunkTicks.startTiming(); // Paper
- this.chunkMap.getChunks().forEach((playerchunk) -> { // Paper - no... just no...
+ this.chunkMap.forEachVisibleChunk((playerchunk) -> { // Paper - safe iterator incase chunk loads, also no wrapping
Optional<LevelChunk> optional = ((Either) playerchunk.getTickingChunkFuture().getNow(ChunkHolder.UNLOADED_LEVEL_CHUNK)).left();
if (optional.isPresent()) {
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
index b6134895d1b04d3ea7340e77f70efa23cff8b568..72c9ad9f75c20d6c1a6d54e2913e2f9918c11ffd 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
@@ -287,6 +287,7 @@ public class CraftWorld implements World {
@Override
public int getTileEntityCount() {
+ return net.minecraft.server.MCUtil.ensureMain(() -> {
// We don't use the full world tile entity list, so we must iterate chunks
Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = world.getChunkSource().chunkMap.visibleChunkMap;
int size = 0;
@@ -298,6 +299,7 @@ public class CraftWorld implements World {
size += chunk.blockEntities.size();
}
return size;
+ });
}
@Override
@@ -307,6 +309,7 @@ public class CraftWorld implements World {
@Override
public int getChunkCount() {
+ return net.minecraft.server.MCUtil.ensureMain(() -> {
int ret = 0;
for (ChunkHolder chunkHolder : world.getChunkSource().chunkMap.visibleChunkMap.values()) {
@@ -315,7 +318,7 @@ public class CraftWorld implements World {
}
}
- return ret;
+ return ret; });
}
@Override
@@ -442,6 +445,14 @@ public class CraftWorld implements World {
@Override
public Chunk[] getLoadedChunks() {
+ // Paper start
+ if (Thread.currentThread() != world.getLevel().thread) {
+ synchronized (world.getChunkSource().chunkMap.visibleChunkMap) {
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = world.getChunkSource().chunkMap.visibleChunkMap;
+ return chunks.values().stream().map(ChunkHolder::getFullChunk).filter(Objects::nonNull).map(net.minecraft.world.level.chunk.LevelChunk::getBukkitChunk).toArray(Chunk[]::new);
+ }
+ }
+ // Paper end
Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = this.world.getChunkSource().chunkMap.visibleChunkMap;
return chunks.values().stream().map(ChunkHolder::getFullChunk).filter(Objects::nonNull).map(net.minecraft.world.level.chunk.LevelChunk::getBukkitChunk).toArray(Chunk[]::new);
}
Reference in a new issue View git blame Copy permalink