382 lines
20 KiB
Diff
382 lines
20 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
|
|
Date: Thu, 20 May 2021 07:02:22 -0700
|
|
Subject: [PATCH] Fix and optimise world force upgrading
|
|
|
|
The WorldUpgrader class was incorrectly modified by
|
|
CB. It will store an IChunkLoader instance for all
|
|
dimension types in the world, but obviously with how
|
|
CB shifts around worlds only one dimension type exists
|
|
per world. But this would be OK if CB did this
|
|
change correctly. All IChunkLoader instances
|
|
will point to the same regionfiles. And all
|
|
IChunkLoader instances are going to be read from.
|
|
|
|
This problem hasn't really been reported because
|
|
it relies on the persistent legacy data to be converted
|
|
as well to cause corruption. Why? Because the legacy
|
|
data is also shared, it will result in different
|
|
outputs from conversion (as once conversion for legacy
|
|
persistent data takes place, it is REMOVED - so the next
|
|
convert will _not_ have the data). Which means different
|
|
sizes on disk. Which means different regionfile sector
|
|
allocations. Which means there are 3 different possible
|
|
regionfile sector allocations in memory, and none of them
|
|
are going to be correct.
|
|
|
|
I've fixed this by writing a world upgrader suited to
|
|
CB's changes to world folder format. It was brain dead
|
|
easy to add threading, so I did.
|
|
|
|
== AT ==
|
|
public net.minecraft.util.worldupdate.WorldUpgrader REGEX
|
|
|
|
diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
|
|
new file mode 100644
|
|
index 0000000000000000000000000000000000000000..513833c2ea23df5b079d157bc5cb89d5c9754c0b
|
|
--- /dev/null
|
|
+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
|
|
@@ -0,0 +1,209 @@
|
|
+package io.papermc.paper.world;
|
|
+
|
|
+import com.mojang.datafixers.DataFixer;
|
|
+import com.mojang.serialization.Codec;
|
|
+import net.minecraft.SharedConstants;
|
|
+import net.minecraft.nbt.CompoundTag;
|
|
+import net.minecraft.resources.ResourceKey;
|
|
+import net.minecraft.util.worldupdate.WorldUpgrader;
|
|
+import net.minecraft.world.level.ChunkPos;
|
|
+import net.minecraft.world.level.Level;
|
|
+import net.minecraft.world.level.chunk.ChunkGenerator;
|
|
+import net.minecraft.world.level.chunk.storage.ChunkStorage;
|
|
+import net.minecraft.world.level.chunk.storage.RegionFileStorage;
|
|
+import net.minecraft.world.level.dimension.DimensionType;
|
|
+import net.minecraft.world.level.dimension.LevelStem;
|
|
+import net.minecraft.world.level.levelgen.WorldGenSettings;
|
|
+import net.minecraft.world.level.storage.DimensionDataStorage;
|
|
+import net.minecraft.world.level.storage.LevelStorageSource;
|
|
+import org.apache.logging.log4j.LogManager;
|
|
+import org.apache.logging.log4j.Logger;
|
|
+import java.io.File;
|
|
+import java.io.IOException;
|
|
+import java.text.DecimalFormat;
|
|
+import java.util.Optional;
|
|
+import java.util.concurrent.ExecutorService;
|
|
+import java.util.concurrent.Executors;
|
|
+import java.util.concurrent.ThreadFactory;
|
|
+import java.util.concurrent.atomic.AtomicInteger;
|
|
+import java.util.concurrent.atomic.AtomicLong;
|
|
+import java.util.function.Supplier;
|
|
+
|
|
+public class ThreadedWorldUpgrader {
|
|
+
|
|
+ private static final Logger LOGGER = LogManager.getLogger();
|
|
+
|
|
+ private final ResourceKey<LevelStem> dimensionType;
|
|
+ private final String worldName;
|
|
+ private final File worldDir;
|
|
+ private final ExecutorService threadPool;
|
|
+ private final DataFixer dataFixer;
|
|
+ private final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
|
|
+ private final boolean removeCaches;
|
|
+
|
|
+ public ThreadedWorldUpgrader(final ResourceKey<LevelStem> dimensionType, final String worldName, final File worldDir, final int threads,
|
|
+ final DataFixer dataFixer, final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey, final boolean removeCaches) {
|
|
+ this.dimensionType = dimensionType;
|
|
+ this.worldName = worldName;
|
|
+ this.worldDir = worldDir;
|
|
+ this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() {
|
|
+ private final AtomicInteger threadCounter = new AtomicInteger();
|
|
+
|
|
+ @Override
|
|
+ public Thread newThread(final Runnable run) {
|
|
+ final Thread ret = new Thread(run);
|
|
+
|
|
+ ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement());
|
|
+ ret.setUncaughtExceptionHandler((thread, throwable) -> {
|
|
+ LOGGER.fatal("Error upgrading world", throwable);
|
|
+ });
|
|
+
|
|
+ return ret;
|
|
+ }
|
|
+ });
|
|
+ this.dataFixer = dataFixer;
|
|
+ this.generatorKey = generatorKey;
|
|
+ this.removeCaches = removeCaches;
|
|
+ }
|
|
+
|
|
+ public void convert() {
|
|
+ final File worldFolder = LevelStorageSource.getStorageFolder(this.worldDir.toPath(), this.dimensionType).toFile();
|
|
+ final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer);
|
|
+
|
|
+ final File regionFolder = new File(worldFolder, "region");
|
|
+
|
|
+ LOGGER.info("Force upgrading " + this.worldName);
|
|
+ LOGGER.info("Counting regionfiles for " + this.worldName);
|
|
+ final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> {
|
|
+ return WorldUpgrader.REGEX.matcher(name).matches();
|
|
+ });
|
|
+ if (regionFiles == null) {
|
|
+ LOGGER.info("Found no regionfiles to convert for world " + this.worldName);
|
|
+ return;
|
|
+ }
|
|
+ LOGGER.info("Found " + regionFiles.length + " regionfiles to convert");
|
|
+ LOGGER.info("Starting conversion now for world " + this.worldName);
|
|
+
|
|
+ final WorldInfo info = new WorldInfo(() -> worldPersistentData,
|
|
+ new ChunkStorage(regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey);
|
|
+
|
|
+ long expectedChunks = (long)regionFiles.length * (32L * 32L);
|
|
+
|
|
+ for (final File regionFile : regionFiles) {
|
|
+ final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile.toPath());
|
|
+ if (regionPos == null) {
|
|
+ expectedChunks -= (32L * 32L);
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5));
|
|
+ }
|
|
+ this.threadPool.shutdown();
|
|
+
|
|
+ final DecimalFormat format = new DecimalFormat("#0.00");
|
|
+
|
|
+ final long start = System.nanoTime();
|
|
+
|
|
+ while (!this.threadPool.isTerminated()) {
|
|
+ final long current = info.convertedChunks.get();
|
|
+
|
|
+ LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks);
|
|
+
|
|
+ try {
|
|
+ Thread.sleep(1000L);
|
|
+ } catch (final InterruptedException ignore) {}
|
|
+ }
|
|
+
|
|
+ final long end = System.nanoTime();
|
|
+
|
|
+ try {
|
|
+ info.loader.close();
|
|
+ } catch (final IOException ex) {
|
|
+ LOGGER.fatal("Failed to close chunk loader", ex);
|
|
+ }
|
|
+ LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)",
|
|
+ (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0));
|
|
+ }
|
|
+
|
|
+ private static final class WorldInfo {
|
|
+
|
|
+ public final Supplier<DimensionDataStorage> persistentDataSupplier;
|
|
+ public final ChunkStorage loader;
|
|
+ public final boolean removeCaches;
|
|
+ public final ResourceKey<LevelStem> worldKey;
|
|
+ public final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
|
|
+ public final AtomicLong convertedChunks = new AtomicLong();
|
|
+ public final AtomicLong modifiedChunks = new AtomicLong();
|
|
+
|
|
+ private WorldInfo(final Supplier<DimensionDataStorage> persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches,
|
|
+ final ResourceKey<LevelStem> worldKey, Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey) {
|
|
+ this.persistentDataSupplier = persistentDataSupplier;
|
|
+ this.loader = loader;
|
|
+ this.removeCaches = removeCaches;
|
|
+ this.worldKey = worldKey;
|
|
+ this.generatorKey = generatorKey;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ private static final class ConvertTask implements Runnable {
|
|
+
|
|
+ private final WorldInfo worldInfo;
|
|
+ private final int regionX;
|
|
+ private final int regionZ;
|
|
+
|
|
+ public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) {
|
|
+ this.worldInfo = worldInfo;
|
|
+ this.regionX = regionX;
|
|
+ this.regionZ = regionZ;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public void run() {
|
|
+ final int regionCX = this.regionX << 5;
|
|
+ final int regionCZ = this.regionZ << 5;
|
|
+
|
|
+ final Supplier<DimensionDataStorage> persistentDataSupplier = this.worldInfo.persistentDataSupplier;
|
|
+ final ChunkStorage loader = this.worldInfo.loader;
|
|
+ final boolean removeCaches = this.worldInfo.removeCaches;
|
|
+ final ResourceKey<LevelStem> worldKey = this.worldInfo.worldKey;
|
|
+
|
|
+ for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) {
|
|
+ for (int cx = regionCX; cx < (regionCX + 32); ++cx) {
|
|
+ final ChunkPos chunkPos = new ChunkPos(cx, cz);
|
|
+ try {
|
|
+ // no need to check the coordinate of the chunk, the regionfilecache does that for us
|
|
+
|
|
+ CompoundTag chunkNBT = (loader.read(chunkPos).join()).orElse(null);
|
|
+
|
|
+ if (chunkNBT == null) {
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ final int versionBefore = ChunkStorage.getVersion(chunkNBT);
|
|
+
|
|
+ chunkNBT = loader.upgradeChunkTag(worldKey, persistentDataSupplier, chunkNBT, this.worldInfo.generatorKey, chunkPos, null);
|
|
+
|
|
+ boolean modified = versionBefore < SharedConstants.getCurrentVersion().getDataVersion().getVersion();
|
|
+
|
|
+ if (removeCaches) {
|
|
+ final CompoundTag level = chunkNBT.getCompound("Level");
|
|
+ modified |= level.contains("Heightmaps");
|
|
+ level.remove("Heightmaps");
|
|
+ modified |= level.contains("isLightOn");
|
|
+ level.remove("isLightOn");
|
|
+ }
|
|
+
|
|
+ if (modified) {
|
|
+ this.worldInfo.modifiedChunks.getAndIncrement();
|
|
+ loader.write(chunkPos, chunkNBT);
|
|
+ }
|
|
+ } catch (final Exception ex) {
|
|
+ LOGGER.error("Error upgrading chunk {}", chunkPos, ex);
|
|
+ } finally {
|
|
+ this.worldInfo.convertedChunks.getAndIncrement();
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+}
|
|
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
|
|
index 329471af4f40e0a74612707cce96bb00819e6cf2..bc391d27399d8c22e78735ca39aa8ab45efb6413 100644
|
|
--- a/src/main/java/net/minecraft/server/Main.java
|
|
+++ b/src/main/java/net/minecraft/server/Main.java
|
|
@@ -388,6 +388,15 @@ public class Main {
|
|
return new WorldLoader.InitConfig(worldloader_d, Commands.CommandSelection.DEDICATED, serverPropertiesHandler.functionPermissionLevel);
|
|
}
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ public static void convertWorldButItWorks(net.minecraft.resources.ResourceKey<net.minecraft.world.level.dimension.LevelStem> dimensionType, net.minecraft.world.level.storage.LevelStorageSource.LevelStorageAccess worldSession,
|
|
+ DataFixer dataFixer, Optional<net.minecraft.resources.ResourceKey<com.mojang.serialization.Codec<? extends net.minecraft.world.level.chunk.ChunkGenerator>>> generatorKey, boolean removeCaches) {
|
|
+ int threads = Runtime.getRuntime().availableProcessors() * 3 / 8;
|
|
+ final io.papermc.paper.world.ThreadedWorldUpgrader worldUpgrader = new io.papermc.paper.world.ThreadedWorldUpgrader(dimensionType, worldSession.getLevelId(), worldSession.levelDirectory.path().toFile(), threads, dataFixer, generatorKey, removeCaches);
|
|
+ worldUpgrader.convert();
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
+
|
|
public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier continueCheck, Registry<LevelStem> dimensionOptionsRegistry) {
|
|
Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit
|
|
WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, dimensionOptionsRegistry, eraseCache);
|
|
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
index 54d6e197bd6357bf2d31d8d5d1cb3707d22ef03e..884c0e4f58f0e374c910bc0d8b5d3ab1b8ade226 100644
|
|
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
@@ -584,11 +584,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
worlddata = new PrimaryLevelData(worldsettings, worldoptions, worlddimensions_b.specialWorldProperty(), lifecycle);
|
|
}
|
|
worlddata.checkName(name); // CraftBukkit - Migration did not rewrite the level.dat; This forces 1.8 to take the last loaded world as respawn (in this case the end)
|
|
- if (this.options.has("forceUpgrade")) {
|
|
- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.options.has("eraseCache"), () -> {
|
|
- return true;
|
|
- }, dimensions);
|
|
- }
|
|
+ // Paper - fix and optimise world upgrading; move down
|
|
|
|
PrimaryLevelData iworlddataserver = worlddata;
|
|
boolean flag = worlddata.isDebugWorld();
|
|
@@ -603,6 +599,13 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
biomeProvider = gen.getDefaultBiomeProvider(worldInfo);
|
|
}
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ if (options.has("forceUpgrade")) {
|
|
+ net.minecraft.server.Main.convertWorldButItWorks(
|
|
+ dimensionKey, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), options.has("eraseCache")
|
|
+ );
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
ResourceKey<Level> worldKey = ResourceKey.create(Registries.DIMENSION, dimensionKey.location());
|
|
|
|
if (dimensionKey == LevelStem.OVERWORLD) {
|
|
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
|
|
index a2f8d2858c4a43ff642761fd86512a5f0666a0d2..16e0b4bcc903e6decbdf66ac4fb3d0dc261dbded 100644
|
|
--- a/src/main/java/net/minecraft/world/level/Level.java
|
|
+++ b/src/main/java/net/minecraft/world/level/Level.java
|
|
@@ -181,6 +181,15 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
|
|
public final Map<Explosion.CacheKey, Float> explosionDensityCache = new HashMap<>(); // Paper - Optimize explosions
|
|
public java.util.ArrayDeque<net.minecraft.world.level.block.RedstoneTorchBlock.Toggle> redstoneUpdateInfos; // Paper - Faster redstone torch rapid clock removal; Move from Map in BlockRedstoneTorch to here
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ // copied from below
|
|
+ public static ResourceKey<DimensionType> getDimensionKey(DimensionType manager) {
|
|
+ return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryAccess().registryOrThrow(net.minecraft.core.registries.Registries.DIMENSION_TYPE).getResourceKey(manager).orElseThrow(() -> {
|
|
+ return new IllegalStateException("Unregistered dimension type: " + manager);
|
|
+ });
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
+
|
|
public CraftWorld getWorld() {
|
|
return this.world;
|
|
}
|
|
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
|
|
index 0db8ee3b640e6d1268e9c1cccda85459bd447105..42d37bee3a459adcd46408596ccf93abbcff51fe 100644
|
|
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
|
|
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
|
|
@@ -60,6 +60,29 @@ public class RegionFileStorage implements AutoCloseable {
|
|
}
|
|
|
|
// Paper start
|
|
+ @Nullable
|
|
+ public static ChunkPos getRegionFileCoordinates(Path file) {
|
|
+ String fileName = file.getFileName().toString();
|
|
+ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) {
|
|
+ return null;
|
|
+ }
|
|
+
|
|
+ String[] split = fileName.split("\\.");
|
|
+
|
|
+ if (split.length != 4) {
|
|
+ return null;
|
|
+ }
|
|
+
|
|
+ try {
|
|
+ int x = Integer.parseInt(split[1]);
|
|
+ int z = Integer.parseInt(split[2]);
|
|
+
|
|
+ return new ChunkPos(x << 5, z << 5);
|
|
+ } catch (NumberFormatException ex) {
|
|
+ return null;
|
|
+ }
|
|
+ }
|
|
+
|
|
public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) {
|
|
return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()));
|
|
}
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
index c9137151f0d2978adb432c40da68689465d2325d..ab7bc27e870227e6746b77a7b5e109e2cf198b5f 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
@@ -1357,9 +1357,7 @@ public final class CraftServer implements Server {
|
|
worlddata.checkName(name);
|
|
worlddata.setModdedInfo(this.console.getServerModName(), this.console.getModdedStatus().shouldReportAsModified());
|
|
|
|
- if (this.console.options.has("forceUpgrade")) {
|
|
- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.console.options.has("eraseCache"), () -> true, iregistry);
|
|
- }
|
|
+ // Paper - fix and optimise world upgrading; move down
|
|
|
|
long j = BiomeManager.obfuscateSeed(worlddata.worldGenOptions().seed()); // Paper - use world seed
|
|
List<CustomSpawner> list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata));
|
|
@@ -1370,6 +1368,13 @@ public final class CraftServer implements Server {
|
|
biomeProvider = generator.getDefaultBiomeProvider(worldInfo);
|
|
}
|
|
|
|
+ // Paper start - fix and optimise world upgrading
|
|
+ if (this.console.options.has("forceUpgrade")) {
|
|
+ net.minecraft.server.Main.convertWorldButItWorks(
|
|
+ actualDimension, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), this.console.options.has("eraseCache")
|
|
+ );
|
|
+ }
|
|
+ // Paper end - fix and optimise world upgrading
|
|
ResourceKey<net.minecraft.world.level.Level> worldKey;
|
|
String levelName = this.getServer().getProperties().levelName;
|
|
if (name.equals(levelName + "_nether")) {
|