From e09fec3373abcf836635dc33a95e9b9975058eb9 Mon Sep 17 00:00:00 2001
From: Ciaran Gultnieks <ciaran@ciarang.com>
Date: Wed, 18 May 2011 16:48:27 +0100
Subject: [PATCH] New map directory structure that avoids map size being
limited by filesystem
---
src/filesys.cpp | 19 ++++++++
src/filesys.h | 3 ++
src/map.cpp | 124 +++++++++++++++++++++++++++++++++---------------
src/map.h | 12 ++---
4 files changed, 111 insertions(+), 47 deletions(-)
diff --git a/src/filesys.cpp b/src/filesys.cpp
index 287090e8a..8248a13d4 100644
--- a/src/filesys.cpp
+++ b/src/filesys.cpp
@@ -290,5 +290,24 @@ bool RecursiveDeleteContent(std::string path)
return true;
}
+bool CreateAllDirs(std::string path)
+{
+
+ size_t pos;
+ std::vector<std::string> tocreate;
+ std::string basepath = path;
+ while(!PathExists(basepath))
+ {
+ tocreate.push_back(basepath);
+ pos = basepath.rfind('/');
+ if(pos == std::string::npos)
+ return false;
+ basepath = basepath.substr(0,pos);
+ }
+ for(int i=tocreate.size()-1;i>=0;i--)
+ CreateDir(tocreate[i]);
+ return true;
+}
+
} // namespace fs
diff --git a/src/filesys.h b/src/filesys.h
index 4dd90b84e..b74b34f3d 100644
--- a/src/filesys.h
+++ b/src/filesys.h
@@ -38,6 +38,9 @@ std::vector<DirListNode> GetDirListing(std::string path);
// Returns true if already exists
bool CreateDir(std::string path);
+// Create all directories on the given path that don't already exist.
+bool CreateAllDirs(std::string path);
+
bool PathExists(std::string path);
// Only pass full paths to this one. True on success.
diff --git a/src/map.cpp b/src/map.cpp
index 63f01ddee..cd2ba9154 100644
--- a/src/map.cpp
+++ b/src/map.cpp
@@ -4875,9 +4875,9 @@ s16 ServerMap::findGroundLevel(v2s16 p2d)
return (s16)level;
}
-void ServerMap::createDir(std::string path)
+void ServerMap::createDirs(std::string path)
{
- if(fs::CreateDir(path) == false)
+ if(fs::CreateAllDirs(path) == false)
{
m_dout<<DTIME<<"ServerMap: Failed to create directory "
<<"\""<<path<<"\""<<std::endl;
@@ -4885,29 +4885,52 @@ void ServerMap::createDir(std::string path)
}
}
-std::string ServerMap::getSectorSubDir(v2s16 pos)
+std::string ServerMap::getSectorDir(v2s16 pos, int layout)
{
char cc[9];
- snprintf(cc, 9, "%.4x%.4x",
- (unsigned int)pos.X&0xffff,
- (unsigned int)pos.Y&0xffff);
+ switch(layout)
+ {
+ case 1:
+ snprintf(cc, 9, "%.4x%.4x",
+ (unsigned int)pos.X&0xffff,
+ (unsigned int)pos.Y&0xffff);
- return std::string(cc);
-}
+ return m_savedir + "/sectors/" + cc;
+ case 2:
+ snprintf(cc, 9, "%.3x/%.3x",
+ (unsigned int)pos.X&0xfff,
+ (unsigned int)pos.Y&0xfff);
-std::string ServerMap::getSectorDir(v2s16 pos)
-{
- return m_savedir + "/sectors/" + getSectorSubDir(pos);
+ return m_savedir + "/sectors2/" + cc;
+ default:
+ assert(false);
+ }
}
v2s16 ServerMap::getSectorPos(std::string dirname)
{
- if(dirname.size() != 8)
- throw InvalidFilenameException("Invalid sector directory name");
unsigned int x, y;
- int r = sscanf(dirname.c_str(), "%4x%4x", &x, &y);
- if(r != 2)
- throw InvalidFilenameException("Invalid sector directory name");
+ int r;
+ size_t spos = dirname.rfind('/') + 1;
+ assert(spos != std::string::npos);
+ if(dirname.size() - spos == 8)
+ {
+ // Old layout
+ r = sscanf(dirname.substr(spos).c_str(), "%4x%4x", &x, &y);
+ }
+ else if(dirname.size() - spos == 3)
+ {
+ // New layout
+ r = sscanf(dirname.substr(spos-4).c_str(), "%3x/%3x", &x, &y);
+ // Sign-extend the 12 bit values up to 16 bits...
+ if(x&0x800) x|=0xF000;
+ if(y&0x800) y|=0xF000;
+ }
+ else
+ {
+ assert(false);
+ }
+ assert(r == 2);
v2s16 pos((s16)x, (s16)y);
return pos;
}
@@ -5106,7 +5129,7 @@ void ServerMap::saveMapMeta()
<<"seed="<<m_seed<<", chunksize="<<m_chunksize
<<std::endl;
- createDir(m_savedir);
+ createDirs(m_savedir);
std::string fullpath = m_savedir + "/map_meta.txt";
std::ofstream os(fullpath.c_str(), std::ios_base::binary);
@@ -5179,7 +5202,7 @@ void ServerMap::saveChunkMeta()
dstream<<"INFO: ServerMap::saveChunkMeta(): Saving metadata of "
<<count<<" chunks"<<std::endl;
- createDir(m_savedir);
+ createDirs(m_savedir);
std::string fullpath = m_savedir + "/chunk_meta";
std::ofstream os(fullpath.c_str(), std::ios_base::binary);
@@ -5267,10 +5290,8 @@ void ServerMap::saveSectorMeta(ServerMapSector *sector)
u8 version = SER_FMT_VER_HIGHEST;
// Get destination
v2s16 pos = sector->getPos();
- createDir(m_savedir);
- createDir(m_savedir+"/sectors");
std::string dir = getSectorDir(pos);
- createDir(dir);
+ createDirs(dir);
std::string fullpath = dir + "/meta";
std::ofstream o(fullpath.c_str(), std::ios_base::binary);
@@ -5282,22 +5303,21 @@ void ServerMap::saveSectorMeta(ServerMapSector *sector)
sector->differs_from_disk = false;
}
-MapSector* ServerMap::loadSectorMeta(std::string dirname)
+MapSector* ServerMap::loadSectorMeta(std::string sectordir, bool save_after_load)
{
DSTACK(__FUNCTION_NAME);
// Get destination
- v2s16 p2d = getSectorPos(dirname);
- std::string dir = m_savedir + "/sectors/" + dirname;
+ v2s16 p2d = getSectorPos(sectordir);
ServerMapSector *sector = NULL;
-
- std::string fullpath = dir + "/meta";
+
+ std::string fullpath = sectordir + "/meta";
std::ifstream is(fullpath.c_str(), std::ios_base::binary);
if(is.good() == false)
{
// If the directory exists anyway, it probably is in some old
// format. Just go ahead and create the sector.
- if(fs::PathExists(dir))
+ if(fs::PathExists(sectordir))
{
dstream<<"ServerMap::loadSectorMeta(): Sector metafile "
<<fullpath<<" doesn't exist but directory does."
@@ -5307,12 +5327,16 @@ MapSector* ServerMap::loadSectorMeta(std::string dirname)
m_sectors.insert(p2d, sector);
}
else
+ {
throw FileNotGoodException("Cannot open sector metafile");
+ }
}
else
{
sector = ServerMapSector::deSerialize
(is, this, p2d, m_sectors);
+ if(save_after_load)
+ saveSectorMeta(sector);
}
sector->differs_from_disk = false;
@@ -5323,14 +5347,31 @@ MapSector* ServerMap::loadSectorMeta(std::string dirname)
bool ServerMap::loadSectorFull(v2s16 p2d)
{
DSTACK(__FUNCTION_NAME);
- std::string sectorsubdir = getSectorSubDir(p2d);
MapSector *sector = NULL;
+ // The directory layout we're going to load from.
+ // 1 - original sectors/xxxxzzzz/
+ // 2 - new sectors2/xxx/zzz/
+ // If we load from anything but the latest structure, we will
+ // immediately save to the new one, and remove the old.
+ int loadlayout = 1;
+ std::string sectordir1 = getSectorDir(p2d, 1);
+ std::string sectordir;
+ if(fs::PathExists(sectordir1))
+ {
+ sectordir = sectordir1;
+ }
+ else
+ {
+ loadlayout = 2;
+ sectordir = getSectorDir(p2d, 2);
+ }
+
//JMutexAutoLock lock(m_sector_mutex); // Bulk comment-out
try{
- sector = loadSectorMeta(sectorsubdir);
+ sector = loadSectorMeta(sectordir, loadlayout != 2);
}
catch(InvalidFilenameException &e)
{
@@ -5349,7 +5390,7 @@ bool ServerMap::loadSectorFull(v2s16 p2d)
Load blocks
*/
std::vector<fs::DirListNode> list2 = fs::GetDirListing
- (m_savedir+"/sectors/"+sectorsubdir);
+ (sectordir);
std::vector<fs::DirListNode>::iterator i2;
for(i2=list2.begin(); i2!=list2.end(); i2++)
{
@@ -5357,16 +5398,25 @@ bool ServerMap::loadSectorFull(v2s16 p2d)
if(i2->dir)
continue;
try{
- loadBlock(sectorsubdir, i2->name, sector);
+ loadBlock(sectordir, i2->name, sector, loadlayout != 2);
}
catch(InvalidFilenameException &e)
{
// This catches unknown crap in directory
}
}
+
+ if(loadlayout != 2)
+ {
+ dstream<<"Sector converted to new layout - deleting "<<
+ sectordir1<<std::endl;
+ fs::RecursiveDelete(sectordir1);
+ }
+
return true;
}
+
void ServerMap::saveBlock(MapBlock *block)
{
DSTACK(__FUNCTION_NAME);
@@ -5386,12 +5436,9 @@ void ServerMap::saveBlock(MapBlock *block)
// Get destination
v3s16 p3d = block->getPos();
v2s16 p2d(p3d.X, p3d.Z);
- createDir(m_savedir);
- createDir(m_savedir+"/sectors");
std::string dir = getSectorDir(p2d);
- createDir(dir);
+ createDirs(dir);
- // Block file is map/sectors/xxxxxxxx/xxxx
char cc[5];
snprintf(cc, 5, "%.4x", (unsigned int)p3d.Y&0xffff);
std::string fullpath = dir + "/" + cc;
@@ -5427,12 +5474,11 @@ void ServerMap::saveBlock(MapBlock *block)
block->resetChangedFlag();
}
-void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSector *sector)
+void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load)
{
DSTACK(__FUNCTION_NAME);
- // Block file is map/sectors/xxxxxxxx/xxxx
- std::string fullpath = m_savedir+"/sectors/"+sectordir+"/"+blockfile;
+ std::string fullpath = sectordir+"/"+blockfile;
try{
std::ifstream is(fullpath.c_str(), std::ios_base::binary);
@@ -5496,7 +5542,7 @@ void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSecto
*/
// Save old format blocks in new format
- if(version < SER_FMT_VER_HIGHEST)
+ if(version < SER_FMT_VER_HIGHEST || save_after_load)
{
saveBlock(block);
}
diff --git a/src/map.h b/src/map.h
index fa52dc26b..09154547c 100644
--- a/src/map.h
+++ b/src/map.h
@@ -545,13 +545,9 @@ class ServerMap : public Map
Misc. helper functions for fiddling with directory and file
names when saving
*/
- void createDir(std::string path);
- void createSaveDir();
- // returns something like "xxxxxxxx"
- std::string getSectorSubDir(v2s16 pos);
+ void createDirs(std::string path);
// returns something like "map/sectors/xxxxxxxx"
- std::string getSectorDir(v2s16 pos);
- std::string createSectorDir(v2s16 pos);
+ std::string getSectorDir(v2s16 pos, int layout = 2);
// dirname: final directory name
v2s16 getSectorPos(std::string dirname);
v3s16 getBlockPos(std::string sectordir, std::string blockfile);
@@ -572,7 +568,7 @@ class ServerMap : public Map
// (no MapBlocks)
// DEPRECATED? Sectors have no metadata anymore.
void saveSectorMeta(ServerMapSector *sector);
- MapSector* loadSectorMeta(std::string dirname);
+ MapSector* loadSectorMeta(std::string dirname, bool save_after_load);
// Full load of a sector including all blocks.
// returns true on success, false on failure.
@@ -583,7 +579,7 @@ class ServerMap : public Map
void saveBlock(MapBlock *block);
// This will generate a sector with getSector if not found.
- void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector);
+ void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load=false);
// For debug printing
virtual void PrintInfo(std::ostream &out);
--
GitLab