Implement AreaStore serialization

doc/lua_api.txt

@@ -2737,6 +2737,10 @@ If you chose the parameter-less constructor, a fast implementation will be autom
         block_radius = number, -- the radius (in nodes) of the areas the cache generates prefiltered lists for, minimum 16, default 64
         limit = number, -- the cache's size, minimum 20, default 1000
       }
+* `to_string()`: Experimental. Returns area store serialized as a (binary) string.
+* `to_file(filename)`: Experimental. Like `to_string()`, but writes the data to a file.
+* `from_string(str)`: Experimental. Deserializes string and loads it into the AreaStore.  Returns success and, optionally, an error message.
+* `from_file(filename)`: Experimental. Like `from_string()`, but reads the data from a file.
 
 ### `ItemStack`
 An `ItemStack` is a stack of items.

src/script/lua_api/l_areastore.cpp

@@ -70,6 +70,22 @@ static inline void push_areas(lua_State *L, const std::vector<Area *> &areas,
 	}
 }
 
+// Deserializes value and handles errors
+static int deserialization_helper(lua_State *L, AreaStore *as,
+		std::istream &is)
+{
+	try {	
+		as->deserialize(is);
+	} catch (const SerializationError &e) {
+		lua_pushboolean(L, false);
+		lua_pushstring(L, e.what());
+		return 2;
+	}
+
+	lua_pushboolean(L, true);
+	return 1;
+}
+
 // garbage collector
 int LuaAreaStore::gc_object(lua_State *L)
 {
@@ -217,17 +233,15 @@ int LuaAreaStore::l_set_cache_params(lua_State *L)
 	return 0;
 }
 
-#if 0
 // to_string()
 int LuaAreaStore::l_to_string(lua_State *L)
 {
 	NO_MAP_LOCK_REQUIRED;
 
 	LuaAreaStore *o = checkobject(L, 1);
-	AreaStore *ast = o->as;
 
 	std::ostringstream os(std::ios_base::binary);
-	ast->serialize(os);
+	o->as->serialize(os);
 	std::string str = os.str();
 
 	lua_pushlstring(L, str.c_str(), str.length());
@@ -258,16 +272,12 @@ int LuaAreaStore::l_from_string(lua_State *L)
 	NO_MAP_LOCK_REQUIRED;
 
 	LuaAreaStore *o = checkobject(L, 1);
-	AreaStore *ast = o->as;
 
 	size_t len;
 	const char *str = luaL_checklstring(L, 2, &len);
 
 	std::istringstream is(std::string(str, len), std::ios::binary);
-	bool success = ast->deserialize(is);
-
-	lua_pushboolean(L, success);
-	return 1;
+	return deserialization_helper(L, o->as, is);
 }
 
 // from_file(filename)
@@ -276,18 +286,13 @@ int LuaAreaStore::l_from_file(lua_State *L)
 	NO_MAP_LOCK_REQUIRED;
 
 	LuaAreaStore *o = checkobject(L, 1);
-	AreaStore *ast = o->as;
 
 	const char *filename = luaL_checkstring(L, 2);
 	CHECK_SECURE_PATH_OPTIONAL(L, filename);
 
 	std::ifstream is(filename, std::ios::binary);
-	bool success = ast->deserialize(is);
-
-	lua_pushboolean(L, success);
-	return 1;
+	return deserialization_helper(L, o->as, is);
 }
-#endif
 
 LuaAreaStore::LuaAreaStore()
 {
@@ -377,9 +382,9 @@ const luaL_reg LuaAreaStore::methods[] = {
 	luamethod(LuaAreaStore, reserve),
 	luamethod(LuaAreaStore, remove_area),
 	luamethod(LuaAreaStore, set_cache_params),
-	/* luamethod(LuaAreaStore, to_string),
+	luamethod(LuaAreaStore, to_string),
 	luamethod(LuaAreaStore, to_file),
 	luamethod(LuaAreaStore, from_string),
-	luamethod(LuaAreaStore, from_file),*/
+	luamethod(LuaAreaStore, from_file),
 	{0,0}
 };

src/script/lua_api/l_areastore.h

@@ -43,11 +43,11 @@ class LuaAreaStore : public ModApiBase {
 
 	static int l_set_cache_params(lua_State *L);
 
-	/* static int l_to_string(lua_State *L);
+	static int l_to_string(lua_State *L);
 	static int l_to_file(lua_State *L);
 
 	static int l_from_string(lua_State *L);
-	static int l_from_file(lua_State *L); */
+	static int l_from_file(lua_State *L);
 
 public:
 	AreaStore *as;

src/unittest/test_areastore.cpp

@@ -31,6 +31,7 @@ class TestAreaStore : public TestBase {
 	void genericStoreTest(AreaStore *store);
 	void testVectorStore();
 	void testSpatialStore();
+	void testSerialization();
 };
 
 static TestAreaStore g_test_instance;
@@ -41,6 +42,7 @@ void TestAreaStore::runTests(IGameDef *gamedef)
 #if USE_SPATIAL
 	TEST(testSpatialStore);
 #endif
+	TEST(testSerialization);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -121,3 +123,40 @@ void TestAreaStore::genericStoreTest(AreaStore *store)
 	store->removeArea(d.id);
 }
 
+void TestAreaStore::testSerialization()
+{
+	VectorAreaStore store;
+
+	Area a(v3s16(-1, 0, 1), v3s16(0, 1, 2));
+	a.data = "Area A";
+	store.insertArea(&a);
+
+	Area b(v3s16(123, 456, 789), v3s16(32000, 100, 10));
+	b.data = "Area B";
+	store.insertArea(&b);
+
+	std::ostringstream os;
+	store.serialize(os);
+	std::string str = os.str();
+
+	std::string str_wanted("\x00"  // Version
+			"\x00\x02"  // Count
+			"\xFF\xFF\x00\x00\x00\x01"  // Area A min edge
+			"\x00\x00\x00\x01\x00\x02"  // Area A max edge
+			"\x00\x06"  // Area A data length
+			"Area A"  // Area A data
+			"\x00\x7B\x00\x64\x00\x0A"  // Area B min edge (last two swapped with max edge for sorting)
+			"\x7D\x00\x01\xC8\x03\x15"  // Area B max edge (^)
+			"\x00\x06"  // Area B data length
+			"Area B",  // Area B data
+			1 + 2 +
+			6 + 6 + 2 + 6 +
+			6 + 6 + 2 + 6);
+	UASSERTEQ(std::string, str, str_wanted);
+
+	std::istringstream is(str);
+	store.deserialize(is);
+
+	UASSERTEQ(size_t, store.size(), 4);  // deserialize() doesn't clear the store
+}
+

src/util/areastore.cpp

@@ -62,57 +62,42 @@ const Area *AreaStore::getArea(u32 id) const
 	return &it->second;
 }
 
-#if 0
-Currently, serialisation is commented out. This is because of multiple reasons:
-1. Why do we store the areastore into a file, why not into the database?
-2. We don't use libspatial's serialisation, but we should, or perhaps not, because
-	it would remove the ability to switch. Perhaps write migration routines?
-3. Various things need fixing, e.g. the size is serialized as
-	c++ implementation defined size_t
-bool AreaStore::deserialize(std::istream &is)
+void AreaStore::serialize(std::ostream &os) const
 {
-	u8 ver = readU8(is);
-	if (ver != 1)
-		return false;
-	u16 count_areas = readU16(is);
-	for (u16 i = 0; i < count_areas; i++) {
-		// deserialize an area
-		Area a;
-		a.id = readU32(is);
-		a.minedge = readV3S16(is);
-		a.maxedge = readV3S16(is);
-		a.datalen = readU16(is);
-		a.data = new char[a.datalen];
-		is.read((char *) a.data, a.datalen);
-		insertArea(a);
+	writeU8(os, 0); // Serialisation version
+
+	// TODO: Compression?
+	writeU16(os, areas_map.size());
+	for (AreaMap::const_iterator it = areas_map.begin();
+			it != areas_map.end(); ++it) {
+		const Area &a = it->second;
+		writeV3S16(os, a.minedge);
+		writeV3S16(os, a.maxedge);
+		writeU16(os, a.data.size());
+		os.write(a.data.data(), a.data.size());
 	}
-	return true;
 }
 
-
-static bool serialize_area(void *ostr, Area *a)
+void AreaStore::deserialize(std::istream &is)
 {
-	std::ostream &os = *((std::ostream *) ostr);
-	writeU32(os, a->id);
-	writeV3S16(os, a->minedge);
-	writeV3S16(os, a->maxedge);
-	writeU16(os, a->datalen);
-	os.write(a->data, a->datalen);
-
-	return false;
-}
-
+	u8 ver = readU8(is);
+	if (ver != 0)
+		throw SerializationError("Unknown AreaStore "
+				"serialization version!");
 
-void AreaStore::serialize(std::ostream &os) const
-{
-	// write initial data
-	writeU8(os, 1); // serialisation version
-	writeU16(os, areas_map.size()); //DANGER: not platform independent
-	forEach(&serialize_area, &os);
+	u16 num_areas = readU16(is);
+	for (u32 i = 0; i < num_areas; ++i) {
+		Area a;
+		a.minedge = readV3S16(is);
+		a.maxedge = readV3S16(is);
+		u16 data_len = readU16(is);
+		char *data = new char[data_len];
+		is.read(data, data_len);
+		a.data = std::string(data, data_len);
+		insertArea(&a);
+	}
 }
 
-#endif
-
 void AreaStore::invalidateCache()
 {
 	if (m_cache_enabled) {
@@ -226,18 +211,6 @@ void VectorAreaStore::getAreasInArea(std::vector<Area *> *result,
 	}
 }
 
-#if 0
-bool SimpleAreaStore::forEach(ForEachCallback callback, void *arg) const
-{
-	for (size_t i = 0; i < m_areas.size(); ++i) {
-		if (callback(m_areas[i], arg)) {
-			return true;
-		}
-	}
-	return false;
-}
-#endif
-
 #if USE_SPATIAL
 
 static inline SpatialIndex::Region get_spatial_region(const v3s16 minedge,
@@ -301,14 +274,6 @@ void SpatialAreaStore::getAreasInArea(std::vector<Area *> *result,
 	}
 }
 
-#if 0
-bool SpatialAreaStore::forEach(ForEachCallback callback, void *arg) const
-{
-	// TODO ?? (this is only needed for serialisation, but libspatial has its own serialisation)
-	return false;
-}
-#endif
-
 SpatialAreaStore::~SpatialAreaStore()
 {
 	delete m_tree;

src/util/areastore.h

@@ -92,16 +92,14 @@ class AreaStore {
 	/// or NULL if it doesn't exist.
 	const Area *getArea(u32 id) const;
 
-#if 0
-	typedef bool (*ForEachCallback)(const Area *a, void *arg);
-	/// Calls a passed function for every stored area, until the
-	/// callback returns true.  If that happens, it returns true,
-	/// if the search is exhausted, it returns false.
-	virtual bool forEach(ForEachCallback, void *arg=NULL) const = 0;
-
+	/// Serializes the store's areas to a binary ostream.
 	void serialize(std::ostream &is) const;
-	bool deserialize(std::istream &is);
-#endif
+
+	/// Deserializes the Areas from a binary istream.
+	/// This does not currently clear the AreaStore before adding the
+	/// areas, making it possible to deserialize multiple serialized
+	/// AreaStores.
+	void deserialize(std::istream &is);
 
 protected:
 	/// Invalidates the getAreasForPos cache.
@@ -141,7 +139,6 @@ class VectorAreaStore : public AreaStore {
 	virtual bool removeArea(u32 id);
 	virtual void getAreasInArea(std::vector<Area *> *result,
 		v3s16 minedge, v3s16 maxedge, bool accept_overlap);
-	//virtual bool forEach(ForEachCallback, void *arg) const;
 
 protected:
 	virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);
@@ -162,7 +159,6 @@ class SpatialAreaStore : public AreaStore {
 	virtual bool removeArea(u32 id);
 	virtual void getAreasInArea(std::vector<Area *> *result,
 		v3s16 minedge, v3s16 maxedge, bool accept_overlap);
-	//virtual bool forEach(ForEachCallback, void *arg) const;
 
 protected:
 	virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);