diff --git a/cmake/Modules/FindCURL.cmake b/cmake/Modules/FindCURL.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..ec0503fd901579704985d2a8e0781e853a39ccc9
--- /dev/null
+++ b/cmake/Modules/FindCURL.cmake
@@ -0,0 +1,17 @@
+# - Find curl
+# Find the native CURL headers and libraries.
+#
+# CURL_INCLUDE_DIR - where to find curl/curl.h, etc.
+# CURL_LIBRARY - List of libraries when using curl.
+# CURL_FOUND - True if curl found.
+
+# Look for the header file.
+FIND_PATH(CURL_INCLUDE_DIR NAMES curl/curl.h)
+
+# Look for the library.
+FIND_LIBRARY(CURL_LIBRARY NAMES curl)
+
+# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if
+# all listed variables are TRUE
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARY CURL_INCLUDE_DIR)
diff --git a/minetest.conf.example b/minetest.conf.example
index a0aa691258a2d6741eee11d4bc0d67936047921d..e80991936fa6ab037cbf78a1ecb34e2e58f8056c 100644
--- a/minetest.conf.example
+++ b/minetest.conf.example
@@ -141,6 +141,10 @@
# 2: enable high level shaders
#enable_shaders = 2
+# will only work for servers which use remote_media setting
+# and only for clients compiled with cURL
+#media_fetch_threads = 8
+
#
# Server stuff
#
@@ -217,3 +221,4 @@
#congestion_control_aim_rtt = 0.2
#congestion_control_max_rate = 400
#congestion_control_min_rate = 10
+#remote_media =
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 3830ef3b66636a862690bbe0a7eb836b83ee8b81..38410f7d2d3b50fc94b1663023a80c996673c0f2 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -6,6 +6,19 @@ mark_as_advanced(EXECUTABLE_OUTPUT_PATH LIBRARY_OUTPUT_PATH)
mark_as_advanced(JTHREAD_INCLUDE_DIR JTHREAD_LIBRARY)
mark_as_advanced(SQLITE3_INCLUDE_DIR SQLITE3_LIBRARY)
+option(ENABLE_CURL "Enable cURL support for fetching media" 1)
+
+if (NOT ENABLE_CURL)
+ mark_as_advanced(CLEAR CURL_LIBRARY CURL_INCLUDE_DIR)
+endif(NOT ENABLE_CURL)
+
+find_package(CURL)
+set(USE_CURL 0)
+if (CURL_FOUND AND ENABLE_CURL)
+ message(STATUS "cURL support enabled")
+ set(USE_CURL 1)
+endif(CURL_FOUND AND ENABLE_CURL)
+
# user-visible option to enable/disable gettext usage
OPTION(ENABLE_GETTEXT "Use GetText for internationalization" 0)
@@ -307,6 +320,16 @@ if(BUILD_CLIENT)
${PLATFORM_LIBS}
${CLIENT_PLATFORM_LIBS}
)
+
+ if(USE_CURL)
+ target_link_libraries(
+ ${PROJECT_NAME}
+ ${CURL_LIBRARY}
+ )
+ include_directories(
+ ${CURL_INCLUDE_DIR}
+ )
+ endif(USE_CURL)
endif(BUILD_CLIENT)
if(BUILD_SERVER)
diff --git a/src/client.cpp b/src/client.cpp
index 3463e926269683605fcf24f42fdc754cde274d45..46b53c6d725e4a4ddda2c534d389cba0f15bae26 100644
--- a/src/client.cpp
+++ b/src/client.cpp
@@ -44,21 +44,17 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "hex.h"
#include "IMeshCache.h"
#include "util/serialize.h"
+#include "config.h"
+
+#if USE_CURL
+#include <curl/curl.h>
+#endif
static std::string getMediaCacheDir()
{
return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
}
-struct MediaRequest
-{
- std::string name;
-
- MediaRequest(const std::string &name_=""):
- name(name_)
- {}
-};
-
/*
QueuedMeshUpdate
*/
@@ -223,6 +219,45 @@ void * MeshUpdateThread::Thread()
return NULL;
}
+void * MediaFetchThread::Thread()
+{
+ ThreadStarted();
+
+ log_register_thread("MediaFetchThread");
+
+ DSTACK(__FUNCTION_NAME);
+
+ BEGIN_DEBUG_EXCEPTION_HANDLER
+
+ #if USE_CURL
+ CURL *curl;
+ CURLcode res;
+ for (core::list<MediaRequest>::Iterator i = m_file_requests.begin();
+ i != m_file_requests.end(); i++) {
+ curl = curl_easy_init();
+ assert(curl);
+ curl_easy_setopt(curl, CURLOPT_URL, (m_remote_url + i->name).c_str());
+ curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
+ std::ostringstream stream;
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream);
+ res = curl_easy_perform(curl);
+ if (res == CURLE_OK) {
+ std::string data = stream.str();
+ m_file_data.push_back(make_pair(i->name, data));
+ } else {
+ m_failed.push_back(*i);
+ infostream << "cURL request failed for " << i->name << std::endl;
+ }
+ curl_easy_cleanup(curl);
+ }
+ #endif
+
+ END_DEBUG_EXCEPTION_HANDLER(errorstream)
+
+ return NULL;
+}
+
Client::Client(
IrrlichtDevice *device,
const char *playername,
@@ -263,8 +298,9 @@ Client::Client(
m_password(password),
m_access_denied(false),
m_media_cache(getMediaCacheDir()),
- m_media_receive_progress(0),
- m_media_received(false),
+ m_media_receive_started(false),
+ m_media_count(0),
+ m_media_received_count(0),
m_itemdef_received(false),
m_nodedef_received(false),
m_time_of_day_set(false),
@@ -730,6 +766,63 @@ void Client::step(float dtime)
g_profiler->graphAdd("num_processed_meshes", num_processed_meshes);
}
+ /*
+ Load fetched media
+ */
+ if (m_media_receive_started) {
+ bool all_stopped = true;
+ for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
+ thread != m_media_fetch_threads.end(); thread++) {
+ all_stopped &= !thread->IsRunning();
+ while (thread->m_file_data.size() > 0) {
+ std::pair <std::string, std::string> out = thread->m_file_data.pop_front();
+ ++m_media_received_count;
+
+ bool success = loadMedia(out.second, out.first);
+ if(success){
+ verbosestream<<"Client: Loaded received media: "
+ <<"\""<<out.first<<"\". Caching."<<std::endl;
+ } else{
+ infostream<<"Client: Failed to load received media: "
+ <<"\""<<out.first<<"\". Not caching."<<std::endl;
+ continue;
+ }
+
+ bool did = fs::CreateAllDirs(getMediaCacheDir());
+ if(!did){
+ errorstream<<"Could not create media cache directory"
+ <<std::endl;
+ }
+
+ {
+ core::map<std::string, std::string>::Node *n;
+ n = m_media_name_sha1_map.find(out.first);
+ if(n == NULL)
+ errorstream<<"The server sent a file that has not "
+ <<"been announced."<<std::endl;
+ else
+ m_media_cache.update_sha1(out.second);
+ }
+ }
+ }
+ if (all_stopped) {
+ core::list<MediaRequest> fetch_failed;
+ for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
+ thread != m_media_fetch_threads.end(); thread++) {
+ for (core::list<MediaRequest>::Iterator request = thread->m_failed.begin();
+ request != thread->m_failed.end(); request++)
+ fetch_failed.push_back(*request);
+ thread->m_failed.clear();
+ }
+ if (fetch_failed.size() > 0) {
+ infostream << "Failed to remote-fetch " << fetch_failed.size() << " files. "
+ << "Requesting them the usual way." << std::endl;
+ request_media(fetch_failed);
+ }
+ m_media_fetch_threads.clear();
+ }
+ }
+
/*
If the server didn't update the inventory in a while, revert
the local inventory (so the player notices the lag problem
@@ -907,6 +1000,34 @@ void Client::deletingPeer(con::Peer *peer, bool timeout)
<<"(timeout="<<timeout<<")"<<std::endl;
}
+/*
+ u16 command
+ u16 number of files requested
+ for each file {
+ u16 length of name
+ string name
+ }
+*/
+void Client::request_media(const core::list<MediaRequest> &file_requests)
+{
+ std::ostringstream os(std::ios_base::binary);
+ writeU16(os, TOSERVER_REQUEST_MEDIA);
+ writeU16(os, file_requests.size());
+
+ for(core::list<MediaRequest>::ConstIterator i = file_requests.begin();
+ i != file_requests.end(); i++) {
+ os<<serializeString(i->name);
+ }
+
+ // Make data buffer
+ std::string s = os.str();
+ SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+ // Send as reliable
+ Send(0, data, true);
+ infostream<<"Client: Sending media request list to server ("
+ <<file_requests.size()<<" files)"<<std::endl;
+}
+
void Client::ReceiveAll()
{
DSTACK(__FUNCTION_NAME);
@@ -1514,37 +1635,56 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
file_requests.push_back(MediaRequest(name));
}
- ClientEvent event;
- event.type = CE_TEXTURES_UPDATED;
- m_client_event_queue.push_back(event);
+ std::string remote_media = "";
+ try {
+ remote_media = deSerializeString(is);
+ }
+ catch(SerializationError) {
+ // not supported by server or turned off
+ }
- /*
- u16 command
- u16 number of files requested
- for each file {
- u16 length of name
- string name
+ m_media_count = file_requests.size();
+ m_media_receive_started = true;
+
+ if (remote_media == "" || !USE_CURL) {
+ request_media(file_requests);
+ } else {
+ #if USE_CURL
+ for (size_t i = 0; i < g_settings->getU16("media_fetch_threads"); ++i) {
+ m_media_fetch_threads.push_back(MediaFetchThread(this));
}
- */
- std::ostringstream os(std::ios_base::binary);
- writeU16(os, TOSERVER_REQUEST_MEDIA);
- writeU16(os, file_requests.size());
- for(core::list<MediaRequest>::Iterator i = file_requests.begin();
- i != file_requests.end(); i++) {
- os<<serializeString(i->name);
- }
+ core::list<MediaFetchThread>::Iterator cur = m_media_fetch_threads.begin();
+ for(core::list<MediaRequest>::Iterator i = file_requests.begin();
+ i != file_requests.end(); i++) {
+ cur->m_file_requests.push_back(*i);
+ cur++;
+ if (cur == m_media_fetch_threads.end())
+ cur = m_media_fetch_threads.begin();
+ }
+ for (core::list<MediaFetchThread>::Iterator i = m_media_fetch_threads.begin();
+ i != m_media_fetch_threads.end(); i++) {
+ i->m_remote_url = remote_media;
+ i->Start();
+ }
+ #endif
- // Make data buffer
- std::string s = os.str();
- SharedBuffer<u8> data((u8*)s.c_str(), s.size());
- // Send as reliable
- Send(0, data, true);
- infostream<<"Client: Sending media request list to server ("
- <<file_requests.size()<<" files)"<<std::endl;
+ // notify server we received everything
+ std::ostringstream os(std::ios_base::binary);
+ writeU16(os, TOSERVER_RECEIVED_MEDIA);
+ std::string s = os.str();
+ SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+ // Send as reliable
+ Send(0, data, true);
+ }
+ ClientEvent event;
+ event.type = CE_TEXTURES_UPDATED;
+ m_client_event_queue.push_back(event);
}
else if(command == TOCLIENT_MEDIA)
{
+ if (m_media_count == 0)
+ return;
std::string datastring((char*)&data[2], datasize-2);
std::istringstream is(datastring, std::ios_base::binary);
@@ -1566,17 +1706,12 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
*/
int num_bunches = readU16(is);
int bunch_i = readU16(is);
- if(num_bunches >= 2)
- m_media_receive_progress = (float)bunch_i / (float)(num_bunches - 1);
- else
- m_media_receive_progress = 1.0;
- if(bunch_i == num_bunches - 1)
- m_media_received = true;
int num_files = readU32(is);
infostream<<"Client: Received files: bunch "<<bunch_i<<"/"
<<num_bunches<<" files="<<num_files
<<" size="<<datasize<<std::endl;
for(int i=0; i<num_files; i++){
+ m_media_received_count++;
std::string name = deSerializeString(is);
std::string data = deSerializeLongString(is);
@@ -2458,7 +2593,7 @@ void Client::afterContentReceived()
infostream<<"Client::afterContentReceived() started"<<std::endl;
assert(m_itemdef_received);
assert(m_nodedef_received);
- assert(m_media_received);
+ assert(texturesReceived());
// remove the information about which checksum each texture
// ought to have
diff --git a/src/client.h b/src/client.h
index f85e8ac7be643a5af87b0f7d049780e347675022..c6858f5497c42847951ec2f4666eb4ac05858134 100644
--- a/src/client.h
+++ b/src/client.h
@@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "filesys.h"
#include "filecache.h"
#include "localplayer.h"
+#include "server.h"
#include "util/pointedthing.h"
struct MeshMakeData;
@@ -129,6 +130,24 @@ class MeshUpdateThread : public SimpleThread
IGameDef *m_gamedef;
};
+class MediaFetchThread : public SimpleThread
+{
+public:
+
+ MediaFetchThread(IGameDef *gamedef):
+ m_gamedef(gamedef)
+ {
+ }
+
+ void * Thread();
+
+ core::list<MediaRequest> m_file_requests;
+ MutexedQueue<std::pair<std::string, std::string> > m_file_data;
+ core::list<MediaRequest> m_failed;
+ std::string m_remote_url;
+ IGameDef *m_gamedef;
+};
+
enum ClientEventType
{
CE_NONE,
@@ -289,10 +308,13 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
{ return m_access_denied_reason; }
float mediaReceiveProgress()
- { return m_media_receive_progress; }
+ {
+ if (!m_media_receive_started) return 0;
+ return 1.0 * m_media_received_count / m_media_count;
+ }
bool texturesReceived()
- { return m_media_received; }
+ { return m_media_receive_started && m_media_received_count == m_media_count; }
bool itemdefReceived()
{ return m_itemdef_received; }
bool nodedefReceived()
@@ -318,7 +340,9 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
// Insert a media file appropriately into the appropriate manager
bool loadMedia(const std::string &data, const std::string &filename);
-
+
+ void request_media(const core::list<MediaRequest> &file_requests);
+
// Virtual methods from con::PeerHandler
void peerAdded(con::Peer *peer);
void deletingPeer(con::Peer *peer, bool timeout);
@@ -347,6 +371,7 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
MtEventManager *m_event;
MeshUpdateThread m_mesh_update_thread;
+ core::list<MediaFetchThread> m_media_fetch_threads;
ClientEnvironment m_env;
con::Connection m_con;
IrrlichtDevice *m_device;
@@ -375,8 +400,9 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
FileCache m_media_cache;
// Mapping from media file name to SHA1 checksum
core::map<std::string, std::string> m_media_name_sha1_map;
- float m_media_receive_progress;
- bool m_media_received;
+ bool m_media_receive_started;
+ u32 m_media_count;
+ u32 m_media_received_count;
bool m_itemdef_received;
bool m_nodedef_received;
friend class FarMesh;
diff --git a/src/clientserver.h b/src/clientserver.h
index 6f9396c02ceaf607689b0eccaf1235eb513e777a..db551a90c410c37164a8de69e80c4a352047c3a2 100644
--- a/src/clientserver.h
+++ b/src/clientserver.h
@@ -267,6 +267,8 @@ enum ToClientCommand
u32 length of data
data
}
+ u16 length of remote media server url (if applicable)
+ string url
*/
TOCLIENT_TOOLDEF = 0x39,
@@ -571,6 +573,10 @@ enum ToServerCommand
}
*/
+ TOSERVER_RECEIVED_MEDIA = 0x41,
+ /*
+ u16 command
+ */
};
#endif
diff --git a/src/cmake_config.h.in b/src/cmake_config.h.in
index c2bdc967040a0c3e72c427dfafd1e26a76bb1a6c..4853d854f73783b48c9d167ae487f84e5399fbd1 100644
--- a/src/cmake_config.h.in
+++ b/src/cmake_config.h.in
@@ -7,6 +7,7 @@
#define CMAKE_VERSION_STRING "@VERSION_STRING@"
#define CMAKE_RUN_IN_PLACE @RUN_IN_PLACE@
#define CMAKE_USE_GETTEXT @USE_GETTEXT@
+#define CMAKE_USE_CURL @USE_CURL@
#define CMAKE_USE_SOUND @USE_SOUND@
#define CMAKE_STATIC_SHAREDIR "@SHAREDIR@"
diff --git a/src/config.h b/src/config.h
index aedca8b20a61269ba8f6402291eb3a078ef648a1..f37ec0fed466d78a3585930df8ff2ad675a692fa 100644
--- a/src/config.h
+++ b/src/config.h
@@ -11,6 +11,7 @@
#define RUN_IN_PLACE 0
#define USE_GETTEXT 0
#define USE_SOUND 0
+#define USE_CURL 0
#define STATIC_SHAREDIR ""
#define BUILD_INFO "non-cmake"
@@ -26,6 +27,8 @@
#define USE_GETTEXT CMAKE_USE_GETTEXT
#undef USE_SOUND
#define USE_SOUND CMAKE_USE_SOUND
+ #undef USE_CURL
+ #define USE_CURL CMAKE_USE_CURL
#undef STATIC_SHAREDIR
#define STATIC_SHAREDIR CMAKE_STATIC_SHAREDIR
#undef BUILD_INFO
diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp
index d824d292387542a50975819f13babc54a469b2ce..7cb781276b1da9d29def86ca15e68d03d97615fc 100644
--- a/src/defaultsettings.cpp
+++ b/src/defaultsettings.cpp
@@ -119,6 +119,8 @@ void set_default_settings(Settings *settings)
settings->setDefault("preload_item_visuals", "true");
settings->setDefault("enable_shaders", "2");
+ settings->setDefault("media_fetch_threads", "8");
+
// Server stuff
// "map-dir" doesn't exist by default.
settings->setDefault("default_game", "minetest");
@@ -158,5 +160,6 @@ void set_default_settings(Settings *settings)
settings->setDefault("congestion_control_aim_rtt", "0.2");
settings->setDefault("congestion_control_max_rate", "400");
settings->setDefault("congestion_control_min_rate", "10");
+ settings->setDefault("remote_media", "");
}
diff --git a/src/server.cpp b/src/server.cpp
index 2449f423659f21418b2664e76d7aa0485bc4303a..4b43a3205dd5583c38ebd354b6363a16eb9aa0d6 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -2885,6 +2885,9 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
// (definitions and files)
getClient(peer_id)->definitions_sent = true;
}
+ else if(command == TOSERVER_RECEIVED_MEDIA) {
+ getClient(peer_id)->definitions_sent = true;
+ }
else if(command == TOSERVER_INTERACT)
{
std::string datastring((char*)&data[2], datasize-2);
@@ -4217,6 +4220,7 @@ void Server::sendMediaAnnouncement(u16 peer_id)
os<<serializeString(j->name);
os<<serializeString(j->sha1_digest);
}
+ os<<serializeString(g_settings->get("remote_media"));
// Make data buffer
std::string s = os.str();
@@ -4224,7 +4228,6 @@ void Server::sendMediaAnnouncement(u16 peer_id)
// Send as reliable
m_con.Send(peer_id, 0, data, true);
-
}
struct SendableMedia
diff --git a/src/util/string.cpp b/src/util/string.cpp
index fb39a24c3a3ad9e0481c22f7cec9210b39651e77..215ac299d13cab211219cda3bfc55c1b772a58ca 100644
--- a/src/util/string.cpp
+++ b/src/util/string.cpp
@@ -41,3 +41,9 @@ std::string translatePassword(std::string playername, std::wstring password)
return pwd;
}
+size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata) {
+ std::ostringstream *stream = (std::ostringstream*)userdata;
+ size_t count = size * nmemb;
+ stream->write(ptr, count);
+ return count;
+}
diff --git a/src/util/string.h b/src/util/string.h
index 71b11de3d1109cd8687fd6dfa91af21c6d754912..58274c6773cd7a3807c52c7df7674e29734a81fc 100644
--- a/src/util/string.h
+++ b/src/util/string.h
@@ -282,6 +282,7 @@ inline std::string wrap_rows(const std::string &from, u32 rowlen)
}
std::string translatePassword(std::string playername, std::wstring password);
+size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata);
#endif