CLOUD: Add ConnectionManager and NetworkReadStream

NetworkReadStream actually saves whole response in the memory now.

There is a pause mechanism in libcurl, but if libcurl is requesting
something compressed, it would have to uncompress data as it goes even
if we paused the request. Even though our own stream won't be notified
about this data when when "pause" the request, libcurl's own buffer
wound be expanding.

backends/cloud/curl/connectionmanager.cpp

@@ -0,0 +1,72 @@
+/* ScummVM - Graphic Adventure Engine
+*
+* ScummVM is the legal property of its developers, whose names
+* are too numerous to list here. Please refer to the COPYRIGHT
+* file distributed with this source distribution.
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 2
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*/
+
+#define FORBIDDEN_SYMBOL_ALLOW_ALL
+
+#include "backends/cloud/curl/connectionmanager.h"
+#include "backends/cloud/curl/networkreadstream.h"
+#include "common/debug.h"
+#include <curl/curl.h>
+
+namespace Cloud {
+
+ConnectionManager::ConnectionManager(): _multi(0) {
+	curl_global_init(CURL_GLOBAL_ALL);
+	_multi = curl_multi_init();
+}
+
+ConnectionManager::~ConnectionManager() {
+	curl_multi_cleanup(_multi);
+	curl_global_cleanup();
+}
+
+NetworkReadStream *ConnectionManager::makeRequest(const char *url) {
+	NetworkReadStream *stream = new NetworkReadStream(url);
+	curl_multi_add_handle(_multi, stream->getEasyHandle());
+	return stream;
+}
+
+void ConnectionManager::handle() {
+	int U;
+	curl_multi_perform(_multi, &U);
+
+	int Q;
+	CURLMsg *curlMsg;
+	while ((curlMsg = curl_multi_info_read(_multi, &Q))) {
+		if (curlMsg->msg == CURLMSG_DONE) {
+			CURL *e = curlMsg->easy_handle;
+
+			NetworkReadStream *stream;
+			curl_easy_getinfo(e, CURLINFO_PRIVATE, &stream);
+			if (stream) stream->done();			
+
+			debug("ConnectionManager: SUCCESS (%d - %s)", curlMsg->data.result, curl_easy_strerror(curlMsg->data.result));
+			curl_multi_remove_handle(_multi, e);			
+		}
+		else {
+			debug("ConnectionManager: FAILURE (CURLMsg (%d))", curlMsg->msg);
+			//TODO: notify stream on this case also
+		}
+	}
+}
+
+} //end of namespace Cloud

backends/cloud/curl/connectionmanager.h

@@ -0,0 +1,47 @@
+/* ScummVM - Graphic Adventure Engine
+*
+* ScummVM is the legal property of its developers, whose names
+* are too numerous to list here. Please refer to the COPYRIGHT
+* file distributed with this source distribution.
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 2
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*/
+
+#ifndef BACKENDS_CLOUD_CURL_CONNECTIONMANAGER_H
+#define BACKENDS_CLOUD_CURL_CONNECTIONMANAGER_H
+
+#include "common/str.h"
+
+typedef void CURLM;
+
+namespace Cloud {
+
+class NetworkReadStream;
+
+class ConnectionManager {
+	CURLM *_multi;
+
+public:
+	ConnectionManager();
+	virtual ~ConnectionManager();
+
+	NetworkReadStream *makeRequest(const char *url);
+	void handle();
+};
+
+} //end of namespace Cloud
+
+#endif

backends/cloud/curl/networkreadstream.cpp

@@ -0,0 +1,83 @@
+/* ScummVM - Graphic Adventure Engine
+*
+* ScummVM is the legal property of its developers, whose names
+* are too numerous to list here. Please refer to the COPYRIGHT
+* file distributed with this source distribution.
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 2
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*/
+
+#define FORBIDDEN_SYMBOL_ALLOW_ALL
+
+#include "backends/cloud/curl/networkreadstream.h"
+#include "common/debug.h"
+#include <curl/curl.h>
+
+namespace Cloud {
+
+static size_t curlDataCallback(char *d, size_t n, size_t l, void *p) {	
+	NetworkReadStream *stream = (NetworkReadStream *)p;
+	if (stream) return stream->dataCallback(d, n, l);
+	return 0;
+}
+
+NetworkReadStream::NetworkReadStream(const char *url): _easy(0), _eos(false), _requestComplete(false) {
+	_easy = curl_easy_init();
+	curl_easy_setopt(_easy, CURLOPT_WRITEFUNCTION, curlDataCallback);
+	curl_easy_setopt(_easy, CURLOPT_WRITEDATA, this); //so callback can call us
+	curl_easy_setopt(_easy, CURLOPT_PRIVATE, this); //so ConnectionManager can call us when request is complete
+	curl_easy_setopt(_easy, CURLOPT_HEADER, 0L);
+	curl_easy_setopt(_easy, CURLOPT_URL, url);
+	curl_easy_setopt(_easy, CURLOPT_VERBOSE, 0L);
+}
+
+NetworkReadStream::~NetworkReadStream() {
+	curl_easy_cleanup(_easy);
+}
+
+bool NetworkReadStream::eos() const {
+	return _eos;
+}
+
+uint32 NetworkReadStream::read(void *dataPtr, uint32 dataSize) {
+	uint32 available = _bytes.size();
+
+	if (available == 0) {
+		if (_requestComplete) _eos = true;
+		return 0;
+	}
+
+	char *data = (char *)dataPtr;
+	uint32 actuallyRead = (dataSize < available ? dataSize : available);	
+	for (uint32 i = 0; i < actuallyRead; ++i) data[i] = _bytes[i];
+	data[actuallyRead] = 0;
+	_bytes.erase(0, actuallyRead);
+	return actuallyRead;
+}
+
+void NetworkReadStream::done() {
+	_requestComplete = true;
+}
+
+size_t NetworkReadStream::dataCallback(char *d, size_t n, size_t l) {
+	//TODO: return CURL_WRITEFUNC_PAUSE if _bytes is too long
+	//TODO: remember https://curl.haxx.se/libcurl/c/curl_easy_pause.html (Memory Use / compressed data case)
+	//TODO: if using pause, don't forget to unpause it somehow from read() up there
+	_bytes += Common::String(d, n*l);
+	return n*l;
+}
+
+} //end of namespace Cloud

backends/cloud/curl/networkreadstream.h

@@ -0,0 +1,76 @@
+/* ScummVM - Graphic Adventure Engine
+*
+* ScummVM is the legal property of its developers, whose names
+* are too numerous to list here. Please refer to the COPYRIGHT
+* file distributed with this source distribution.
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 2
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*
+*/
+
+#ifndef BACKENDS_CLOUD_CURL_NETWORKREADSTREAM_H
+#define BACKENDS_CLOUD_CURL_NETWORKREADSTREAM_H
+
+#include "common/stream.h"
+#include "common/str.h"
+
+typedef void CURL;
+
+namespace Cloud {
+
+class NetworkReadStream: public Common::ReadStream {	
+	CURL *_easy;
+	bool _eos, _requestComplete;
+	Common::String _bytes;
+
+public:	
+	NetworkReadStream(const char *url);
+	virtual ~NetworkReadStream();
+
+	CURL *getEasyHandle() const { return _easy; }
+
+	/**
+	* Returns true if a read failed because the stream end has been reached.
+	* This flag is cleared by clearErr().
+	* For a SeekableReadStream, it is also cleared by a successful seek.
+	*
+	* @note The semantics of any implementation of this method are
+	* supposed to match those of ISO C feof(). In particular, in a stream
+	* with N bytes, reading exactly N bytes from the start should *not*
+	* set eos; only reading *beyond* the available data should set it.
+	*/
+	virtual bool eos() const;
+
+	/**
+	* Read data from the stream. Subclasses must implement this
+	* method; all other read methods are implemented using it.
+	*
+	* @note The semantics of any implementation of this method are
+	* supposed to match those of ISO C fread(), in particular where
+	* it concerns setting error and end of file/stream flags.
+	*
+	* @param dataPtr	pointer to a buffer into which the data is read
+	* @param dataSize	number of bytes to be read
+	* @return the number of bytes which were actually read.
+	*/
+	virtual uint32 read(void *dataPtr, uint32 dataSize);
+
+	void done();
+	size_t dataCallback(char *d, size_t n, size_t l);
+};
+
+} //end of namespace Cloud
+
+#endif

backends/cloud/dropbox/curlrequest.cpp

@@ -23,55 +23,37 @@
 #define FORBIDDEN_SYMBOL_ALLOW_ALL
 
 #include "backends/cloud/dropbox/curlrequest.h"
+#include "backends/cloud/curl/networkreadstream.h"
 #include "common/debug.h"
 #include <curl/curl.h>
 
 namespace Cloud {
 namespace Dropbox {
 
-static size_t curlDataCallback(char *d, size_t n, size_t l, void *p) {
+CurlRequest::CurlRequest(Callback cb, const char *url) : Request(cb), _firstTime(true), _stream(0) {	
-	debug("%p got %d more bytes", p, n * l);
-	return n * l;
-}
-
-CurlRequest::CurlRequest(Callback cb, char *url) : Request(cb), _firstTime(true) {
-	_curlm = curl_multi_init();
 	_url = url;
 }
 
 CurlRequest::~CurlRequest() {
-	curl_multi_cleanup(_curlm);
+	if (_stream) delete _stream;
 }
 
-bool CurlRequest::handle() {
+bool CurlRequest::handle(ConnectionManager& manager) {
 	if (_firstTime) {
-		CURL *eh = curl_easy_init();
+		_stream = manager.makeRequest(_url);
-		curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, curlDataCallback);
-		curl_easy_setopt(eh, CURLOPT_WRITEDATA, this);
-		curl_easy_setopt(eh, CURLOPT_HEADER, 0L);
-		curl_easy_setopt(eh, CURLOPT_URL, _url);
-		curl_easy_setopt(eh, CURLOPT_VERBOSE, 0L);
-		curl_multi_add_handle(_curlm, eh);
-
 		_firstTime = false;
 	}
 
-	int U;
+	if (_stream) {
-	curl_multi_perform(_curlm, &U);
+		const int kBufSize = 10000;
-
+		char buf[kBufSize+1];
-	int Q;
+		uint32 readBytes = _stream->read(buf, kBufSize);
-	CURLMsg *_curlMsg;
+		debug("%d", readBytes);
-	while ((_curlMsg = curl_multi_info_read(_curlm, &Q))) {
+		//if(readBytes != 0) debug("%s", buf);
-		if (_curlMsg->msg == CURLMSG_DONE) {			
-			CURL *e = _curlMsg->easy_handle;
-			debug("R: %d - %s\n", _curlMsg->data.result, curl_easy_strerror(_curlMsg->data.result));
-			curl_multi_remove_handle(_curlm, e);
-			curl_easy_cleanup(e);
 
+		if(_stream->eos()) {
 			_callback(0);
 			return true;
-		} else {
-			debug("E: CURLMsg (%d)\n", _curlMsg->msg);
 		}
 	}
 

backends/cloud/dropbox/curlrequest.h

@@ -25,21 +25,22 @@
 
 #include "backends/cloud/request.h"
 
-typedef void CURLM;
-
 namespace Cloud {
+
+class NetworkReadStream;
+
 namespace Dropbox {
 
 class CurlRequest : public Cloud::Request {
 	bool _firstTime;
-	CURLM *_curlm;
+	const char *_url;
-	char *_url;
+	NetworkReadStream *_stream;
 
 public:
-	CurlRequest(Callback cb, char *url);
+	CurlRequest(Callback cb, const char *url);
 	virtual ~CurlRequest();
 
-	virtual bool handle();
+	virtual bool handle(ConnectionManager& manager);
 };
 
 }  //end of namespace Dropbox

backends/cloud/dropbox/dropboxstorage.cpp

@@ -47,7 +47,7 @@ void DropboxStorage::listDirectory(Common::String path) {
 
 void DropboxStorage::syncSaves() {	
 	addRequest(new CurlRequest(curlCallback, "tkachov.ru"));
-	addRequest(new CurlRequest(curlCallback, "bash.im"));
+	addRequest(new CurlRequest(curlCallback, "scummvm.org"));
 }
 
 } //end of namespace Dropbox

backends/cloud/request.h

@@ -23,6 +23,8 @@
 #ifndef BACKENDS_CLOUD_REQUEST_H
 #define BACKENDS_CLOUD_REQUEST_H
 
+#include "backends/cloud/curl/connectionmanager.h"
+
 namespace Cloud {
 
 class Request {
@@ -45,7 +47,7 @@ public:
 	* @return true if request's work is complete and it may be removed from Storage's list
 	*/
 
-	virtual bool handle() = 0;
+	virtual bool handle(ConnectionManager& manager) = 0;
 };
 
 } //end of namespace Cloud

backends/cloud/storage.cpp

@@ -21,6 +21,7 @@
 */
 
 #include "backends/cloud/storage.h"
+#include "common/debug.h"
 #include "common/system.h"
 #include "common/timer.h"
 
@@ -40,15 +41,17 @@ void Storage::addRequest(Request *request) {
 
 void Storage::handler() {
 	//TODO: lock mutex here (in case another handler() would be called before this one ends)
-	warning("handler's here");
+	debug("\nhandler's here");
 	for (Common::Array<Request *>::iterator i = _requests.begin(); i != _requests.end();) {
-		if ((*i)->handle()) {
+		if ((*i)->handle(_connectionManager)) {
 			delete (*i);
 			_requests.erase(i);
 		}
 		else ++i;
 	}
 	if (_requests.empty()) stopTimer();
+
+	_connectionManager.handle();
 	//TODO: unlock mutex here
 }
 

backends/cloud/storage.h

@@ -26,6 +26,7 @@
 #include "common/str.h"
 #include "common/array.h"
 #include "backends/cloud/request.h"
+#include "backends/cloud/curl/connectionmanager.h"
 
 namespace Cloud {
 
@@ -35,6 +36,7 @@ class Storage {
 
 protected:
 	Common::Array<Request *> _requests;
+	ConnectionManager _connectionManager;
 
 	virtual void addRequest(Request *request); //starts the timer if it's not started
 	virtual void handler();	

backends/module.mk

@@ -24,7 +24,9 @@ MODULE_OBJS += \
 	cloud/manager.o \
 	cloud/storage.o \
 	cloud/dropbox/dropboxstorage.o \
-	cloud/dropbox/curlrequest.o
+	cloud/dropbox/curlrequest.o \
+	cloud/curl/connectionmanager.o \
+	cloud/curl/networkreadstream.o
 endif
 
 ifdef USE_ELF_LOADER