From 7c59a77aca7740a7b8a3d1a54c7aef96d2ba6e9b Mon Sep 17 00:00:00 2001
From: Eugene Sandulenko <sev@scummvm.org>
Date: Sun, 23 Apr 2023 16:20:37 +0200
Subject: [PATCH] COMMON: Added string splitter to StringTokenizer class

---
 common/tokenizer.cpp | 18 ++++++++++++++++++
 common/tokenizer.h   |  5 +++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/common/tokenizer.cpp b/common/tokenizer.cpp
index f5ef77cc131..45effce118a 100644
--- a/common/tokenizer.cpp
+++ b/common/tokenizer.cpp
@@ -52,6 +52,15 @@ String StringTokenizer::nextToken() {
 	return String(_str.c_str() + _tokenBegin, _tokenEnd - _tokenBegin);
 }
 
+StringArray StringTokenizer::split() {
+	StringArray res;
+
+	while (!empty())
+		res.push_back(nextToken());
+
+	return res;
+}
+
 U32StringTokenizer::U32StringTokenizer(const U32String &str, const String &delimiters) : _str(str), _delimiters(delimiters) {
 	reset();
 }
@@ -99,5 +108,14 @@ U32String U32StringTokenizer::nextToken() {
 		return U32String();
 }
 
+U32StringArray U32StringTokenizer::split() {
+	U32StringArray res;
+
+	while (!empty())
+		res.push_back(nextToken());
+
+	return res;
+}
+
 
 } // End of namespace Common
diff --git a/common/tokenizer.h b/common/tokenizer.h
index a7d8d5f342e..009b2edb9d8 100644
--- a/common/tokenizer.h
+++ b/common/tokenizer.h
@@ -23,8 +23,7 @@
 #define COMMON_TOKENIZER_H
 
 #include "common/scummsys.h"
-#include "common/str.h"
-#include "common/ustr.h"
+#include "common/str-array.h"
 
 namespace Common {
 
@@ -55,6 +54,7 @@ public:
 	void reset();       ///< Resets the tokenizer to its initial state
 	bool empty() const; ///< Returns true if there are no more tokens left in the string, false otherwise
 	String nextToken(); ///< Returns the next token from the string (Or an empty string if there are no more tokens)
+	StringArray split(); ///< Returns StringArray with all tokens. Beware of the memory usage
 
 private:
 	const String _str;        ///< The string to be tokenized
@@ -82,6 +82,7 @@ public:
 	void reset();       ///< Resets the tokenizer to its initial state, i.e points boten token iterators to the beginning
 	bool empty() const; ///< Returns true if there are no more tokens left in the string, false otherwise
 	U32String nextToken(); ///< Returns the next token from the string (Or an empty string if there are no more tokens)
+	U32StringArray split(); ///< Returns StringArray with all tokens. Beware of the memory usage
 
 private:
 	const U32String _str;        ///< The unicode string to be tokenized