diff --git a/backends/keymapper/action.h b/backends/keymapper/action.h
index 11e4b82e070..0e35d80bad5 100644
--- a/backends/keymapper/action.h
+++ b/backends/keymapper/action.h
@@ -51,7 +51,7 @@ private:
 	Array<String> _defaultInputMapping;
 
 public:
-	Action(const char *id, const String &description = "");
+	Action(const char *id, const String &description);
 
 	void setEvent(const Event &evt) {
 		event = evt;
diff --git a/backends/keymapper/hardware-input.cpp b/backends/keymapper/hardware-input.cpp
index 91d5b3c2a98..42041b13ead 100644
--- a/backends/keymapper/hardware-input.cpp
+++ b/backends/keymapper/hardware-input.cpp
@@ -23,10 +23,11 @@
 #include "backends/keymapper/hardware-input.h"
 
 #include "backends/keymapper/keymapper.h"
+#include "common/tokenizer.h"
 
 namespace Common {
 
-static const KeyTableEntry defaultKeys[] = {
+const KeyTableEntry defaultKeys[] = {
 	{"BACKSPACE", KEYCODE_BACKSPACE, "Backspace"},
 	{"TAB", KEYCODE_TAB, "Tab"},
 	{"CLEAR", KEYCODE_CLEAR, "Clear"},
@@ -205,100 +206,175 @@ static const KeyTableEntry defaultKeys[] = {
 	{0, KEYCODE_INVALID, 0}
 };
 
-// TODO: Add META and NUM_LOCK
-static const ModifierTableEntry defaultModifiers[] = {
-	{ 0, "", "" },
-	{ KBD_CTRL, "C+", "Ctrl+" },
-	{ KBD_ALT, "A+", "Alt+" },
-	{ KBD_SHIFT, "S+", "Shift+" },
-	{ KBD_CTRL | KBD_ALT, "C+A+", "Ctrl+Alt+" },
-	{ KBD_SHIFT | KBD_CTRL, "S+C+", "Shift+Ctrl+" },
-	{ KBD_SHIFT | KBD_CTRL | KBD_ALT, "S+C+A+", "Shift+Ctrl+Alt+" },
-	{ 0, 0, 0 }
+// TODO: Add NUM_LOCK
+const ModifierTableEntry defaultModifiers[] = {
+	{ KBD_CTRL,  "C", "Ctrl+"  },
+	{ KBD_SHIFT, "S", "Shift+" },
+	{ KBD_ALT,   "A", "Alt+"   },
+	{ KBD_META,  "M", "Meta+"  },
+	{ 0,     nullptr, nullptr }
 };
 
-HardwareInputSet::HardwareInputSet(bool useDefault, const KeyTableEntry *keys, const ModifierTableEntry *modifiers) {
-	if (useDefault)
-		addHardwareInputs(defaultKeys, defaultModifiers);
-	if (keys)
-		addHardwareInputs(keys, modifiers ? modifiers : defaultModifiers);
-}
-
 HardwareInputSet::~HardwareInputSet() {
-	for (KeyInputMap::iterator it = _keyInput.begin(); it != _keyInput.end(); ++it)
-		delete it->_value;
-	for (CustomInputMap::iterator it = _customInput.begin(); it != _customInput.end(); ++it)
-		delete it->_value;
 }
 
-const HardwareInput *HardwareInputSet::findHardwareInput(const String &id) const {
-	for (KeyInputMap::const_iterator it = _keyInput.begin(); it != _keyInput.end(); ++it) {
-		if ((*it)._value->id == id)
-			return (*it)._value;
-	}
-	for (CustomInputMap::const_iterator it = _customInput.begin(); it != _customInput.end(); ++it) {
-		if ((*it)._value->id == id)
-			return (*it)._value;
-	}
-
-	return nullptr;
+KeyboardHardwareInputSet::KeyboardHardwareInputSet(const KeyTableEntry *keys, const ModifierTableEntry *modifiers) :
+		_keys(keys),
+		_modifiers(modifiers) {
+	assert(_keys);
+	assert(_modifiers);
 }
 
-const HardwareInput *HardwareInputSet::findHardwareInput(const HardwareInputCode code) const {
-	return _customInput[code];
-}
+HardwareInput KeyboardHardwareInputSet::findHardwareInput(const String &id) const {
+	StringTokenizer tokenizer(id, "+");
 
-const HardwareInput *HardwareInputSet::findHardwareInput(const KeyState &keystate) const {
-	return _keyInput[keystate];
-}
+	byte modifierFlags = 0;
 
-void HardwareInputSet::addHardwareInputs(const HardwareInputTableEntry inputs[]) {
-	for (const HardwareInputTableEntry *entry = inputs; entry->hwId; ++entry) {
-		const HardwareInput *existingInput = findHardwareInput(entry->code);
-		if (existingInput) {
-			warning("Ignoring hardware input %s (code %d) because an input with the same code is already defined",
-			        entry->desc, entry->code);
-			continue;
-		}
+	// TODO: Normalize modifier order
+	String fullKeyDesc;
 
-		existingInput = findHardwareInput(entry->hwId);
-		if (existingInput) {
-			warning("Ignoring hardware input %s (id %s) because an input with the same id is already defined",
-			        entry->desc, entry->hwId);
-			continue;
-		}
+	String token;
+	while (!tokenizer.empty()) {
+		token = tokenizer.nextToken();
 
-		_customInput[entry->code] = new HardwareInput(entry->hwId, entry->code, entry->desc);
-	}
-}
-
-void HardwareInputSet::addHardwareInputs(const KeyTableEntry keys[], const ModifierTableEntry modifiers[]) {
-	const KeyTableEntry *key;
-	const ModifierTableEntry *mod;
-
-	for (mod = modifiers; mod->id; mod++) {
-		for (key = keys; key->hwId; key++) {
-			String keyId = String::format("%s%s", mod->id, key->hwId);
-			KeyState keystate = KeyState(key->keycode, 0, mod->flag);
-
-			const HardwareInput *existingInput = findHardwareInput(keystate);
-			if (existingInput) {
-				warning("Ignoring hardware input %s%s (id %s) because an input with the same keystate is already defined",
-				        keys->desc, mod->desc, keyId.c_str());
-				continue;
+		const ModifierTableEntry *modifier = nullptr;
+		for (modifier = _modifiers;  modifier->id; modifier++) {
+			if (token == modifier->id) {
+				break;
 			}
+		}
 
-			existingInput = findHardwareInput(keyId);
-			if (existingInput) {
-				warning("Ignoring hardware input %s%s (id %s) because an input with the same id is already defined",
-				        keys->desc, mod->desc, keyId.c_str());
-				continue;
-			}
-
-			String fullKeyDesc = String::format("%s%s", mod->desc, key->desc);
-			_keyInput[keystate] = new HardwareInput(keyId, keystate, fullKeyDesc);
+		if (modifier && modifier->id) {
+			modifierFlags |= modifier->flag;
+			fullKeyDesc += modifier->desc;
+		} else {
+			// We reached the end of the modifiers, the token is a keycode
+			break;
 		}
 	}
+
+	if (!tokenizer.empty()) {
+		return HardwareInput();
+	}
+
+	const KeyTableEntry *key = nullptr;
+	for (key = _keys;  key->hwId; key++) {
+		if (token.equals(key->hwId)) {
+			break;
+		}
+	}
+
+	if (!key || !key->hwId) {
+		return HardwareInput();
+	}
+
+	const KeyState keystate = KeyState(key->keycode, 0, modifierFlags);
+	return HardwareInput(id, keystate, fullKeyDesc + key->desc);
+}
+
+HardwareInput KeyboardHardwareInputSet::findHardwareInput(const Event &event) const {
+	switch (event.type) {
+	case EVENT_KEYDOWN:
+	case EVENT_KEYUP: {
+		const KeyTableEntry *key = nullptr;
+		for (key = _keys;  key->hwId; key++) {
+			if (event.kbd.keycode == key->keycode) {
+				break;
+			}
+		}
+
+		if (!key || !key->hwId) {
+			return HardwareInput();
+		}
+
+		String id;
+		String fullKeyDesc;
+		byte modifierFlags = 0;
+
+		for (const ModifierTableEntry *modifier = _modifiers;  modifier->id; modifier++) {
+			if (event.kbd.hasFlags(modifier->flag)) {
+				id += modifier->id;
+				id += "+";
+				fullKeyDesc += modifier->desc;
+				modifierFlags |= modifier->flag;
+			}
+		}
+
+		const KeyState keystate = KeyState(key->keycode, 0, modifierFlags);
+		return HardwareInput(id + key->hwId, keystate, fullKeyDesc + key->desc);
+	}
+	default:
+		return HardwareInput();
+	}
+}
+
+CustomHardwareInputSet::CustomHardwareInputSet(const HardwareInputTableEntry *hardwareEntries) :
+		_hardwareEntries(hardwareEntries) {
+	assert(_hardwareEntries);
+}
+
+HardwareInput CustomHardwareInputSet::findHardwareInput(const String &id) const {
+	const HardwareInputTableEntry *hw = nullptr;
+	for (hw = _hardwareEntries;  hw->hwId; hw++) {
+		if (id.equals(hw->hwId)) {
+			break;
+		}
+	}
+
+	if (!hw || !hw->hwId) {
+		return HardwareInput();
+	}
+
+	return HardwareInput(hw->hwId, hw->code, hw->desc);
+}
+
+HardwareInput CustomHardwareInputSet::findHardwareInput(const Event &event) const {
+	switch (event.type) {
+	case EVENT_CUSTOM_BACKEND_HARDWARE: {
+		const HardwareInputTableEntry *hw = nullptr;
+		for (hw = _hardwareEntries;  hw->hwId; hw++) {
+			if (event.customType == hw->code) {
+				break;
+			}
+		}
+
+		if (!hw || !hw->hwId) {
+			return HardwareInput();
+		}
+
+		return HardwareInput(hw->hwId, hw->code, hw->desc);
+	}
+	default:
+		return HardwareInput();
+	}
+}
+
+CompositeHardwareInputSet::~CompositeHardwareInputSet() {
+	for (uint i = 0; i < _inputSets.size(); i++) {
+		delete _inputSets[i];
+	}
+}
+
+HardwareInput CompositeHardwareInputSet::findHardwareInput(const String &id) const {
+	for (uint i = 0; i < _inputSets.size(); i++) {
+		HardwareInput hardwareInput = _inputSets[i]->findHardwareInput(id);
+		if (hardwareInput.type != kHardwareInputTypeInvalid) {
+			return hardwareInput;
+		}
+	}
+
+	return HardwareInput();
+}
+
+HardwareInput CompositeHardwareInputSet::findHardwareInput(const Event &event) const {
+	for (uint i = 0; i < _inputSets.size(); i++) {
+		HardwareInput hardwareInput = _inputSets[i]->findHardwareInput(event);
+		if (hardwareInput.type != kHardwareInputTypeInvalid) {
+			return hardwareInput;
+		}
+	}
+
+	return HardwareInput();
 }
 
 } //namespace Common
diff --git a/backends/keymapper/hardware-input.h b/backends/keymapper/hardware-input.h
index 07d42bd7029..d2055dc5dfc 100644
--- a/backends/keymapper/hardware-input.h
+++ b/backends/keymapper/hardware-input.h
@@ -25,16 +25,18 @@
 
 #include "common/scummsys.h"
 
-#include "common/hashmap.h"
+#include "common/array.h"
+#include "common/events.h"
 #include "common/keyboard.h"
 #include "common/str.h"
-#include "common/textconsole.h"
 
 namespace Common {
 
 typedef uint32 HardwareInputCode;
 
 enum HardwareInputType {
+	/** Empty / invalid input type */
+	kHardwareInputTypeInvalid,
 	/** Input that sends single events */
 	kHardwareInputTypeGeneric,
 	/** Input that usually send -up and -down events */
@@ -51,7 +53,8 @@ struct HardwareInput {
 	/** Human readable description */
 	String description;
 
-	const HardwareInputType type;
+	/** Type tag */
+	HardwareInputType type;
 
 	/**
 	 * A platform specific unique identifier for an input event
@@ -67,13 +70,19 @@ struct HardwareInput {
 	 */
 	KeyState key;
 
-	HardwareInput(const String &i, HardwareInputCode ic = 0, const String &desc = "")
+	HardwareInput()
+		: inputCode(0), type(kHardwareInputTypeInvalid) { }
+
+	HardwareInput(const String &i, HardwareInputCode ic, const String &desc)
 		: id(i), inputCode(ic), description(desc), type(kHardwareInputTypeGeneric) { }
 
-	HardwareInput(const String &i, KeyState ky, const String &desc = "")
-		: id(i), key(ky), description(desc), type(kHardwareInputTypeKeyboard) { }
+	HardwareInput(const String &i, KeyState ky, const String &desc)
+		: id(i), inputCode(0), key(ky), description(desc), type(kHardwareInputTypeKeyboard) { }
 };
 
+/**
+ * Entry in a static table of custom backend hardware inputs
+ */
 struct HardwareInputTableEntry {
 	const char *hwId;
 	HardwareInputCode code;
@@ -99,62 +108,95 @@ struct ModifierTableEntry {
 };
 
 /**
- * Hash function for KeyState
- */
-template<> struct Hash<KeyState>
-		: public UnaryFunction<KeyState, uint> {
-
-	uint operator()(const KeyState &val) const {
-		return (uint)val.keycode | ((uint)val.flags << 24);
-	}
-};
-
-/**
- * Simple class to encapsulate a device's set of HardwareInputs.
- * Each device should instantiate this and call addHardwareInput a number of times
- * in its constructor to define the device's available keys.
+ * Interface for querying information about a hardware input device
  */
 class HardwareInputSet {
 public:
-
-	/**
-	 * Add hardware input keys to the set out of key and modifier tables.
-	 * @param useDefault	auto-add the built-in default inputs
-	 * @param keys       	table of available keys
-	 * @param modifiers  	table of available modifiers
-	 */
-	HardwareInputSet(bool useDefault = false, const KeyTableEntry keys[] = 0, const ModifierTableEntry modifiers[] = 0);
-
 	virtual ~HardwareInputSet();
 
-	const HardwareInput *findHardwareInput(const String &id) const;
-
-	const HardwareInput *findHardwareInput(const HardwareInputCode code) const;
-
-	const HardwareInput *findHardwareInput(const KeyState &keystate) const;
+	/**
+	 * Retrieve a hardware input description from an unique identifier
+	 *
+	 * In case no input was found with the specified id, an empty
+	 * HardwareInput structure is return with the type set to
+	 * kHardwareInputTypeInvalid.
+	 */
+	virtual HardwareInput findHardwareInput(const String &id) const = 0;
 
 	/**
-	 * Add hardware inputs to the set out of a table.
-	 * @param inputs       table of available inputs
+	 * Retrieve a hardware input description from one of the events
+	 * produced when the input is triggered.
+	 *
+	 * In case the specified event is not produced by this device,
+	 * an empty HardwareInput structure is return with the type set to
+	 * kHardwareInputTypeInvalid.
 	 */
-	void addHardwareInputs(const HardwareInputTableEntry inputs[]);
+	virtual HardwareInput findHardwareInput(const Event &event) const = 0;
+};
 
-	/**
-	 * Add hardware inputs to the set out of key and modifier tables.
-	 * @param keys       table of available keys
-	 * @param modifiers  table of available modifiers
-	 */
-	void addHardwareInputs(const KeyTableEntry keys[], const ModifierTableEntry modifiers[]);
+/**
+ * A keyboard input device
+ *
+ * Describes the keys and key + modifiers combinations as HardwareInputs
+ */
+class KeyboardHardwareInputSet : public HardwareInputSet {
+public:
+	KeyboardHardwareInputSet(const KeyTableEntry *keys, const ModifierTableEntry *modifiers);
+
+	// HardwareInputSet API
+	HardwareInput findHardwareInput(const String &id) const override;
+	HardwareInput findHardwareInput(const Event &event) const override;
 
 private:
-
-	typedef HashMap<KeyState, const HardwareInput *> KeyInputMap;
-	typedef HashMap<HardwareInputCode, const HardwareInput *> CustomInputMap;
-
-	KeyInputMap _keyInput;
-	CustomInputMap _customInput;
+	const KeyTableEntry *_keys;
+	const ModifierTableEntry *_modifiers;
 };
 
+/**
+ * A custom backend input device
+ *
+ * @todo This is currently unused. Perhaps it should be removed.
+ */
+class CustomHardwareInputSet : public HardwareInputSet {
+public:
+	CustomHardwareInputSet(const HardwareInputTableEntry *hardwareEntries);
+
+	// HardwareInputSet API
+	HardwareInput findHardwareInput(const String &id) const override;
+	HardwareInput findHardwareInput(const Event &event) const override;
+
+private:
+	const HardwareInputTableEntry *_hardwareEntries;
+};
+
+/**
+ * A composite input device that delegates to a set of actual input devices.
+ */
+class CompositeHardwareInputSet : public HardwareInputSet {
+public:
+	~CompositeHardwareInputSet() override;
+
+	// HardwareInputSet API
+	HardwareInput findHardwareInput(const String &id) const override;
+	HardwareInput findHardwareInput(const Event &event) const override;
+
+	/**
+	 * Add an input device to this composite device
+	 *
+	 * Takes ownership of the hardware input set
+	 */
+	void addHardwareInputSet(HardwareInputSet *hardwareInputSet);
+
+private:
+	Array<HardwareInputSet *> _inputSets;
+};
+
+/** A standard set of keyboard keys */
+extern const KeyTableEntry defaultKeys[];
+
+/** A standard set of keyboard modifiers */
+extern const ModifierTableEntry defaultModifiers[];
+
 } // End of namespace Common
 
 #endif // #ifndef COMMON_HARDWARE_KEY_H
diff --git a/backends/keymapper/input-watcher.cpp b/backends/keymapper/input-watcher.cpp
index 0dc989bc674..90b0218dfa4 100644
--- a/backends/keymapper/input-watcher.cpp
+++ b/backends/keymapper/input-watcher.cpp
@@ -30,14 +30,13 @@ namespace Common {
 InputWatcher::InputWatcher(EventDispatcher *eventDispatcher, Keymapper *keymapper) :
 		_eventDispatcher(eventDispatcher),
 		_keymapper(keymapper),
-		_watching(false),
-		_hwInput(nullptr) {
+		_watching(false) {
 
 }
 
 void InputWatcher::startWatching() {
 	assert(!_watching);
-	assert(!_hwInput);
+	assert(_hwInput.type == kHardwareInputTypeInvalid);
 
 	_keymapper->setEnabled(false);
 	_eventDispatcher->registerObserver(this, EventManager::kEventRemapperPriority, false);
@@ -56,7 +55,7 @@ bool InputWatcher::isWatching() const {
 
 bool InputWatcher::notifyEvent(const Event &event) {
 	assert(_watching);
-	assert(!_hwInput);
+	assert(_hwInput.type == kHardwareInputTypeInvalid);
 
 	switch (event.type) {
 		case EVENT_KEYDOWN:
@@ -64,7 +63,7 @@ bool InputWatcher::notifyEvent(const Event &event) {
 		case EVENT_KEYUP:
 		case EVENT_CUSTOM_BACKEND_HARDWARE:
 			_hwInput = _keymapper->findHardwareInput(event);
-			if (_hwInput) {
+			if (_hwInput.type != kHardwareInputTypeInvalid) {
 				stopWatching();
 			}
 			return true;
@@ -75,9 +74,9 @@ bool InputWatcher::notifyEvent(const Event &event) {
 	return false;
 }
 
-const HardwareInput *InputWatcher::checkForCapturedInput() {
-	const HardwareInput *hwInput = _hwInput;
-	_hwInput = nullptr;
+HardwareInput InputWatcher::checkForCapturedInput() {
+	HardwareInput hwInput = _hwInput;
+	_hwInput = HardwareInput();
 	return hwInput;
 }
 
diff --git a/backends/keymapper/input-watcher.h b/backends/keymapper/input-watcher.h
index 5818fe66184..672edb6bafd 100644
--- a/backends/keymapper/input-watcher.h
+++ b/backends/keymapper/input-watcher.h
@@ -25,6 +25,7 @@
 
 #include "common/scummsys.h"
 
+#include "backends/keymapper/hardware-input.h"
 #include "common/events.h"
 
 namespace Common {
@@ -48,7 +49,7 @@ public:
 	void stopWatching();
 
 	bool isWatching() const;
-	const HardwareInput *checkForCapturedInput();
+	HardwareInput checkForCapturedInput();
 
 private:
 	bool notifyEvent(const Event &event) override;
@@ -57,7 +58,7 @@ private:
 	Keymapper *_keymapper;
 
 	bool _watching;
-	const HardwareInput *_hwInput;
+	HardwareInput _hwInput;
 };
 
 } // End of namespace Common
diff --git a/backends/keymapper/keymap.cpp b/backends/keymapper/keymap.cpp
index 917d3eccc2d..5a306dbff11 100644
--- a/backends/keymapper/keymap.cpp
+++ b/backends/keymapper/keymap.cpp
@@ -55,7 +55,7 @@ void Keymap::addAction(Action *action) {
 	_actions.push_back(action);
 }
 
-void Keymap::registerMapping(Action *action, const HardwareInput *hwInput) {
+void Keymap::registerMapping(Action *action, const HardwareInput &hwInput) {
 	ActionArray &actionArray = _hwActionMap.getVal(hwInput);
 
 	// Don't allow an input to map to the same action multiple times
@@ -87,8 +87,8 @@ void Keymap::resetMapping(Action *action) {
 	registerMappings(action, hwInputIds);
 }
 
-Array<const HardwareInput *> Keymap::getActionMapping(Action *action) const {
-	Array<const HardwareInput *> inputs;
+Array<HardwareInput> Keymap::getActionMapping(Action *action) const {
+	Array<HardwareInput> inputs;
 
 	for (HardwareActionMap::iterator itInput = _hwActionMap.begin(); itInput != _hwActionMap.end(); itInput++) {
 		for (ActionArray::iterator itAction = itInput->_value.begin(); itAction != itInput->_value.end(); itAction++) {
@@ -111,8 +111,20 @@ const Action *Keymap::findAction(const char *id) const {
 	return nullptr;
 }
 
-const Keymap::ActionArray &Keymap::getMappedActions(const HardwareInput *hardwareInput) const {
-	return _hwActionMap[hardwareInput];
+Keymap::ActionArray Keymap::getMappedActions(const Event &event) const {
+	switch (event.type) {
+	case EVENT_KEYDOWN:
+	case EVENT_KEYUP: {
+		HardwareInput hardwareInput("", event.kbd, "");
+		return _hwActionMap[hardwareInput];
+	}
+	case EVENT_CUSTOM_BACKEND_HARDWARE: {
+		HardwareInput hardwareInput("", event.customType, "");
+		return _hwActionMap[hardwareInput];
+	}
+	default:
+		return ActionArray();
+	}
 }
 
 void Keymap::setConfigDomain(ConfigManager::Domain *configDomain) {
@@ -188,9 +200,9 @@ void Keymap::registerMappings(Action *action, const Array <String> &hwInputIds)
 	assert(_hardwareInputSet);
 
 	for (uint i = 0; i < hwInputIds.size(); i++) {
-			const HardwareInput *hwInput = _hardwareInputSet->findHardwareInput(hwInputIds[i].c_str());
+			HardwareInput hwInput = _hardwareInputSet->findHardwareInput(hwInputIds[i].c_str());
 
-			if (!hwInput) {
+			if (hwInput.type == kHardwareInputTypeInvalid) {
 				// Silently ignore unknown hardware ids because the current device may not have inputs matching the defaults
 				debug(1, "HardwareInput with ID '%s' not known", hwInputIds[i].c_str());
 				continue;
@@ -209,7 +221,7 @@ void Keymap::saveMappings() {
 
 	for (ActionArray::const_iterator it = _actions.begin(); it != _actions.end(); it++) {
 		Action *action = *it;
-		Array<const HardwareInput *> mappedInputs = getActionMapping(action);
+		Array<HardwareInput> mappedInputs = getActionMapping(action);
 
 		if (areMappingsIdentical(mappedInputs, action->getDefaultInputMapping())) {
 			// If the current mapping is the default, don't write anything to the config manager
@@ -224,14 +236,14 @@ void Keymap::saveMappings() {
 				confValue += " ";
 			}
 
-			confValue += mappedInputs[j]->id;
+			confValue += mappedInputs[j].id;
 		}
 
 		_configDomain->setVal(prefix + action->id, confValue);
 	}
 }
 
-bool Keymap::areMappingsIdentical(const Array<const HardwareInput *> &inputs, const Array<String> &mapping) {
+bool Keymap::areMappingsIdentical(const Array<HardwareInput> &inputs, const StringArray &mapping) {
 	if (inputs.size() != mapping.size()) {
 		return false;
 	}
@@ -241,7 +253,7 @@ bool Keymap::areMappingsIdentical(const Array<const HardwareInput *> &inputs, co
 	uint foundCount = 0;
 	for (uint i = 0; i < inputs.size(); i++) {
 		for (uint j = 0; j < mapping.size(); j++) {
-			if (inputs[i]->id == mapping[j]) {
+			if (inputs[i].id == mapping[j]) {
 				foundCount++;
 				break;
 			}
diff --git a/backends/keymapper/keymap.h b/backends/keymapper/keymap.h
index 1cdc538e6fb..7abc4390d0d 100644
--- a/backends/keymapper/keymap.h
+++ b/backends/keymapper/keymap.h
@@ -25,21 +25,44 @@
 
 #include "common/scummsys.h"
 
+#include "backends/keymapper/hardware-input.h"
+
 #include "common/config-manager.h"
 #include "common/func.h"
 #include "common/hashmap.h"
 #include "common/hash-ptr.h"
 #include "common/list.h"
+#include "common/str-array.h"
 
 namespace Common {
 
 const char *const kStandardActionsKeymapName = "standard-actions";
 
 class Action;
+class Event;
 struct HardwareInput;
 class HardwareInputSet;
 class KeymapperDefaultBindings;
 
+struct Event_EqualTo {
+	bool operator()(const HardwareInput& x, const HardwareInput& y) const {
+		return (x.type == y.type)
+		        && (x.key == y.key) // TODO: Remove the equality operator from KeyState
+		        && (x.inputCode == y.inputCode);
+	}
+};
+
+struct Event_Hash {
+	uint operator()(const HardwareInput& x) const {
+		uint hash = 7;
+		hash = 31 * hash + x.type;
+		hash = 31 * hash + x.key.keycode;
+		hash = 31 * hash + (x.key.flags & ~KBD_STICKY);
+		hash = 31 * hash + x.inputCode;
+		return hash;
+	}
+};
+
 class Keymap {
 public:
 	enum KeymapType {
@@ -62,7 +85,7 @@ public:
 	* @param key pointer to HardwareInput to map
 	* @see Action::mapKey
 	*/
-	void registerMapping(Action *action, const HardwareInput *input);
+	void registerMapping(Action *action, const HardwareInput &input);
 
 	/**
 	* Unregisters a HardwareInput from the given Action (if one is mapped)
@@ -80,14 +103,14 @@ public:
 	/**
 	 * Find the hardware input an action is mapped to, if any
 	 */
-	Array<const HardwareInput *> getActionMapping(Action *action) const;
+	Array<HardwareInput> getActionMapping(Action *action) const;
 
 	/**
 	 * Find the Actions that a hardware input is mapped to
 	 * @param hardwareInput	the input that is mapped to the required Action
 	 * @return		an array containing pointers to the actions
 	 */
-	const ActionArray &getMappedActions(const HardwareInput *hardwareInput) const;
+	ActionArray getMappedActions(const Event &event) const;
 
 	/**
 	 * Adds a new Action to this Map
@@ -129,10 +152,10 @@ private:
 
 	const Action *findAction(const char *id) const;
 
-	void registerMappings(Action *action, const Array<String> &hwInputIds);
-	bool areMappingsIdentical(const Array<const HardwareInput *> &inputs, const Array <String> &mapping);
+	void registerMappings(Action *action, const StringArray &hwInputIds);
+	bool areMappingsIdentical(const Array<HardwareInput> &inputs, const StringArray &mapping);
 
-	typedef HashMap<const HardwareInput *, ActionArray> HardwareActionMap;
+	typedef HashMap<HardwareInput, ActionArray, Event_Hash, Event_EqualTo> HardwareActionMap;
 
 	KeymapType _type;
 	String _name;
diff --git a/backends/keymapper/keymapper.cpp b/backends/keymapper/keymapper.cpp
index ce399803e14..46202c6e206 100644
--- a/backends/keymapper/keymapper.cpp
+++ b/backends/keymapper/keymapper.cpp
@@ -56,7 +56,7 @@ void Keymapper::registerHardwareInputSet(HardwareInputSet *inputs) {
 
 	if (!inputs) {
 		warning("No hardware input were defined, using defaults");
-		inputs = new HardwareInputSet(true);
+		inputs = new KeyboardHardwareInputSet(defaultKeys, defaultModifiers);
 	}
 
 	_hardwareInputs = inputs;
@@ -147,13 +147,6 @@ List<Event> Keymapper::mapEvent(const Event &ev) {
 
 	hardcodedEventMapping(ev);
 
-	const HardwareInput *hwInput = findHardwareInput(ev);
-	if (!hwInput) {
-		List<Event> originalEvent;
-		originalEvent.push_back(ev);
-		return originalEvent;
-	}
-
 	IncomingEventType incomingEventType = convertToIncomingEventType(ev);
 
 	List<Event> mappedEvents;
@@ -169,7 +162,7 @@ List<Event> Keymapper::mapEvent(const Event &ev) {
 
 		debug(5, "Keymapper::mapKey keymap: %s", _keymaps[i]->getName().c_str());
 
-		const Keymap::ActionArray &actions = _keymaps[i]->getMappedActions(hwInput);
+		const Keymap::ActionArray &actions = _keymaps[i]->getMappedActions(ev);
 		for (Keymap::ActionArray::const_iterator it = actions.begin(); it != actions.end(); it++) {
 			mappedEvents.push_back(executeAction(*it, incomingEventType));
 		}
@@ -261,16 +254,8 @@ EventType Keymapper::convertStartToEnd(EventType type) {
 	return result;
 }
 
-const HardwareInput *Keymapper::findHardwareInput(const Event &event) {
-	switch (event.type) {
-		case EVENT_KEYDOWN:
-		case EVENT_KEYUP:
-			return _hardwareInputs->findHardwareInput(event.kbd);
-		case EVENT_CUSTOM_BACKEND_HARDWARE:
-			return _hardwareInputs->findHardwareInput(event.customType);
-		default:
-			return nullptr;
-	}
+HardwareInput Keymapper::findHardwareInput(const Event &event) {
+	return _hardwareInputs->findHardwareInput(event);
 }
 
 void Keymapper::hardcodedEventMapping(Event ev) {
diff --git a/backends/keymapper/keymapper.h b/backends/keymapper/keymapper.h
index c5ee304afb9..9e3e3f79bd6 100644
--- a/backends/keymapper/keymapper.h
+++ b/backends/keymapper/keymapper.h
@@ -121,7 +121,7 @@ public:
 	/**
 	 * Return a HardwareInput pointer for the given event
 	 */
-	const HardwareInput *findHardwareInput(const Event &event);
+	HardwareInput findHardwareInput(const Event &event);
 
 	void initKeymap(Keymap *keymap, ConfigManager::Domain *domain);
 
diff --git a/backends/keymapper/remap-widget.cpp b/backends/keymapper/remap-widget.cpp
index d96b73767d8..7a875193dea 100644
--- a/backends/keymapper/remap-widget.cpp
+++ b/backends/keymapper/remap-widget.cpp
@@ -216,8 +216,8 @@ void RemapWidget::handleMouseDown(int x, int y, int button, int clickCount) {
 }
 
 void RemapWidget::handleTickle() {
-	const HardwareInput *hardwareInput = _remapInputWatcher->checkForCapturedInput();
-	if (hardwareInput) {
+	const HardwareInput hardwareInput = _remapInputWatcher->checkForCapturedInput();
+	if (hardwareInput.type != kHardwareInputTypeInvalid) {
 		_remapKeymap->registerMapping(_remapAction, hardwareInput);
 
 		_changes = true;
@@ -260,7 +260,7 @@ void RemapWidget::refreshKeymap() {
 
 		row.actionText->setLabel(row.action->description);
 
-		Array<const HardwareInput *> mappedInputs = row.keymap->getActionMapping(row.action);
+		Array<HardwareInput> mappedInputs = row.keymap->getActionMapping(row.action);
 
 		String keysLabel;
 		for (uint j = 0; j < mappedInputs.size(); j++) {
@@ -268,7 +268,7 @@ void RemapWidget::refreshKeymap() {
 				keysLabel += ", ";
 			}
 
-			keysLabel += mappedInputs[j]->description;
+			keysLabel += mappedInputs[j].description;
 		}
 
 		if (!keysLabel.empty()) {
diff --git a/backends/platform/linuxmoto/hardwarekeys.cpp b/backends/platform/linuxmoto/hardwarekeys.cpp
index 5d18aee1073..449dd3f03b9 100644
--- a/backends/platform/linuxmoto/hardwarekeys.cpp
+++ b/backends/platform/linuxmoto/hardwarekeys.cpp
@@ -103,7 +103,3 @@ static const Mod modifiers[] = {
 	{ KBD_SHIFT | KBD_CTRL | KBD_ALT, "C+A+", "Ctrl+Alt+", true },
 	{ 0, 0, 0, false }
 };
-
-Common::HardwareInputSet *OSystem_LINUXMOTO::getHardwareInputSet() {
-	return OSystem_SDL::getHardwareInputSet();
-}
diff --git a/backends/platform/maemo/maemo.cpp b/backends/platform/maemo/maemo.cpp
index b2e4990032a..517b3ce48fa 100644
--- a/backends/platform/maemo/maemo.cpp
+++ b/backends/platform/maemo/maemo.cpp
@@ -184,7 +184,11 @@ static const Common::KeyTableEntry maemoKeys[] = {
 };
 
 Common::HardwareInputSet *OSystem_SDL_Maemo::getHardwareInputSet() {
-	return new Common::HardwareInputSet(true, maemoKeys);
+	Common::CompositeHardwareInputSet inputSet = new Common::CompositeHardwareInputSet();
+	inputSet->addHardwareInputSet(new Common::KeyboardHardwareInputSet(maemoKeys, defaultModifiers));
+	inputSet->addHardwareInputSet(new Common::KeyboardHardwareInputSet(defaultKeys, defaultModifiers));
+
+	return inputSet;
 }
 
 Common::KeymapArray OSystem_SDL_Maemo::getGlobalKeymaps() {