diff --git a/backends/events/default/default-events.cpp b/backends/events/default/default-events.cpp
index 51b357cc08f..e2f18189abe 100644
--- a/backends/events/default/default-events.cpp
+++ b/backends/events/default/default-events.cpp
@@ -309,12 +309,14 @@ Common::Keymap *DefaultEventManager::getGlobalKeymap() {
 	Action *act;
 	act = new Action("MENU", _("Menu"));
 	act->addDefaultInputMapping("C+F5");
+	act->addDefaultInputMapping("JOY_START");
 	act->setEvent(EVENT_MAINMENU);
 	globalKeymap->addAction(act);
 
 #ifdef ENABLE_VKEYBD
 	act = new Action("VIRT", _("Display keyboard"));
 	act->addDefaultInputMapping("C+F7");
+	act->addDefaultInputMapping("JOY_BACK");
 	act->setEvent(EVENT_VIRTUAL_KEYBOARD);
 	globalKeymap->addAction(act);
 #endif
diff --git a/backends/events/sdl/sdl-events.cpp b/backends/events/sdl/sdl-events.cpp
index dae7397bc11..98cb5eb01fd 100644
--- a/backends/events/sdl/sdl-events.cpp
+++ b/backends/events/sdl/sdl-events.cpp
@@ -33,23 +33,9 @@
 #include "engines/engine.h"
 #include "gui/gui-manager.h"
 
-// FIXME move joystick defines out and replace with confile file options
-// we should really allow users to map any key to a joystick button
-
 // #define JOY_INVERT_Y
 #define JOY_XAXIS 0
 #define JOY_YAXIS 1
-// buttons
-#define JOY_BUT_LMOUSE 0
-#define JOY_BUT_RMOUSE 2
-#define JOY_BUT_ESCAPE 3
-#define JOY_BUT_PERIOD 1
-#define JOY_BUT_SPACE 4
-#define JOY_BUT_F5 5
-#ifdef ENABLE_VKEYBD
-#define JOY_BUT_VKEYBOARD 7
-#endif
-
 
 #if SDL_VERSION_ATLEAST(2, 0, 0)
 #define GAMECONTROLLERDB_FILE "gamecontrollerdb.txt"
@@ -881,17 +867,6 @@ void SdlEventSource::closeJoystick() {
 	}
 }
 
-bool SdlEventSource::shouldGenerateMouseEvents() {
-	// Engine doesn't support joystick -> emulate mouse events
-	if (g_engine && !g_engine->hasFeature(Engine::kSupportsJoystick)) {
-		return true;
-	}
-	if (g_gui.isActive()) {
-		return true;
-	}
-	return false;
-}
-
 int SdlEventSource::mapSDLJoystickButtonToOSystem(Uint8 sdlButton) {
 	Common::JoystickButton osystemButtons[] = {
 	    Common::JOYSTICK_BUTTON_A,
@@ -914,91 +889,27 @@ int SdlEventSource::mapSDLJoystickButtonToOSystem(Uint8 sdlButton) {
 }
 
 bool SdlEventSource::handleJoyButtonDown(SDL_Event &ev, Common::Event &event) {
-	if (!shouldGenerateMouseEvents()) {
-		event.type = Common::EVENT_JOYBUTTON_DOWN;
-		event.joystick.button = mapSDLJoystickButtonToOSystem(ev.jbutton.button);
-		return true;
+	int button = mapSDLJoystickButtonToOSystem(ev.jbutton.button);
+	if (button < 0) {
+		return false;
 	}
 
-	if (ev.jbutton.button == JOY_BUT_LMOUSE) {
-		event.type = Common::EVENT_LBUTTONDOWN;
-		return processMouseEvent(event, _km.x / MULTIPLIER, _km.y / MULTIPLIER);
-	} else if (ev.jbutton.button == JOY_BUT_RMOUSE) {
-		event.type = Common::EVENT_RBUTTONDOWN;
-		return processMouseEvent(event, _km.x / MULTIPLIER, _km.y / MULTIPLIER);
-	} else {
-		event.type = Common::EVENT_KEYDOWN;
-		switch (ev.jbutton.button) {
-		case JOY_BUT_ESCAPE:
-			event.kbd.keycode = Common::KEYCODE_ESCAPE;
-			event.kbd.ascii = mapKey(SDLK_ESCAPE, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-		case JOY_BUT_PERIOD:
-			event.kbd.keycode = Common::KEYCODE_PERIOD;
-			event.kbd.ascii = mapKey(SDLK_PERIOD, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-		case JOY_BUT_SPACE:
-			event.kbd.keycode = Common::KEYCODE_SPACE;
-			event.kbd.ascii = mapKey(SDLK_SPACE, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-		case JOY_BUT_F5:
-			event.kbd.keycode = Common::KEYCODE_F5;
-			event.kbd.ascii = mapKey(SDLK_F5, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-#ifdef ENABLE_VKEYBD
-		case JOY_BUT_VKEYBOARD: // Toggles virtual keyboard
-			event.type = Common::EVENT_VIRTUAL_KEYBOARD;
-			break;
-#endif
-		default:
-			break;
-		}
-		return true;
-	}
+	event.type = Common::EVENT_JOYBUTTON_DOWN;
+	event.joystick.button = button;
+
+	return true;
 }
 
 bool SdlEventSource::handleJoyButtonUp(SDL_Event &ev, Common::Event &event) {
-	if (!shouldGenerateMouseEvents()) {
-		event.type = Common::EVENT_JOYBUTTON_UP;
-		event.joystick.button = mapSDLJoystickButtonToOSystem(ev.jbutton.button);
-		return true;
+	int button = mapSDLJoystickButtonToOSystem(ev.jbutton.button);
+	if (button < 0) {
+		return false;
 	}
 
-	if (ev.jbutton.button == JOY_BUT_LMOUSE) {
-		event.type = Common::EVENT_LBUTTONUP;
-		return processMouseEvent(event, _km.x / MULTIPLIER, _km.y / MULTIPLIER);
-	} else if (ev.jbutton.button == JOY_BUT_RMOUSE) {
-		event.type = Common::EVENT_RBUTTONUP;
-		return processMouseEvent(event, _km.x / MULTIPLIER, _km.y / MULTIPLIER);
-	} else {
-		event.type = Common::EVENT_KEYUP;
-		switch (ev.jbutton.button) {
-		case JOY_BUT_ESCAPE:
-			event.kbd.keycode = Common::KEYCODE_ESCAPE;
-			event.kbd.ascii = mapKey(SDLK_ESCAPE, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-		case JOY_BUT_PERIOD:
-			event.kbd.keycode = Common::KEYCODE_PERIOD;
-			event.kbd.ascii = mapKey(SDLK_PERIOD, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-		case JOY_BUT_SPACE:
-			event.kbd.keycode = Common::KEYCODE_SPACE;
-			event.kbd.ascii = mapKey(SDLK_SPACE, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-		case JOY_BUT_F5:
-			event.kbd.keycode = Common::KEYCODE_F5;
-			event.kbd.ascii = mapKey(SDLK_F5, (SDLMod)ev.key.keysym.mod, 0);
-			break;
-#ifdef ENABLE_VKEYBD
-		case JOY_BUT_VKEYBOARD: // Toggles virtual keyboard
-			// Handled in key down
-			break;
-#endif
-		default:
-			break;
-		}
-		return true;
-	}
+	event.type = Common::EVENT_JOYBUTTON_UP;
+	event.joystick.button = button;
+
+	return true;
 }
 
 bool SdlEventSource::handleJoyAxisMotion(SDL_Event &ev, Common::Event &event) {
@@ -1036,9 +947,6 @@ bool SdlEventSource::handleJoyAxisMotion(SDL_Event &ev, Common::Event &event) {
 	}
 
 bool SdlEventSource::handleJoyHatMotion(SDL_Event &ev, Common::Event &event) {
-	if (shouldGenerateMouseEvents())
-		return false;
-
 	event.type = Common::EVENT_JOYBUTTON_UP;
 	HANDLE_HAT_UP(ev.jhat.value, _lastHatPosition, SDL_HAT_UP, Common::JOYSTICK_BUTTON_DPAD_UP)
 	HANDLE_HAT_UP(ev.jhat.value, _lastHatPosition, SDL_HAT_DOWN, Common::JOYSTICK_BUTTON_DPAD_DOWN)
@@ -1121,102 +1029,20 @@ int SdlEventSource::mapSDLControllerButtonToOSystem(Uint8 sdlButton) {
 }
 
 bool SdlEventSource::handleControllerButton(const SDL_Event &ev, Common::Event &event, bool buttonUp) {
-	using namespace Common;
+	int button = mapSDLControllerButtonToOSystem(ev.cbutton.button);
 
-	struct ControllerEventMapping {
-		EventType normalType;
-		KeyState normalKeystate;
-		EventType modifierType;
-		KeyState modifierKeystate;
-	};
-
-	static const ControllerEventMapping mapping[] = {
-			// SDL_CONTROLLER_BUTTON_A: Left mouse button
-			{ EVENT_LBUTTONDOWN, KeyState(), EVENT_LBUTTONDOWN, KeyState() },
-			// SDL_CONTROLLER_BUTTON_B: Right mouse button
-			{ EVENT_RBUTTONDOWN, KeyState(), EVENT_RBUTTONDOWN, KeyState() },
-			// SDL_CONTROLLER_BUTTON_X: Period (+R_trigger: Space)
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_PERIOD, '.'), EVENT_KEYDOWN, KeyState(KEYCODE_SPACE, ASCII_SPACE) },
-			// SDL_CONTROLLER_BUTTON_Y: Escape (+R_trigger: Return)
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_ESCAPE, ASCII_ESCAPE), EVENT_KEYDOWN, KeyState(KEYCODE_RETURN, ASCII_RETURN) },
-			// SDL_CONTROLLER_BUTTON_BACK: Virtual keyboard (+R_trigger: Predictive Input Dialog)
-#ifdef ENABLE_VKEYBD
-			{ EVENT_VIRTUAL_KEYBOARD, KeyState(), EVENT_PREDICTIVE_DIALOG, KeyState() },
-#else
-			{ EVENT_INVALID, KeyState(), EVENT_PREDICTIVE_DIALOG, KeyState() },
-#endif
-			// SDL_CONTROLLER_BUTTON_GUIDE: Unmapped
-			{ EVENT_INVALID, KeyState(), EVENT_INVALID, KeyState() },
-			// SDL_CONTROLLER_BUTTON_START: ScummVM in game menu
-			{ EVENT_MAINMENU, KeyState(), EVENT_MAINMENU, KeyState() },
-			// SDL_CONTROLLER_BUTTON_LEFTSTICK: Unmapped
-			{ EVENT_INVALID, KeyState(), EVENT_INVALID, KeyState() },
-			// SDL_CONTROLLER_BUTTON_RIGHTSTICK: Unmapped
-			{ EVENT_INVALID, KeyState(), EVENT_INVALID, KeyState() },
-			// SDL_CONTROLLER_BUTTON_LEFTSHOULDER: Game menu
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_F5, ASCII_F5), EVENT_KEYDOWN, KeyState(KEYCODE_F5, ASCII_F5) },
-			// SDL_CONTROLLER_BUTTON_RIGHTSHOULDER: Modifier + Shift
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_INVALID, 0, KBD_SHIFT), EVENT_KEYDOWN, KeyState(KEYCODE_INVALID, 0, 0) },
-			// SDL_CONTROLLER_BUTTON_DPAD_UP: Up (+R_trigger: Up+Right)
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_KP8, 0), EVENT_KEYDOWN, KeyState(KEYCODE_KP9, 0) },
-			// SDL_CONTROLLER_BUTTON_DPAD_DOWN: Down (+R_trigger: Down+Left)
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_KP2, 0), EVENT_KEYDOWN, KeyState(KEYCODE_KP1, 0) },
-			// SDL_CONTROLLER_BUTTON_DPAD_LEFT: Left (+R_trigger: Up+Left)
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_KP4, 0), EVENT_KEYDOWN, KeyState(KEYCODE_KP7, 0) },
-			// SDL_CONTROLLER_BUTTON_DPAD_RIGHT: Right (+R_trigger: Down+Right)
-			{ EVENT_KEYDOWN, KeyState(KEYCODE_KP6, 0), EVENT_KEYDOWN, KeyState(KEYCODE_KP3, 0) }
-	};
-
-	if (!shouldGenerateMouseEvents()) {
-		event.type = buttonUp ? Common::EVENT_JOYBUTTON_UP : Common::EVENT_JOYBUTTON_DOWN;
-		event.joystick.button = mapSDLControllerButtonToOSystem(ev.cbutton.button);
-		if (event.joystick.button == -1)
-				return false;
-
-		return true;
-	}
-
-	if (ev.cbutton.button > SDL_CONTROLLER_BUTTON_DPAD_RIGHT) {
-		warning("Unknown SDL controller button: '%d'", ev.cbutton.button);
+	if (button < 0)
 		return false;
-	}
 
-	if (!_km.modifier) {
-		event.type = mapping[ev.cbutton.button].normalType;
-		event.kbd = mapping[ev.cbutton.button].normalKeystate;
-	} else {
-		event.type = mapping[ev.cbutton.button].modifierType;
-		event.kbd = mapping[ev.cbutton.button].modifierKeystate;
-	}
+	event.type = buttonUp ? Common::EVENT_JOYBUTTON_UP : Common::EVENT_JOYBUTTON_DOWN;
+	event.joystick.button = button;
 
-	// Setting the mouse speed modifier after filling the event structure above
-	// ensures that the shift key events are correctly handled
-	if (ev.cbutton.button == SDL_CONTROLLER_BUTTON_RIGHTSHOULDER) {
-		// Right shoulder is the modifier button that makes the mouse go slower
-		// and allows access to an extended layout while pressed.
+	if (event.joystick.button == Common::JOYSTICK_BUTTON_RIGHT_SHOULDER) {
+		// Right shoulder is the modifier button that makes the mouse go slower.
 		_km.modifier = !buttonUp;
 	}
 
-	if (event.type == EVENT_LBUTTONDOWN || event.type == EVENT_RBUTTONDOWN) {
-		processMouseEvent(event, _km.x / MULTIPLIER, _km.y / MULTIPLIER);
-	}
-
-	if (buttonUp) {
-		// The event mapping table is for button down events. If we received a button up event,
-		// transform the event type to the corresponding up type.
-		if (event.type == EVENT_KEYDOWN) {
-			event.type = EVENT_KEYUP;
-		} else if (event.type == EVENT_LBUTTONDOWN) {
-			event.type = EVENT_LBUTTONUP;
-		} else if (event.type == EVENT_RBUTTONDOWN) {
-			event.type = EVENT_RBUTTONUP;
-		} else {
-			// Handled in key down
-			event.type = EVENT_INVALID;
-		}
-	}
-
-	return event.type != EVENT_INVALID;
+	return true;
 }
 
 bool SdlEventSource::handleControllerAxisMotion(const SDL_Event &ev, Common::Event &event) {
diff --git a/backends/events/sdl/sdl-events.h b/backends/events/sdl/sdl-events.h
index bf17cc918fc..f1f2bdd645a 100644
--- a/backends/events/sdl/sdl-events.h
+++ b/backends/events/sdl/sdl-events.h
@@ -155,8 +155,6 @@ protected:
 	virtual bool handleControllerAxisMotion(const SDL_Event &ev, Common::Event &event);
 #endif
 
-	bool shouldGenerateMouseEvents();
-
 	//@}
 
 	/**
diff --git a/backends/keymapper/hardware-input.cpp b/backends/keymapper/hardware-input.cpp
index 42041b13ead..195910b3e17 100644
--- a/backends/keymapper/hardware-input.cpp
+++ b/backends/keymapper/hardware-input.cpp
@@ -23,7 +23,9 @@
 #include "backends/keymapper/hardware-input.h"
 
 #include "backends/keymapper/keymapper.h"
+
 #include "common/tokenizer.h"
+#include "common/translation.h"
 
 namespace Common {
 
@@ -215,6 +217,25 @@ const ModifierTableEntry defaultModifiers[] = {
 	{ 0,     nullptr, nullptr }
 };
 
+const HardwareInputTableEntry defaultJoystickButtons[] = {
+    { "JOY_A",              JOYSTICK_BUTTON_A,              _s("Joy A")              },
+    { "JOY_B",              JOYSTICK_BUTTON_B,              _s("Joy B")              },
+    { "JOY_X",              JOYSTICK_BUTTON_X,              _s("Joy X")              },
+    { "JOY_Y",              JOYSTICK_BUTTON_Y,              _s("Joy Y")              },
+    { "JOY_BACK",           JOYSTICK_BUTTON_BACK,           _s("Joy Back")           },
+    { "JOY_GUIDE",          JOYSTICK_BUTTON_GUIDE,          _s("Joy Guide")          },
+    { "JOY_START",          JOYSTICK_BUTTON_START,          _s("Joy Start")          },
+    { "JOY_LEFT_STICK",     JOYSTICK_BUTTON_LEFT_STICK,     _s("Joy Left Stick")     },
+    { "JOY_RIGHT_STICK",    JOYSTICK_BUTTON_RIGHT_STICK,    _s("Joy Right Stick")    },
+    { "JOY_LEFT_SHOULDER",  JOYSTICK_BUTTON_LEFT_SHOULDER,  _s("Joy Left Shoulder")  },
+    { "JOY_RIGHT_SHOULDER", JOYSTICK_BUTTON_RIGHT_SHOULDER, _s("Joy Right Shoulder") },
+    { "JOY_UP",             JOYSTICK_BUTTON_DPAD_UP,        _s("Joy D-pad Up")       },
+    { "JOY_DOWN",           JOYSTICK_BUTTON_DPAD_DOWN,      _s("Joy D-pad Down")     },
+    { "JOY_LEFT",           JOYSTICK_BUTTON_DPAD_LEFT,      _s("Joy D-pad Left")     },
+    { "JOY_RIGHT",          JOYSTICK_BUTTON_DPAD_RIGHT,     _s("Joy D-pad Right")    },
+    { nullptr,              0,                              nullptr                 }
+};
+
 HardwareInputSet::~HardwareInputSet() {
 }
 
@@ -269,7 +290,7 @@ HardwareInput KeyboardHardwareInputSet::findHardwareInput(const String &id) cons
 	}
 
 	const KeyState keystate = KeyState(key->keycode, 0, modifierFlags);
-	return HardwareInput(id, keystate, fullKeyDesc + key->desc);
+	return HardwareInput::createKeyboard(id, keystate, fullKeyDesc + key->desc);
 }
 
 HardwareInput KeyboardHardwareInputSet::findHardwareInput(const Event &event) const {
@@ -301,7 +322,48 @@ HardwareInput KeyboardHardwareInputSet::findHardwareInput(const Event &event) co
 		}
 
 		const KeyState keystate = KeyState(key->keycode, 0, modifierFlags);
-		return HardwareInput(id + key->hwId, keystate, fullKeyDesc + key->desc);
+		return HardwareInput::createKeyboard(id + key->hwId, keystate, fullKeyDesc + key->desc);
+	}
+	default:
+		return HardwareInput();
+	}
+}
+
+JoystickHardwareInputSet::JoystickHardwareInputSet(const HardwareInputTableEntry *buttonEntries) :
+    _buttonEntries(buttonEntries) {
+}
+
+HardwareInput JoystickHardwareInputSet::findHardwareInput(const String &id) const {
+	const HardwareInputTableEntry *hw = nullptr;
+	for (hw = _buttonEntries;  hw->hwId; hw++) {
+		if (id.equals(hw->hwId)) {
+			break;
+		}
+	}
+
+	if (!hw || !hw->hwId) {
+		return HardwareInput();
+	}
+
+	return HardwareInput::createJoystick(hw->hwId, hw->code, hw->desc);
+}
+
+HardwareInput JoystickHardwareInputSet::findHardwareInput(const Event &event) const {
+	switch (event.type) {
+	case EVENT_JOYBUTTON_DOWN:
+	case EVENT_JOYBUTTON_UP: {
+		const HardwareInputTableEntry *hw = nullptr;
+		for (hw = _buttonEntries;  hw->hwId; hw++) {
+			if (event.joystick.button == hw->code) {
+				break;
+			}
+		}
+
+		if (!hw || !hw->hwId) {
+			return HardwareInput();
+		}
+
+		return HardwareInput::createJoystick(hw->hwId, hw->code, hw->desc);
 	}
 	default:
 		return HardwareInput();
@@ -325,7 +387,7 @@ HardwareInput CustomHardwareInputSet::findHardwareInput(const String &id) const
 		return HardwareInput();
 	}
 
-	return HardwareInput(hw->hwId, hw->code, hw->desc);
+	return HardwareInput::createCustom(hw->hwId, hw->code, hw->desc);
 }
 
 HardwareInput CustomHardwareInputSet::findHardwareInput(const Event &event) const {
@@ -342,7 +404,7 @@ HardwareInput CustomHardwareInputSet::findHardwareInput(const Event &event) cons
 			return HardwareInput();
 		}
 
-		return HardwareInput(hw->hwId, hw->code, hw->desc);
+		return HardwareInput::createCustom(hw->hwId, hw->code, hw->desc);
 	}
 	default:
 		return HardwareInput();
@@ -377,4 +439,8 @@ HardwareInput CompositeHardwareInputSet::findHardwareInput(const Event &event) c
 	return HardwareInput();
 }
 
+void CompositeHardwareInputSet::addHardwareInputSet(HardwareInputSet *hardwareInputSet) {
+	_inputSets.push_back(hardwareInputSet);
+}
+
 } //namespace Common
diff --git a/backends/keymapper/hardware-input.h b/backends/keymapper/hardware-input.h
index d2055dc5dfc..d6e3f7d6304 100644
--- a/backends/keymapper/hardware-input.h
+++ b/backends/keymapper/hardware-input.h
@@ -38,9 +38,11 @@ enum HardwareInputType {
 	/** Empty / invalid input type */
 	kHardwareInputTypeInvalid,
 	/** Input that sends single events */
-	kHardwareInputTypeGeneric,
-	/** Input that usually send -up and -down events */
-	kHardwareInputTypeKeyboard
+	kHardwareInputTypeCustom,
+	/** Keyboard input that sends -up and -down events */
+	kHardwareInputTypeKeyboard,
+	/** Joystick input that sends -up and -down events */
+	kHardwareInputTypeJoystick
 };
 
 /**
@@ -73,11 +75,33 @@ struct HardwareInput {
 	HardwareInput()
 		: inputCode(0), type(kHardwareInputTypeInvalid) { }
 
-	HardwareInput(const String &i, HardwareInputCode ic, const String &desc)
-		: id(i), inputCode(ic), description(desc), type(kHardwareInputTypeGeneric) { }
+	static HardwareInput createCustom(const String &i, HardwareInputCode ic, const String &desc) {
+		HardwareInput hardwareInput;
+		hardwareInput.id = i;
+		hardwareInput.description = desc;
+		hardwareInput.type = kHardwareInputTypeCustom;
+		hardwareInput.inputCode = ic;
+		return hardwareInput;
+	}
 
-	HardwareInput(const String &i, KeyState ky, const String &desc)
-		: id(i), inputCode(0), key(ky), description(desc), type(kHardwareInputTypeKeyboard) { }
+	static HardwareInput createKeyboard(const String &i, KeyState ky, const String &desc) {
+		HardwareInput hardwareInput;
+		hardwareInput.id = i;
+		hardwareInput.description = desc;
+		hardwareInput.type = kHardwareInputTypeKeyboard;
+		hardwareInput.inputCode = 0;
+		hardwareInput.key = ky;
+		return hardwareInput;
+	}
+
+	static HardwareInput createJoystick(const String &i, uint8 button, const String &desc) {
+		HardwareInput hardwareInput;
+		hardwareInput.id = i;
+		hardwareInput.description = desc;
+		hardwareInput.type = kHardwareInputTypeJoystick;
+		hardwareInput.inputCode = button;
+		return hardwareInput;
+	}
 };
 
 /**
@@ -152,6 +176,21 @@ private:
 	const ModifierTableEntry *_modifiers;
 };
 
+/**
+ * A joystick input device
+ */
+class JoystickHardwareInputSet : public HardwareInputSet {
+public:
+	JoystickHardwareInputSet(const HardwareInputTableEntry *buttonEntries);
+
+	// HardwareInputSet API
+	HardwareInput findHardwareInput(const String &id) const override;
+	HardwareInput findHardwareInput(const Event &event) const override;
+
+private:
+	const HardwareInputTableEntry *_buttonEntries;
+};
+
 /**
  * A custom backend input device
  *
@@ -197,6 +236,9 @@ extern const KeyTableEntry defaultKeys[];
 /** A standard set of keyboard modifiers */
 extern const ModifierTableEntry defaultModifiers[];
 
+/** A standard set of joystick buttons based on the ScummVM event model */
+extern const HardwareInputTableEntry defaultJoystickButtons[];
+
 } // End of namespace Common
 
 #endif // #ifndef COMMON_HARDWARE_KEY_H
diff --git a/backends/keymapper/input-watcher.cpp b/backends/keymapper/input-watcher.cpp
index 90b0218dfa4..236cfdce1e6 100644
--- a/backends/keymapper/input-watcher.cpp
+++ b/backends/keymapper/input-watcher.cpp
@@ -59,8 +59,10 @@ bool InputWatcher::notifyEvent(const Event &event) {
 
 	switch (event.type) {
 		case EVENT_KEYDOWN:
+		case EVENT_JOYBUTTON_DOWN:
 			return true;
 		case EVENT_KEYUP:
+		case EVENT_JOYBUTTON_UP:
 		case EVENT_CUSTOM_BACKEND_HARDWARE:
 			_hwInput = _keymapper->findHardwareInput(event);
 			if (_hwInput.type != kHardwareInputTypeInvalid) {
diff --git a/backends/keymapper/keymap.cpp b/backends/keymapper/keymap.cpp
index 5a306dbff11..228522e1be7 100644
--- a/backends/keymapper/keymap.cpp
+++ b/backends/keymapper/keymap.cpp
@@ -115,11 +115,16 @@ Keymap::ActionArray Keymap::getMappedActions(const Event &event) const {
 	switch (event.type) {
 	case EVENT_KEYDOWN:
 	case EVENT_KEYUP: {
-		HardwareInput hardwareInput("", event.kbd, "");
+		HardwareInput hardwareInput = HardwareInput::createKeyboard("", event.kbd, "");
+		return _hwActionMap[hardwareInput];
+	}
+	case EVENT_JOYBUTTON_DOWN:
+	case EVENT_JOYBUTTON_UP: {
+		HardwareInput hardwareInput = HardwareInput::createJoystick("", event.joystick.button, "");
 		return _hwActionMap[hardwareInput];
 	}
 	case EVENT_CUSTOM_BACKEND_HARDWARE: {
-		HardwareInput hardwareInput("", event.customType, "");
+		HardwareInput hardwareInput = HardwareInput::createCustom("", event.customType, "");
 		return _hwActionMap[hardwareInput];
 	}
 	default:
diff --git a/backends/keymapper/keymapper.cpp b/backends/keymapper/keymapper.cpp
index 46202c6e206..5214d0900c2 100644
--- a/backends/keymapper/keymapper.cpp
+++ b/backends/keymapper/keymapper.cpp
@@ -191,7 +191,7 @@ List<Event> Keymapper::mapEvent(const Event &ev) {
 Keymapper::IncomingEventType Keymapper::convertToIncomingEventType(const Event &ev) const {
 	if (ev.type == EVENT_CUSTOM_BACKEND_HARDWARE) {
 		return kIncomingEventInstant;
-	} else if (ev.type == EVENT_KEYDOWN) {
+	} else if (ev.type == EVENT_KEYDOWN || ev.type == EVENT_JOYBUTTON_DOWN) {
 		return kIncomingEventStart;
 	} else {
 		return kIncomingEventEnd;
@@ -242,6 +242,9 @@ EventType Keymapper::convertStartToEnd(EventType type) {
 	case EVENT_MBUTTONDOWN:
 		result = EVENT_MBUTTONUP;
 		break;
+	case EVENT_JOYBUTTON_DOWN:
+		result = EVENT_JOYBUTTON_UP;
+		break;
 	case EVENT_CUSTOM_BACKEND_ACTION_START:
 		result = EVENT_CUSTOM_BACKEND_ACTION_END;
 		break;
@@ -282,12 +285,6 @@ void Keymapper::hardcodedEventMapping(Event ev) {
 		}
 	}
 #endif
-
-	if (ev.type == EVENT_JOYBUTTON_DOWN) {
-		if (ev.joystick.button == JOYSTICK_BUTTON_START || ev.joystick.button == JOYSTICK_BUTTON_GUIDE) {
-			ev.type = EVENT_MAINMENU;
-		}
-	}
 }
 
 void DelayedEventSource::scheduleEvent(const Event &ev, uint32 delayMillis) {
diff --git a/backends/keymapper/standard-actions.cpp b/backends/keymapper/standard-actions.cpp
index 006a8e5403c..21b0d625dc5 100644
--- a/backends/keymapper/standard-actions.cpp
+++ b/backends/keymapper/standard-actions.cpp
@@ -24,7 +24,7 @@
 
 namespace Common {
 
-const char *kStandardActionInteract      = "INTCT";
+const char *kStandardActionInteract      = "INTRCT";
 const char *kStandardActionSkip          = "SKIP";
 const char *kStandardActionPause         = "PAUSE";
 const char *kStandardActionMoveForward   = "FWD";
diff --git a/backends/platform/maemo/maemo.cpp b/backends/platform/maemo/maemo.cpp
index 517b3ce48fa..ef44b589bc7 100644
--- a/backends/platform/maemo/maemo.cpp
+++ b/backends/platform/maemo/maemo.cpp
@@ -184,9 +184,9 @@ static const Common::KeyTableEntry maemoKeys[] = {
 };
 
 Common::HardwareInputSet *OSystem_SDL_Maemo::getHardwareInputSet() {
-	Common::CompositeHardwareInputSet inputSet = new Common::CompositeHardwareInputSet();
-	inputSet->addHardwareInputSet(new Common::KeyboardHardwareInputSet(maemoKeys, defaultModifiers));
-	inputSet->addHardwareInputSet(new Common::KeyboardHardwareInputSet(defaultKeys, defaultModifiers));
+	Common::CompositeHardwareInputSet *inputSet = new Common::CompositeHardwareInputSet();
+	inputSet->addHardwareInputSet(new Common::KeyboardHardwareInputSet(maemoKeys, Common::defaultModifiers));
+	inputSet->addHardwareInputSet(new Common::KeyboardHardwareInputSet(Common::defaultKeys, Common::defaultModifiers));
 
 	return inputSet;
 }
diff --git a/backends/platform/sdl/sdl.cpp b/backends/platform/sdl/sdl.cpp
index 9ed28ea2e89..309d977419c 100644
--- a/backends/platform/sdl/sdl.cpp
+++ b/backends/platform/sdl/sdl.cpp
@@ -41,6 +41,7 @@
 
 #include "backends/events/default/default-events.h"
 #include "backends/events/sdl/sdl-events.h"
+#include "backends/keymapper/hardware-input.h"
 #include "backends/mutex/sdl/sdl-mutex.h"
 #include "backends/timer/sdl/sdl-timer.h"
 #include "backends/graphics/surfacesdl/surfacesdl-graphics.h"
@@ -398,6 +399,20 @@ Common::KeymapArray OSystem_SDL::getGlobalKeymaps() {
 	return globalMaps;
 }
 
+Common::HardwareInputSet *OSystem_SDL::getHardwareInputSet() {
+	using namespace Common;
+
+	CompositeHardwareInputSet *inputSet = new CompositeHardwareInputSet();
+	inputSet->addHardwareInputSet(new KeyboardHardwareInputSet(defaultKeys, defaultModifiers));
+
+	bool joystickSupportEnabled = ConfMan.getInt("joystick_num") >= 0;
+	if (joystickSupportEnabled) {
+		inputSet->addHardwareInputSet(new JoystickHardwareInputSet(defaultJoystickButtons));
+	}
+
+	return inputSet;
+}
+
 void OSystem_SDL::logMessage(LogMessageType::Type type, const char *message) {
 	// First log to stdout/stderr
 	FILE *output = 0;
diff --git a/backends/platform/sdl/sdl.h b/backends/platform/sdl/sdl.h
index bd7b5386f5b..1cee82699f6 100644
--- a/backends/platform/sdl/sdl.h
+++ b/backends/platform/sdl/sdl.h
@@ -64,6 +64,7 @@ public:
 	virtual void quit();
 	virtual void fatalError();
 	Common::KeymapArray getGlobalKeymaps() override;
+	Common::HardwareInputSet *getHardwareInputSet() override;
 
 	// Logging
 	virtual void logMessage(LogMessageType::Type type, const char *message);
diff --git a/common/events.h b/common/events.h
index 3e4d046e7e8..fdf4f015791 100644
--- a/common/events.h
+++ b/common/events.h
@@ -111,7 +111,7 @@ struct JoystickState {
 	 * Some of the button indices match well-known game controller
 	 * buttons. See JoystickButton.
 	 */
-	int8 button;
+	uint8 button;
 
 	JoystickState() : axis(0), position(0), button(0) {}
 };
diff --git a/engines/engine.h b/engines/engine.h
index db8cc021671..6f2c7fda43e 100644
--- a/engines/engine.h
+++ b/engines/engine.h
@@ -132,15 +132,7 @@ public:
 		 * If this feature is supported, then the corresponding MetaEngine *must*
 		 * support the kSupportsListSaves feature.
 		 */
-		kSupportsSavingDuringRuntime,
-
-		/**
-		 * Engine must receive joystick events because the game uses them.
-		 * For engines which have not this feature, joystick events are converted
-		 * to mouse events.
-		 */
-		kSupportsJoystick
-
+		kSupportsSavingDuringRuntime
 	};
 
 
diff --git a/engines/metaengine.cpp b/engines/metaengine.cpp
index 060e4d599c5..8be0ffa7930 100644
--- a/engines/metaengine.cpp
+++ b/engines/metaengine.cpp
@@ -50,27 +50,13 @@ const char *MetaEngine::getSavegamePattern(const char *target) const {
 Common::Keymap *MetaEngine::initKeymap(const char *target) const {
 	using namespace Common;
 
-	Keymap *const engineKeyMap = new Keymap(Keymap::kKeymapTypeGame, "engine-default");
-
-	// Since the game has multiple built-in keys for each of these anyway,
-	// this just attempts to remap one of them.
-	const KeyActionEntry keyActionEntries[] = {
-		{ "PAUS", KeyState(KEYCODE_SPACE, ' ', 0),           "SPACE",  _("Pause")     },
-		{ "SKCT", KeyState(KEYCODE_ESCAPE, ASCII_ESCAPE, 0), "ESCAPE", _("Skip")      },
-		{ "SKLI", KeyState(KEYCODE_PERIOD, '.', 0),          "PERIOD", _("Skip line") }
-	};
-
-	for (uint i = 0; i < ARRAYSIZE(keyActionEntries); i++) {
-		Action *const act = new Action(keyActionEntries[i].id, keyActionEntries[i].description);
-		act->setKeyEvent(keyActionEntries[i].ks);
-		act->addDefaultInputMapping(keyActionEntries[i].defaultHwId);
-		engineKeyMap->addAction(act);
-	}
+	Keymap *engineKeyMap = new Keymap(Keymap::kKeymapTypeGame, "engine-default");
 
 	Action *act;
 
 	act = new Action("LCLK", _("Left Click"));
 	act->setLeftClickEvent();
+	act->addDefaultInputMapping("JOY_A");
 	engineKeyMap->addAction(act);
 
 	act = new Action("MCLK", _("Middle Click"));
@@ -79,6 +65,57 @@ Common::Keymap *MetaEngine::initKeymap(const char *target) const {
 
 	act = new Action("RCLK", _("Right Click"));
 	act->setRightClickEvent();
+	act->addDefaultInputMapping("JOY_B");
+	engineKeyMap->addAction(act);
+
+	act = new Action("PAUS", _("Pause"));
+	act->setKeyEvent(KeyState(KEYCODE_SPACE, ' '));
+	act->addDefaultInputMapping("SPACE");
+	engineKeyMap->addAction(act);
+
+	act = new Action("MNU", _("Game menu"));
+	act->setKeyEvent(KeyState(KEYCODE_F5, ASCII_F5));
+	act->addDefaultInputMapping("F5");
+	act->addDefaultInputMapping("JOY_LEFT_SHOULDER");
+	engineKeyMap->addAction(act);
+
+	act = new Action("SKCT", _("Skip"));
+	act->setKeyEvent(KeyState(KEYCODE_ESCAPE, ASCII_ESCAPE));
+	act->addDefaultInputMapping("ESCAPE");
+	engineKeyMap->addAction(act);
+
+	act = new Action("SKLI", _("Skip line"));
+	act->setKeyEvent(KeyState(KEYCODE_PERIOD, '.'));
+	act->addDefaultInputMapping("PERIOD");
+	engineKeyMap->addAction(act);
+
+	act = new Action("PIND", _("Predictive input dialog"));
+	act->setEvent(EVENT_PREDICTIVE_DIALOG);
+	engineKeyMap->addAction(act);
+
+	act = new Action("RETURN", _("Confirm"));
+	act->setKeyEvent(KeyState(KEYCODE_RETURN, ASCII_RETURN));
+	act->addDefaultInputMapping("RETURN");
+	engineKeyMap->addAction(act);
+
+	act = new Action("UP", _("Up"));
+	act->setKeyEvent(KEYCODE_KP8);
+	act->addDefaultInputMapping("JOY_UP");
+	engineKeyMap->addAction(act);
+
+	act = new Action("DOWN", _("Down"));
+	act->setKeyEvent(KEYCODE_KP2);
+	act->addDefaultInputMapping("JOY_DOWN");
+	engineKeyMap->addAction(act);
+
+	act = new Action("LEFT", _("Left"));
+	act->setKeyEvent(KEYCODE_KP4);
+	act->addDefaultInputMapping("JOY_LEFT");
+	engineKeyMap->addAction(act);
+
+	act = new Action("RIGHT", _("Right"));
+	act->setKeyEvent(KEYCODE_KP6);
+	act->addDefaultInputMapping("JOY_RIGHT");
 	engineKeyMap->addAction(act);
 
 	return engineKeyMap;
diff --git a/engines/pegasus/detection.cpp b/engines/pegasus/detection.cpp
index 2a23a3cd474..5d723d448ed 100644
--- a/engines/pegasus/detection.cpp
+++ b/engines/pegasus/detection.cpp
@@ -43,8 +43,7 @@ bool PegasusEngine::hasFeature(EngineFeature f) const {
 	return
 		(f == kSupportsRTL)
 		|| (f == kSupportsLoadingDuringRuntime)
-		|| (f == kSupportsSavingDuringRuntime)
-		|| (f == kSupportsJoystick);
+		|| (f == kSupportsSavingDuringRuntime);
 }
 
 bool PegasusEngine::isDemo() const {
diff --git a/engines/pegasus/input.cpp b/engines/pegasus/input.cpp
index 42edf354b5c..bac3c2fd4e2 100644
--- a/engines/pegasus/input.cpp
+++ b/engines/pegasus/input.cpp
@@ -136,37 +136,6 @@ void InputDeviceManager::waitInput(const InputBits filter) {
 	}
 }
 
-PegasusAction InputDeviceManager::convertJoystickToKey(uint joybutton) {
-	switch (joybutton) {
-	case Common::JOYSTICK_BUTTON_A:
-		return kPegasusActionInteract;
-	case Common::JOYSTICK_BUTTON_B:
-		// nothing
-		break;
-	case Common::JOYSTICK_BUTTON_X:
-		return kPegasusActionShowInfoScreen;
-	case Common::JOYSTICK_BUTTON_Y:
-		return kPegasusActionToggleCenterDisplay;
-	case Common::JOYSTICK_BUTTON_LEFT_SHOULDER:
-		return kPegasusActionShowInventory;
-	case Common::JOYSTICK_BUTTON_RIGHT_SHOULDER:
-		return kPegasusActionShowBiochip;
-	case Common::JOYSTICK_BUTTON_BACK:
-		return kPegasusActionShowPauseMenu;
-	case Common::JOYSTICK_BUTTON_DPAD_UP:
-		return kPegasusActionUp;
-	case Common::JOYSTICK_BUTTON_DPAD_DOWN:
-		return kPegasusActionDown;
-	case Common::JOYSTICK_BUTTON_DPAD_LEFT:
-		return kPegasusActionLeft;
-	case Common::JOYSTICK_BUTTON_DPAD_RIGHT:
-		return kPegasusActionRight;
-	default:
-		break;
-	}
-	return kPegasusActionNone;
-}
-
 bool InputDeviceManager::notifyEvent(const Common::Event &event) {
 	if (GUI::GuiManager::instance().isActive()) {
 		// For some reason, the engine hooks in the event system using an EventObserver.
@@ -201,16 +170,6 @@ bool InputDeviceManager::notifyEvent(const Common::Event &event) {
 		if (event.customType != kPegasusActionNone && event.customType < kPegasusActionCount)
 			_keysDown[event.customType] = false;
 		break;
-	case Common::EVENT_JOYAXIS_MOTION:
-		break;
-	case Common::EVENT_JOYBUTTON_DOWN:
-		if (convertJoystickToKey(event.joystick.button) != kPegasusActionNone)
-			_keysDown[convertJoystickToKey(event.joystick.button)] = true;
-		break;
-	case Common::EVENT_JOYBUTTON_UP:
-		if (convertJoystickToKey(event.joystick.button) != kPegasusActionNone)
-			_keysDown[convertJoystickToKey(event.joystick.button)] = false;
-		break;
 	default:
 		break;
 	}
diff --git a/engines/pegasus/input.h b/engines/pegasus/input.h
index f9ec00aa2c1..b5d4d5e7325 100644
--- a/engines/pegasus/input.h
+++ b/engines/pegasus/input.h
@@ -71,8 +71,6 @@ public:
 
 	void pumpEvents();
 
-	PegasusAction convertJoystickToKey(uint joybutton);
-
 protected:
 	friend class Common::Singleton<SingletonBaseType>;
 
diff --git a/gui/gui-manager.cpp b/gui/gui-manager.cpp
index 0f150237298..9114e177617 100644
--- a/gui/gui-manager.cpp
+++ b/gui/gui-manager.cpp
@@ -33,6 +33,7 @@
 #include "backends/keymapper/action.h"
 #include "backends/keymapper/keymap.h"
 #include "backends/keymapper/keymapper.h"
+#include "backends/keymapper/standard-actions.h"
 
 #include "gui/gui-manager.h"
 #include "gui/dialog.h"
@@ -116,8 +117,14 @@ Common::Keymap *GuiManager::getKeymap() const {
 
 	Action *act;
 
+	act = new Action(Common::kStandardActionInteract, _("Interact"));
+	act->addDefaultInputMapping("JOY_A");
+	act->setLeftClickEvent();
+	guiMap->addAction(act);
+
 	act = new Action("CLOS", _("Close"));
 	act->addDefaultInputMapping("ESCAPE");
+	act->addDefaultInputMapping("JOY_Y");
 	act->setKeyEvent(KeyState(KEYCODE_ESCAPE, ASCII_ESCAPE, 0));
 	guiMap->addAction(act);