state: allow different modes for calculating consumed modifiers

The current functions dealing with consumed modifiers use the
traditional XKB definition of consumed modifiers (see description in the
added documentation). However, for several users of the library (e.g.
GTK) this definition is unsuitable or too eager. This is exacerbated by
some less-than-ideal xkeyboard-config type definitions (CTRL+ALT seems
to cause most grief...).

So, because we
- want to enable alternative interpretations, but
- don't want to expose too much internal details, and
- want to keep things simple for all library users,
we add a high-level "mode" parameter which selects the desired
interpretation. New ones can be added as long as they make some sense.

All of the old consumed-modifiers functions keep using the traditional
("XKB") mode. I mark xkb_state_mod_mask_remove_consumed() and as
deprecated without adding a *2 variant because I don't it is very useful
(or used) in practice.

Alternative modes are added in subsequent commits (this commit only adds
a mode for the existing behavior).

https://github.com/xkbcommon/libxkbcommon/issues/17

Signed-off-by: Ran Benita <ran234@gmail.com>

src/state.c

@@ -1311,13 +1311,20 @@ xkb_state_led_name_is_active(struct xkb_state *state, const char *name)
     return xkb_state_led_index_is_active(state, idx);
 }
 
+/**
+ * See:
+ * - XkbTranslateKeyCode(3), mod_rtrn return value, from libX11.
+ * - MyEnhancedXkbTranslateKeyCode(), a modification of the above, from GTK+.
+ */
 static xkb_mod_mask_t
-key_get_consumed(struct xkb_state *state, const struct xkb_key *key)
+key_get_consumed(struct xkb_state *state, const struct xkb_key *key,
+                 enum xkb_consumed_mode mode)
 {
     const struct xkb_key_type *type;
-    const struct xkb_key_type_entry *entry;
-    xkb_mod_mask_t preserve;
+    const struct xkb_key_type_entry *matching_entry;
+    xkb_mod_mask_t preserve = 0;
     xkb_layout_index_t group;
+    xkb_mod_mask_t consumed = 0;
 
     group = xkb_state_key_get_layout(state, key->keycode);
     if (group == XKB_LAYOUT_INVALID)
@@ -1325,47 +1332,40 @@ key_get_consumed(struct xkb_state *state, const struct xkb_key *key)
 
     type = key->groups[group].type;
 
-    entry = get_entry_for_key_state(state, key, group);
-    if (entry)
-        preserve = entry->preserve.mask;
-    else
-        preserve = 0;
+    matching_entry = get_entry_for_key_state(state, key, group);
+    if (matching_entry)
+        preserve = matching_entry->preserve.mask;
 
-    return type->mods.mask & ~preserve;
+    switch (mode) {
+    case XKB_CONSUMED_MODE_XKB:
+        consumed = type->mods.mask;
+        break;
+    }
+
+    return consumed & ~preserve;
 }
 
-/**
- * Tests to see if a modifier is used up by our translation of a
- * keycode to keysyms, taking note of the current modifier state and
- * the appropriate key type's preserve information, if any. This allows
- * the user to mask out the modifier in later processing of the
- * modifiers, e.g. when implementing hot keys or accelerators.
- *
- * See also, for example:
- * - XkbTranslateKeyCode(3), mod_rtrn return value, from libX11.
- * - gdk_keymap_translate_keyboard_state, consumed_modifiers return value,
- *   from gtk+.
- */
 XKB_EXPORT int
-xkb_state_mod_index_is_consumed(struct xkb_state *state, xkb_keycode_t kc,
-                                xkb_mod_index_t idx)
+xkb_state_mod_index_is_consumed2(struct xkb_state *state, xkb_keycode_t kc,
+                                 xkb_mod_index_t idx,
+                                 enum xkb_consumed_mode mode)
 {
     const struct xkb_key *key = XkbKey(state->keymap, kc);
 
     if (!key || idx >= xkb_keymap_num_mods(state->keymap))
         return -1;
 
-    return !!((1u << idx) & key_get_consumed(state, key));
+    return !!((1u << idx) & key_get_consumed(state, key, mode));
+}
+
+XKB_EXPORT int
+xkb_state_mod_index_is_consumed(struct xkb_state *state, xkb_keycode_t kc,
+                                xkb_mod_index_t idx)
+{
+    return xkb_state_mod_index_is_consumed2(state, kc, idx,
+                                            XKB_CONSUMED_MODE_XKB);
 }
 
-/**
- * Calculates which modifiers should be consumed during key processing,
- * and returns the mask with all these modifiers removed.  e.g. if
- * given a state of Alt and Shift active for a two-level alphabetic
- * key containing plus and equal on the first and second level
- * respectively, will return a mask of only Alt, as Shift has been
- * consumed by the type handling.
- */
 XKB_EXPORT xkb_mod_mask_t
 xkb_state_mod_mask_remove_consumed(struct xkb_state *state, xkb_keycode_t kc,
                                    xkb_mod_mask_t mask)
@@ -1375,16 +1375,33 @@ xkb_state_mod_mask_remove_consumed(struct xkb_state *state, xkb_keycode_t kc,
     if (!key)
         return 0;
 
-    return mask & ~key_get_consumed(state, key);
+    return mask & ~key_get_consumed(state, key, XKB_CONSUMED_MODE_XKB);
+}
+
+XKB_EXPORT xkb_mod_mask_t
+xkb_state_key_get_consumed_mods2(struct xkb_state *state, xkb_keycode_t kc,
+                                 enum xkb_consumed_mode mode)
+{
+    const struct xkb_key *key;
+
+    switch (mode) {
+    case XKB_CONSUMED_MODE_XKB:
+        break;
+    default:
+        log_err_func(state->keymap->ctx,
+                     "unrecognized consumed modifiers mode: %d\n", mode);
+        return 0;
+    }
+
+    key = XkbKey(state->keymap, kc);
+    if (!key)
+        return 0;
+
+    return key_get_consumed(state, key, mode);
 }
 
 XKB_EXPORT xkb_mod_mask_t
 xkb_state_key_get_consumed_mods(struct xkb_state *state, xkb_keycode_t kc)
 {
-    const struct xkb_key *key = XkbKey(state->keymap, kc);
-
-    if (!key)
-        return 0;
-
-    return key_get_consumed(state, key);
+    return xkb_state_key_get_consumed_mods2(state, kc, XKB_CONSUMED_MODE_XKB);
 }

test/common.c

@@ -350,7 +350,8 @@ test_compile_rules(struct xkb_context *context, const char *rules,
 void
 test_print_keycode_state(struct xkb_state *state,
                          struct xkb_compose_state *compose_state,
-                         xkb_keycode_t keycode)
+                         xkb_keycode_t keycode,
+                         enum xkb_consumed_mode consumed_mode)
 {
     struct xkb_keymap *keymap;
 
@@ -410,7 +411,8 @@ test_print_keycode_state(struct xkb_state *state,
         if (xkb_state_mod_index_is_active(state, mod,
                                           XKB_STATE_MODS_EFFECTIVE) <= 0)
             continue;
-        if (xkb_state_mod_index_is_consumed(state, keycode, mod))
+        if (xkb_state_mod_index_is_consumed2(state, keycode, mod,
+                                             consumed_mode))
             printf("-%s ", xkb_keymap_mod_get_name(keymap, mod));
         else
             printf("%s ", xkb_keymap_mod_get_name(keymap, mod));

test/interactive-evdev.c

@@ -260,7 +260,8 @@ process_event(struct keyboard *kbd, uint16_t type, uint16_t code, int32_t value)
     }
 
     if (value != KEY_STATE_RELEASE)
-        test_print_keycode_state(kbd->state, kbd->compose_state, keycode);
+        test_print_keycode_state(kbd->state, kbd->compose_state, keycode,
+                                 XKB_CONSUMED_MODE_XKB);
 
     if (with_compose) {
         status = xkb_compose_state_get_status(kbd->compose_state);

test/interactive-wayland.c

@@ -360,7 +360,8 @@ kbd_key(void *data, struct wl_keyboard *wl_kbd, uint32_t serial, uint32_t time,
 		return;
 
 	printf("%s: ", seat->name_str);
-	test_print_keycode_state(seat->state, NULL, key + 8);
+	test_print_keycode_state(seat->state, NULL, key + 8,
+                                 XKB_CONSUMED_MODE_XKB);
 
 	/* Exit on ESC. */
 	if (xkb_state_key_get_one_sym(seat->state, key + 8) == XKB_KEY_Escape)

test/interactive-x11.c

@@ -236,7 +236,8 @@ process_event(xcb_generic_event_t *gevent, struct keyboard *kbd)
         xcb_key_press_event_t *event = (xcb_key_press_event_t *) gevent;
         xkb_keycode_t keycode = event->detail;
 
-        test_print_keycode_state(kbd->state, NULL, keycode);
+        test_print_keycode_state(kbd->state, NULL, keycode,
+                                 XKB_CONSUMED_MODE_XKB);
 
         /* Exit on ESC. */
         if (keycode == 9)

test/test.h

@@ -84,7 +84,8 @@ test_compile_rules(struct xkb_context *context, const char *rules,
 void
 test_print_keycode_state(struct xkb_state *state,
                          struct xkb_compose_state *compose_state,
-                         xkb_keycode_t keycode);
+                         xkb_keycode_t keycode,
+                         enum xkb_consumed_mode consumed_mode);
 
 void
 test_print_state_changes(enum xkb_state_component changed);

xkbcommon.map

@@ -91,3 +91,9 @@ global:
 	xkb_keymap_key_get_name;
 	xkb_keymap_key_by_name;
 } V_0.5.0;
+
+V_0.7.0 {
+global:
+	xkb_state_key_get_consumed_mods2;
+	xkb_state_mod_index_is_consumed2;
+} V_0.6.0;

xkbcommon/xkbcommon.h

@@ -1649,7 +1649,7 @@ xkb_state_mod_indices_are_active(struct xkb_state *state,
  *    Effectively, this means that consumed modifiers (Shift in this example)
  *    are masked out as well, before doing the comparison.
  *
- * In summary, this is how the matching would be performed:
+ * In summary, this is approximately how the matching would be performed:
  * @code
  *   (keysym == shortcut_keysym) &&
  *   ((state_mods & ~consumed_mods & significant_mods) == shortcut_mods)
@@ -1665,16 +1665,84 @@ xkb_state_mod_indices_are_active(struct xkb_state *state,
  * @endparblock
  */
 
+/**
+ * Consumed modifiers mode.
+ *
+ * There are several possible methods for deciding which modifiers are
+ * consumed and which are not, each applicable for different systems or
+ * situations. The mode selects the method to use.
+ *
+ * Keep in mind that in all methods, the keymap may decide to "preserve"
+ * a modifier, meaning it is not reported as consumed even if it would
+ * have otherwise.
+ */
+enum xkb_consumed_mode {
+    /**
+     * This is the mode defined in the XKB specification and used by libX11.
+     *
+     * A modifier is consumed iff it *may affect* key translation.
+     *
+     * For example, if `Control+Alt+<Backspace>` produces some assigned keysym,
+     * then when pressing just `<Backspace>`, `Control` and `Alt` are consumed,
+     * even though they are not active, since if they *were* active they would
+     * have affected key translation.
+     */
+    XKB_CONSUMED_MODE_XKB
+};
+
+/**
+ * Get the mask of modifiers consumed by translating a given key.
+ *
+ * @param state The keyboard state.
+ * @param key   The keycode of the key.
+ * @param mode  The consumed modifiers mode to use; see enum description.
+ *
+ * @returns a mask of the consumed modifiers.
+ *
+ * @memberof xkb_state
+ * @since 0.7.0
+ */
+xkb_mod_mask_t
+xkb_state_key_get_consumed_mods2(struct xkb_state *state, xkb_keycode_t key,
+                                 enum xkb_consumed_mode mode);
+
+/**
+ * Same as xkb_state_key_get_consumed_mods2() with mode XKB_CONSUMED_MODE_XKB.
+ *
+ * @memberof xkb_state
+ * @since 0.4.1
+ */
+xkb_mod_mask_t
+xkb_state_key_get_consumed_mods(struct xkb_state *state, xkb_keycode_t key);
+
 /**
  * Test whether a modifier is consumed by keyboard state translation for
  * a key.
  *
+ * @param state The keyboard state.
+ * @param key   The keycode of the key.
+ * @param idx   The index of the modifier to check.
+ * @param mode  The consumed modifiers mode to use; see enum description.
+ *
  * @returns 1 if the modifier is consumed, 0 if it is not.  If the modifier
  * index is not valid in the keymap, returns -1.
  *
  * @sa xkb_state_mod_mask_remove_consumed()
  * @sa xkb_state_key_get_consumed_mods()
  * @memberof xkb_state
+ * @since 0.7.0
+ */
+int
+xkb_state_mod_index_is_consumed2(struct xkb_state *state,
+                                 xkb_keycode_t key,
+                                 xkb_mod_index_t idx,
+                                 enum xkb_consumed_mode mode);
+
+/**
+ * Same as xkb_state_mod_index_is_consumed2() with mode XKB_CONSUMED_MOD_XKB.
+ *
+ * @memberof xkb_state
+ * @since 0.4.1
  */
 int
 xkb_state_mod_index_is_consumed(struct xkb_state *state, xkb_keycode_t key,
@@ -1683,6 +1751,8 @@ xkb_state_mod_index_is_consumed(struct xkb_state *state, xkb_keycode_t key,
 /**
  * Remove consumed modifiers from a modifier mask for a key.
  *
+ * @deprecated Use xkb_state_key_get_consumed_mods2() instead.
+ *
  * Takes the given modifier mask, and removes all modifiers which are
  * consumed for that particular key (as in xkb_state_mod_index_is_consumed()).
  *
@@ -1693,18 +1763,6 @@ xkb_mod_mask_t
 xkb_state_mod_mask_remove_consumed(struct xkb_state *state, xkb_keycode_t key,
                                    xkb_mod_mask_t mask);
 
-/**
- * Get the mask of modifiers consumed by translating a given key.
- *
- * @returns a mask of the consumed modifiers.
- *
- * @sa xkb_state_mod_index_is_consumed()
- * @memberof xkb_state
- * @since 0.4.1
- */
-xkb_mod_mask_t
-xkb_state_key_get_consumed_mods(struct xkb_state *state, xkb_keycode_t key);
-
 /**
  * Test whether a layout is active in a given keyboard state by name.
  *