Extend the key binding emulator to recognize keystrokes at the root of the

tree that would leak cocoa's input handling down a garden path to selectors
that we don't handle. In other words, if a keystroke is the first keystroke
in a multi-keystroke sequence that can't possibly lead to insertText:,
recognize that and tell the caller not to pass the keystroke to
handleEvent:, but rather to handle the event immediately. This fixes a bug
where a default key bindings like this:

    "^x" = {
        "k"     = "performClose:";
        "^f"    = "openDocument:";
        "^x"    = "swapWithMark:";
        "^m"    = "selectToMark:";
    }

Does not cause ^x to get held up until another key is pressed.

Issue 3710.

sources/PTYTextView.m

@@ -1400,18 +1400,23 @@ static const int kDragThreshold = 3;
     [NSCursor setHiddenUntilMouseMoves:YES];
     NSMutableArray *eventsToHandle = [NSMutableArray array];
-    if ([_keyBindingEmulator handlesEvent:event extraEvents:eventsToHandle]) {
-        DLog(@"iTermNSKeyBindingEmulator reports that event is handled, sending to interpretKeyEvents.");
-        [self interpretKeyEvents:@[ event ]];
+    BOOL pointlessly;
+    if ([_keyBindingEmulator handlesEvent:event pointlessly:&pointlessly extraEvents:eventsToHandle]) {
+        if (!pointlessly) {
+            DLog(@"iTermNSKeyBindingEmulator reports that event is handled, sending to interpretKeyEvents.");
+            [self interpretKeyEvents:@[ event ]];
+        } else {
+            [self handleKeyDownEvent:event eschewCocoaTextHandling:YES];
+        }
         return;
     }
     [eventsToHandle addObject:event];
     for (NSEvent *event in eventsToHandle) {
-        [self handleKeyDownEvent:event];
+        [self handleKeyDownEvent:event eschewCocoaTextHandling:NO];
     }
 }
-- (void)handleKeyDownEvent:(NSEvent *)event {
+- (void)handleKeyDownEvent:(NSEvent *)event eschewCocoaTextHandling:(BOOL)eschewCocoaTextHandling {
     id delegate = [self delegate];
     unsigned int modflag = [event modifierFlags];
     unsigned short keyCode = [event keyCode];
@@ -1486,15 +1491,17 @@ static const int kDragThreshold = 3;
         // calls as needed in -insertText and -doCommandBySelector.
         gCurrentKeyEventTextView = [[self retain] autorelease];
-        if ([iTermAdvancedSettingsModel experimentalKeyHandling]) {
-          // This may cause -insertText:replacementRange: or -doCommandBySelector: to be called.
-          // These methods have a side-effect of setting _keyPressHandled if they dispatched the event
-          // to the delegate. They might not get called: for example, if you hold down certain keys
-          // then repeats might be ignored, or the IME might handle it internally (such as when you press
-          // "L" in AquaSKK's Hiragana mode to enter ASCII mode. See pull request 279 for more on this.
-          [self.inputContext handleEvent:event];
-        } else {
-          [self interpretKeyEvents:[NSArray arrayWithObject:event]];
+        if (!eschewCocoaTextHandling) {
+            if ([iTermAdvancedSettingsModel experimentalKeyHandling]) {
+              // This may cause -insertText:replacementRange: or -doCommandBySelector: to be called.
+              // These methods have a side-effect of setting _keyPressHandled if they dispatched the event
+              // to the delegate. They might not get called: for example, if you hold down certain keys
+              // then repeats might be ignored, or the IME might handle it internally (such as when you press
+              // "L" in AquaSKK's Hiragana mode to enter ASCII mode. See pull request 279 for more on this.
+              [self.inputContext handleEvent:event];
+            } else {
+              [self interpretKeyEvents:[NSArray arrayWithObject:event]];
+            }
         }
         gCurrentKeyEventTextView = nil;
@@ -1503,6 +1510,10 @@ static const int kDragThreshold = 3;
         if ([iTermAdvancedSettingsModel experimentalKeyHandling]) {
             shouldPassToDelegate &= event.isARepeat;
         }
+        if (eschewCocoaTextHandling) {
+            // It was never sent tp cocoa so the delegate must take it
+            shouldPassToDelegate = YES;
+        }
         if (shouldPassToDelegate) {
             DLog(@"PTYTextView keyDown unhandled (likely repeated) keypress with no IME, send to delegate");
             [delegate keyDown:event];

sources/iTermNSKeyBindingEmulator.h

@@ -14,6 +14,9 @@
 // a key binding. If this returns NO, then |extraEvents| may be filled in with additional events
 // to process first. That happens when a series of keys is entered which make up a multi-key binding
 // ending in an unhandleable binding.
-- (BOOL)handlesEvent:(NSEvent *)event extraEvents:(NSMutableArray *)extraEvents;
+// If this returns YES then *pointlessly will also be set. If pointlessly is set to YES then
+// the caller should not pass the event to cocoa, or it will hold on to the event since it's the
+// prefix of a longer series of keystrokes, none of which can possibly lead to insertText:.
+- (BOOL)handlesEvent:(NSEvent *)event pointlessly:(BOOL *)pointlessly extraEvents:(NSMutableArray *)extraEvents;
 @end

sources/iTermNSKeyBindingEmulator.m

@@ -18,6 +18,7 @@
 // Entries map a "normalized key" (as produced by dictionaryKeyForCharacters:andFlags:) to either a
 // dictionary subtree, or to an array with a selector and its arguments.
 static NSDictionary *gRootKeyBindingsDictionary;
+static NSSet<NSString *> *gPointlessFirstKeystrokes;
 @interface iTermNSKeyBindingEmulator ()
@@ -47,26 +48,35 @@ static struct {
 + (void)initialize {
     if (self == [iTermNSKeyBindingEmulator self]) {
-        gRootKeyBindingsDictionary = [[self keyBindingsDictionary] retain];
+        [self loadKeyBindingsDictionary];
+        AppendPinnedDebugLogMessage(@"NSKeyBindingEmulator", @"Key bindings are:\n%@", gRootKeyBindingsDictionary);
     }
 }
-+ (NSDictionary *)keyBindingsDictionary {
++ (void)loadKeyBindingsDictionary {
     NSArray *paths = NSSearchPathForDirectoriesInDomains(NSLibraryDirectory,
                                                          NSUserDomainMask,
                                                          YES);
     
     if (![paths count]) {
-        return nil;
+        AppendPinnedDebugLogMessage(@"NSKeyBindingEmulator", @"Failed to find library directory.");
+        return;
     }
     NSString *bindPath =
         [paths[0] stringByAppendingPathComponent:@"KeyBindings/DefaultKeyBinding.dict"];
     NSDictionary *theDict = [NSDictionary dictionaryWithContentsOfFile:bindPath];
+    AppendPinnedDebugLogMessage(@"NSKeyBindingEmulator", @"Load dictionary\n%@", theDict);
     DLog(@"Loaded key bindings dictionary:\n%@", theDict);
-    return [self keyBindingDictionaryByPruningUselessBranches:[self keyBindingDictionaryByNormalizingModifiersInKeys:theDict]];
+
+    NSDictionary *keyBindingDictionary = [self keyBindingDictionaryByNormalizingModifiersInKeys:theDict];
+    NSMutableSet<NSString *> *temp = [NSMutableSet set];
+    gRootKeyBindingsDictionary = [[self keyBindingDictionaryByPruningUselessBranches:keyBindingDictionary
+                                                                 pointlessKeystrokes:temp] retain];
+    gPointlessFirstKeystrokes = [temp retain];
 }
-+ (NSDictionary *)keyBindingDictionaryByPruningUselessBranches:(NSDictionary *)node {
++ (NSDictionary *)keyBindingDictionaryByPruningUselessBranches:(NSDictionary *)node
+                                           pointlessKeystrokes:(NSMutableSet<NSString *> *)pointless {
     // Remove leafs that do not have an insertText: action.
     // Recursively rune dictionary values, removing them if empty.
     // Returns nil if the result would be an empty dictionary.
@@ -78,19 +88,22 @@ static struct {
     // user input. It brings us closer to Terminal, which no longer supports
     // key bindings at all.
     NSMutableDictionary *replacement = [NSMutableDictionary dictionary];
-    for (id key in node) {
+    for (NSString *key in node) {
         id value = node[key];
         if ([value isKindOfClass:[NSDictionary class]]) {
-            value = [self keyBindingDictionaryByPruningUselessBranches:value];
+            value = [self keyBindingDictionaryByPruningUselessBranches:value pointlessKeystrokes:nil];
         } else if ([value isKindOfClass:[NSArray class]]) {
             if (![[value firstObject] isEqualToString:@"insertText:"]) {
                 value = nil;
             }
         } else {
+            AppendPinnedDebugLogMessage(@"NSKeyBindingEmulator", @"Reject non-container leaf with key %@", key);
             value = nil;
         }
         if (value) {
             replacement[key] = value;
+        } else if ([node[key] isKindOfClass:[NSDictionary class]]) {
+            [pointless addObject:key];
         }
     }
     if ([replacement count]) {
@@ -280,9 +293,10 @@ static struct {
     [super dealloc];
 }
-- (BOOL)handlesEvent:(NSEvent *)event extraEvents:(NSMutableArray *)extraEvents {
-    if (!gRootKeyBindingsDictionary) {
-        DLog(@"Short-circuit DefaultKeyBindings handling because no bindings are defined");
+- (BOOL)handlesEvent:(NSEvent *)event pointlessly:(BOOL *)pointlessly extraEvents:(NSMutableArray *)extraEvents {
+    *pointlessly = NO;
+    if (!gRootKeyBindingsDictionary && [gPointlessFirstKeystrokes count] == 0) {
+        DLog(@"Short-circuit DefaultKeyBindings handling because no bindings are defined and there are no pointless leaders");
         [_savedEvents removeAllObjects];
         return NO;
     }
@@ -318,7 +332,15 @@ static struct {
     // Not (or no longer) in a multi-keystroke binding. Move to the root of the tree.
     self.currentDict = gRootKeyBindingsDictionary;
     DLog(@"Default key binding is %@", obj);
-    
+
+    for (NSString *theKey in possibleKeys) {
+        if ([gPointlessFirstKeystrokes containsObject:theKey]) {
+            *pointlessly = YES;
+            DLog(@"Keystroke is pointless. Caller should not pass to handleEvent or interpretKeyEvents.");
+            return YES;
+        }
+    }
+
     if (![obj isKindOfClass:[NSArray class]]) {
         [extraEvents addObjectsFromArray:_savedEvents];
         [_savedEvents removeAllObjects];