Refactor the CSI parser and add a suite of automatic unit tests to measure xterm compatibility.

.gitignore

 .svn
+*.pyc
 *.mode1v3
 *.mode2v3
 *.move1v3

iTerm2.xcodeproj/project.pbxproj

@@ -1690,6 +1690,7 @@
 		A6DF401D1897607E00F05947 /* NSTextField+iTerm.m in Sources */ = {isa = PBXBuildFile; fileRef = A6DF401B1897607E00F05947 /* NSTextField+iTerm.m */; };
 		A6DF401E1897607E00F05947 /* NSTextField+iTerm.m in Sources */ = {isa = PBXBuildFile; fileRef = A6DF401B1897607E00F05947 /* NSTextField+iTerm.m */; };
 		A6DF7B701A539B4200D96409 /* VT100XtermParserTest.m in Sources */ = {isa = PBXBuildFile; fileRef = A6DF7B6F1A539B4200D96409 /* VT100XtermParserTest.m */; };
+		A6DF7B741A5F581700D96409 /* VT100CSIParserTest.m in Sources */ = {isa = PBXBuildFile; fileRef = A6DF7B731A5F581700D96409 /* VT100CSIParserTest.m */; };
 		A6E7137A18F1D70E008D94DD /* GeneralPreferencesViewController.h in Headers */ = {isa = PBXBuildFile; fileRef = A6E7137818F1D70D008D94DD /* GeneralPreferencesViewController.h */; };
 		A6E7137B18F1D70E008D94DD /* GeneralPreferencesViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = A6E7137918F1D70D008D94DD /* GeneralPreferencesViewController.m */; };
 		A6E7137C18F1D70E008D94DD /* GeneralPreferencesViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = A6E7137918F1D70D008D94DD /* GeneralPreferencesViewController.m */; };
@@ -2517,6 +2518,8 @@
 		A6DF7B6E1A539B4200D96409 /* VT100XtermParserTest.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VT100XtermParserTest.h; sourceTree = "<group>"; };
 		A6DF7B6F1A539B4200D96409 /* VT100XtermParserTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = VT100XtermParserTest.m; sourceTree = "<group>"; };
 		A6DF7B711A5BA33900D96409 /* iTermParser.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = iTermParser.h; sourceTree = "<group>"; };
+		A6DF7B721A5F581700D96409 /* VT100CSIParserTest.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VT100CSIParserTest.h; sourceTree = "<group>"; };
+		A6DF7B731A5F581700D96409 /* VT100CSIParserTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = VT100CSIParserTest.m; sourceTree = "<group>"; };
 		A6E7137818F1D70D008D94DD /* GeneralPreferencesViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 4; lastKnownFileType = sourcecode.c.h; path = GeneralPreferencesViewController.h; sourceTree = "<group>"; tabWidth = 4; };
 		A6E7137918F1D70D008D94DD /* GeneralPreferencesViewController.m */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 4; lastKnownFileType = sourcecode.c.objc; path = GeneralPreferencesViewController.m; sourceTree = "<group>"; tabWidth = 4; };
 		A6E7137D18F1DB1E008D94DD /* iTermPreferences.h */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 4; lastKnownFileType = sourcecode.c.h; path = iTermPreferences.h; sourceTree = "<group>"; tabWidth = 4; };
@@ -3363,6 +3366,8 @@
 				A63679821A328E7D000DB943 /* SemanticHistoryTest.m */,
 				A6DF7B6E1A539B4200D96409 /* VT100XtermParserTest.h */,
 				A6DF7B6F1A539B4200D96409 /* VT100XtermParserTest.m */,
+				A6DF7B721A5F581700D96409 /* VT100CSIParserTest.h */,
+				A6DF7B731A5F581700D96409 /* VT100CSIParserTest.m */,
 			);
 			name = Tests;
 			path = sources/;
@@ -5319,6 +5324,7 @@
 				1D9A555C180FA83900B42CE9 /* EventMonitorView.m in Sources */,
 				A68A30E9186D1429007F550F /* VT100WorkingDirectory.m in Sources */,
 				A697100D18D94CDA007E901D /* iTermAdvancedSettingsModel.m in Sources */,
+				A6DF7B741A5F581700D96409 /* VT100CSIParserTest.m in Sources */,
 				A647E3AB18C353C500450FA1 /* VT100StringParser.m in Sources */,
 				A6E77F771A23D195009B1CB6 /* iTermIndicatorsHelper.m in Sources */,
 				A68A3116186E2F14007F550F /* PopupWindow.m in Sources */,

sources/FileTransferManager.m

@@ -54,22 +54,20 @@ static const NSTimeInterval kMaximumTimeToKeepFinishedDownload = 24 * 60 * 60;
 }
 - (void)cleanUpMenus {
-    for (NSMenu *menu in @[ [self downloadsMenu], [self uploadsMenu] ]) {
-        NSMutableArray *controllersToRemove = [NSMutableArray array];
-        for (TransferrableFileMenuItemViewController *controller in _viewControllers) {
-            if ([controller timeSinceLastStatusChange] > kMaximumTimeToKeepFinishedDownload &&
-                controller.transferrableFile.status != kTransferrableFileStatusStarting &&
-                controller.transferrableFile.status != kTransferrableFileStatusTransferring &&
-                controller.transferrableFile.status != kTransferrableFileStatusCancelling) {
-                [menu removeItem:controller.view.enclosingMenuItem];
-                [controllersToRemove addObject:controller];
-            }
-        }
-        
-        for (TransferrableFileMenuItemViewController *controller in controllersToRemove) {
-            [_viewControllers removeObject:controller];
+    NSMutableArray *controllersToRemove = [NSMutableArray array];
+    for (TransferrableFileMenuItemViewController *controller in _viewControllers) {
+        if ([controller timeSinceLastStatusChange] > kMaximumTimeToKeepFinishedDownload &&
+            controller.transferrableFile.status != kTransferrableFileStatusStarting &&
+            controller.transferrableFile.status != kTransferrableFileStatusTransferring &&
+            controller.transferrableFile.status != kTransferrableFileStatusCancelling) {
+            [controller.view.enclosingMenuItem.menu removeItem:controller.view.enclosingMenuItem];
+            [controllersToRemove addObject:controller];
         }
     }
+    
+    for (TransferrableFileMenuItemViewController *controller in controllersToRemove) {
+        [_viewControllers removeObject:controller];
+    }
 }
 - (NSMenu *)downloadsMenu {

sources/VT100AnsiParser.h

@@ -11,7 +11,7 @@
 NS_INLINE BOOL isANSI(unsigned char *code, int len) {
     // Currently, we only support esc-c as an ANSI code (other ansi codes are CSI).
-    if (len >= 2 && code[0] == ESC && code[1] == 'c') {
+    if (len >= 2 && code[0] == VT100CC_ESC && code[1] == 'c') {
         return YES;
     }
     return NO;

sources/VT100AnsiParser.m

@@ -15,7 +15,7 @@
           bytesUsed:(int *)rmlen
               token:(VT100Token *)result {
     result->type = VT100_UNKNOWNCHAR;
-    if (datalen >= 2 && datap[0] == ESC) {
+    if (datalen >= 2 && datap[0] == VT100CC_ESC) {
         switch (datap[1]) {
             case 'c':
                 result->type = ANSI_RIS;

sources/VT100CSIParser.h

@@ -8,6 +8,7 @@
 #import <Foundation/Foundation.h>
 #import "CVector.h"
+#import "iTermParser.h"
 #import "VT100Token.h"
 typedef enum {
@@ -20,19 +21,14 @@ typedef enum {
 } VT100CSIIncidentalType;
 NS_INLINE BOOL isCSI(unsigned char *code, int len) {
-    if (len >= 2 && code[0] == ESC && (code[1] == '[')) {
-        return YES;
-    }
-    return NO;
+    return (len >= 2 && code[0] == VT100CC_ESC && (code[1] == '['));
 }
 @interface VT100CSIParser : NSObject
-+ (void)decodeBytes:(unsigned char *)datap
-             length:(int)datalen
-          bytesUsed:(int *)rmlen
-        incidentals:(CVector *)incidentals
-              token:(VT100Token *)result;
++ (void)decodeFromContext:(iTermParserContext *)context
+              incidentals:(CVector *)incidentals
+                    token:(VT100Token *)result;
 @end

sources/VT100CSIParserTest.m

@@ -7,7 +7,343 @@
 //
 #import "VT100CSIParserTest.h"
-@implementation VT100CSIParserTest
+#import "VT100CSIParser.h"
+@implementation VT100CSIParserTest {
+    CVector _incidentals;
+    iTermParserContext _context;
+}
+
+- (void)setup {
+    CVectorCreate(&_incidentals, 1);
+}
+
+- (void)teardown {
+    CVectorDestroy(&_incidentals);
+}
+
+- (VT100Token *)tokenForDataWithFormat:(NSString *)formatString, ... {
+    va_list args;
+    va_start(args, formatString);
+    NSString *string = [[[NSString alloc] initWithFormat:formatString arguments:args] autorelease];
+    va_end(args);
+
+    NSData *data = [string dataUsingEncoding:NSUTF8StringEncoding];
+    VT100Token *token = [[[VT100Token alloc] init] autorelease];
+    _context = iTermParserContextMake((unsigned char *)data.bytes, data.length);
+    [VT100CSIParser decodeFromContext:&_context incidentals:&_incidentals token:token];
+    return token;
+}
+
+- (void)testCSIOnly {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[", VT100CC_ESC];
+    assert(token->type == VT100_WAIT);
+}
+
+- (void)testPrefixOnly {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[?", VT100CC_ESC];
+    assert(token->type == VT100_WAIT);
+}
+
+- (void)testPrefixParameterOnly {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[?36", VT100CC_ESC];
+    assert(token->type == VT100_WAIT);
+}
+
+- (void)testPrefixParameterIntermediateOnly {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[?36$", VT100CC_ESC];
+    assert(token->type == VT100_WAIT);
+}
+
+- (void)testFullyFormedPrefixParameterIntermediateFinal {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[?36$p", VT100CC_ESC];
+    assert(token->type == VT100_NOTSUPPORT);  // Sadly, DECRQM isn't supported yet, so this test is incomplete.
+}
+
+- (void)testSimpleCSI {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[D", VT100CC_ESC];
+    assert(token->type == VT100CSI_CUB);
+    assert(token.csi->count == 1);
+    assert(token.csi->p[0] == 1);  // Default
+    assert(token.csi->subCount[0] == 0);
+}
+
+- (void)testSimpleCSIWithParameter {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[2D", VT100CC_ESC];
+    assert(token->type == VT100CSI_CUB);
+    assert(token.csi->count == 1);
+    assert(token.csi->p[0] == 2);  // Parameter
+    assert(token.csi->subCount[0] == 0);
+}
+
+- (void)testSimpleCSIWithTwoDigitParameter {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[23D", VT100CC_ESC];
+    assert(token->type == VT100CSI_CUB);
+    assert(token.csi->count == 1);
+    assert(token.csi->p[0] == 23);  // Parameter
+    assert(token.csi->subCount[0] == 0);
+}
+
+- (void)testParameterPrefix {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[>23c", VT100CC_ESC];
+    assert(token->type == VT100CSI_DA2);
+    assert(token.csi->count == 1);
+    assert(token.csi->p[0] == 23);  // Parameter
+    assert(token.csi->subCount[0] == 0);
+}
+
+- (void)testTwoParameters {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[5;6H", VT100CC_ESC];
+    assert(token->type == VT100CSI_CUP);
+    assert(token.csi->count == 2);
+    assert(token.csi->p[0] == 5);
+    assert(token.csi->p[1] == 6);
+    assert(token.csi->subCount[0] == 0);
+}
+
+- (void)testSubParameter {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[38:2:255:128:64:0:5:1m", VT100CC_ESC];
+    assert(token->type == VT100CSI_SGR);
+    assert(token.csi->count == 1);
+    assert(token.csi->p[0] == 38);
+    assert(token.csi->subCount[0] == 7);
+    assert(token.csi->sub[0][0] == 2);
+    assert(token.csi->sub[0][1] == 255);
+    assert(token.csi->sub[0][2] == 128);
+    assert(token.csi->sub[0][3] == 64);
+    assert(token.csi->sub[0][4] == 0);
+    assert(token.csi->sub[0][5] == 5);
+    assert(token.csi->sub[0][6] == 1);
+}
+
+- (void)testBogusCharacterInParameters {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[38=m", VT100CC_ESC];
+    assert(token->type == VT100_UNKNOWNCHAR);
+}
+
+- (void)testIntermediateByte {
+    // DECSCUSR with paraemter 3 (set cursor to "blink underline"), which has an intermediate byte
+    // of the space character.
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[3 q", VT100CC_ESC];
+    assert(token->type == VT100CSI_DECSCUSR);
+    assert(token.csi->count == 1);
+    assert(token.csi->p[0] == 3);
+    assert(token.csi->subCount[0] == 0);
+}
+
+- (void)testBogusCharInParameterSection {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[1<m", VT100CC_ESC];
+    assert(token->type == VT100_UNKNOWNCHAR);
+}
+
+- (void)testGarbageIgnored {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[1%cm", VT100CC_ESC, 0x7f];
+    assert(token->type == VT100CSI_SGR);
+}
+
+- (void)testGarbageSpaceCausesFailure {
+    VT100Token *token = [self tokenForDataWithFormat:@"%c[1%c m", VT100CC_ESC, 0x7f];
+    assert(token->type == VT100_UNKNOWNCHAR);
+}
+
+- (void)testDefaultParameterValues {
+    struct {
+        char prefix;
+        char intermediate;
+        char final;
+        VT100TerminalTokenType tokenType;
+        int p0;
+        int p1;
+    } simpleCodes[] = {
+        { 0, 0, '@', VT100CSI_ICH, 1, -1 },
+        { 0, 0, 'A', VT100CSI_CUU, 1, -1 },
+        { 0, 0, 'B', VT100CSI_CUD, 1, -1 },
+        { 0, 0, 'C', VT100CSI_CUF, 1, -1 },
+        { 0, 0, 'D', VT100CSI_CUB, 1, -1 },
+        { 0, 0, 'E', VT100CSI_CNL, 1, -1 },
+        { 0, 0, 'F', VT100CSI_CPL, 1, -1 },
+        { 0, 0, 'G', ANSICSI_CHA, 1, -1 },
+        { 0, 0, 'H', VT100CSI_CUP, 1, 1 },
+        // I not supported (Cursor Forward Tabulation P s tab stops (default = 1) (CHT))
+        { 0, 0, 'J', VT100CSI_ED, 0, -1 },
+        // ?J not supported (Erase in Display (DECSED))
+        { 0, 0, 'K', VT100CSI_EL, 0, -1 },
+        // ?K not supported ((Erase in Line (DECSEL))
+        { 0, 0, 'L', XTERMCC_INSLN, 1, -1 },
+        { 0, 0, 'M', XTERMCC_DELLN, 1, -1 },
+        { 0, 0, 'P', XTERMCC_DELCH, 1, -1 },
+        { 0, 0, 'S', XTERMCC_SU, 1, -1 },
+        // ?Pi;Pa;PvS not supported (Sixel/ReGIS)
+        { 0, 0, 'T', XTERMCC_SD, 1, -1 },
+        // Ps;Ps;Ps;Ps;PsT not supported (Initiate highlight mouse tracking)
+        { 0, 0, 'X', ANSICSI_ECH, 1, -1 },
+        { 0, 0, 'Z', ANSICSI_CBT, 1, -1 },
+        // ` not supported (Character Position Absolute [column] (default = [row,1]) (HPA))
+        // a not supported (Character Position Relative [columns] (default = [row,col+1]) (HPR))
+        // b not supported (Repeat the preceding graphic character P s times (REP))
+        { 0, 0, 'c', VT100CSI_DA, 0, -1 },
+        { '>', 0, 'c', VT100CSI_DA2, 0, -1 },
+        { 0, 0, 'd', ANSICSI_VPA, 1, -1 },
+        { 0, 0, 'e', ANSICSI_VPR, 1, -1 },
+        { 0, 0, 'f', VT100CSI_HVP, 1, 1 },
+        { 0, 0, 'g', VT100CSI_TBC, 0, -1 },
+        { 0, 0, 'h', VT100CSI_SM, -1, -1 },
+        { '?', 0, 'h', VT100CSI_DECSET, -1, -1 },
+        { 0, 0, 'i', ANSICSI_PRINT, 0, -1 },
+        // ?i not supported (Media Copy (MC, DEC-specific))
+        { 0, 0, 'l', VT100CSI_RM, -1, -1 },
+        { '?', 0, 'l', VT100CSI_DECRST, -1, -1 },
+        { 0, 0, 'm', VT100CSI_SGR, 0, -1 },
+        { '>', 0, 'm', VT100CSI_SET_MODIFIERS, -1, -1 },
+        { 0, 0, 'n', VT100CSI_DSR, 0, -1 },
+        { '>', 0, 'n', VT100CSI_RESET_MODIFIERS, -1, -1 },
+        { '?', 0, 'n', VT100CSI_DECDSR, 0, -1 },
+        // >p not supported (Set resource value pointerMode. This is used by xterm to decide whether
+        // to hide the pointer cursor as the user types.)
+        { '!', 0, 'p', VT100CSI_DECSTR, -1, -1 },
+        // $p not supported (Request ANSI mode (DECRQM))
+        // ?$p not supported (Request DEC private mode (DECRQM))
+        // "p not supported (Set conformance level (DECSCL))
+        // q not supported (Load LEDs (DECLL))
+        { 0, ' ', 'q', VT100CSI_DECSCUSR, 0, -1 },
+        // "q not supported (Select character protection attribute (DECSCA))
+        { 0, 0, 'r', VT100CSI_DECSTBM, -1, -1 },
+        // $r not supported (Change Attributes in Rectangular Area (DECCARA))
+        { 0, 0, 's', VT100CSI_DECSLRM_OR_ANSICSI_SCP, -1, -1 },
+        // ?s not supported (Save DEC Private Mode Values)
+        // t tested in -testWindowManipulationCodes
+        // $t not supported (Reverse Attributes in Rectangular Area (DECRARA))
+        // >t not supported (Set one or more features of the title modes)
+        // SP t not supported (Set warning-bell volume (DECSWBV, VT520))
+        { 0, 0, 'u', ANSICSI_RCP, -1, -1 },
+        // SP u not supported (Set margin-bell volume (DECSMBV, VT520))
+        // $v not supported (Copy Rectangular Area (DECCRA, VT400 and up))
+        // 'w not supported (Enable Filter Rectangle (DECEFR), VT420 and up)
+        // x not supported (Request Terminal Parameters (DECREQTPARM))
+        // *x not supported (Select Attribute Change Extent (DECSACE))
+        { 0, '*', 'y', VT100CSI_DECRQCRA, -1, -1 },
+        // $x not supported (Fill Rectangular Area (DECFRA), VT420 and up)
+        // 'z not supported (Enable Locator Reporting (DECELR))
+        // $z not supported (Erase Rectangular Area (DECERA), VT400 and up)
+        // '{ not supported (Select Locator Events (DECSLE))
+        // ${ not supported (Selective Erase Rectangular Area (DECSERA), VT400 and up)
+        // '| not supported (Request Locator Position (DECRQLP))
+        // '} not supported (Insert P s Column(s) (default = 1) (DECIC), VT420 and up)
+        // '~ not supported (Delete P s Column(s) (default = 1) (DECDC), VT420 and up)
+    };
+
+    const int n = sizeof(simpleCodes) / sizeof(*simpleCodes);
+    for (int i = 0; i < n; i++) {
+        int maxParams = 0;
+        if (simpleCodes[i].p1 >= 0) {
+            maxParams = 2;
+        } else if (simpleCodes[i].p0 >= 0) {
+            maxParams = 1;
+        }
+
+        NSMutableString *s = [NSMutableString stringWithFormat:@"%c[", VT100CC_ESC];
+        if (simpleCodes[i].prefix) {
+            [s appendFormat:@"%c", simpleCodes[i].prefix];
+        }
+        if (simpleCodes[i].intermediate) {
+            [s appendFormat:@"%c", simpleCodes[i].intermediate];
+        }
+        if (simpleCodes[i].final) {
+            [s appendFormat:@"%c", simpleCodes[i].final];
+        }
+        VT100Token *token = [self tokenForDataWithFormat:@"%@", s];
+        assert(token->type == simpleCodes[i].tokenType);
+        assert(token.csi->count == maxParams);
+        if (maxParams >= 1) {
+            assert(token.csi->count >= 1);
+            assert(token.csi->p[0] == simpleCodes[i].p0);
+        } else if (maxParams >= 2) {
+            assert(token.csi->count >= 2);
+            assert(token.csi->p[1] == simpleCodes[i].p1);
+        }
+        assert(token.csi->p[maxParams] == -1);
+    }
+}
+
+// This test is here to remind you to write a test when implementing support for a new CSI code.
+- (void)testUnsupportedCodes {
+    char *unsupported[] = {
+        "I",
+        "?J",
+        "?K",
+        "?1;1;1S",
+        "1;1;1;1;1T",
+        "`",
+        "a",
+        "1b",
+        "?1i",
+        ">0p",
+        "1$p",
+        "?1$p",
+        "61;0\"p",
+        "q",
+        "\"q",
+        "1;2;3;4;0$r",
+        "?1s",
+        "1;2;3;4;0$t",
+        ">1;60t",
+        "0 t",
+        "1 u",
+        "1;2;3;4;5;6;7;8$v",
+        "1;2;3;4'w",
+        "x",
+        "0*x",
+        "1;2;3;4;5;6*y",
+        "0;1;2;3;4$x",
+        "0;0'z",
+        "1;2;3;4$z",
+        "'{",
+        "1;2;3;4${",
+        "'|",
+        "'}",
+        "'~",
+    };
+    const int n = sizeof(unsupported) / sizeof(*unsupported);
+    for (int i = 0; i < n; i++) {
+        VT100Token *token = [self tokenForDataWithFormat:@"%c[%s", VT100CC_ESC, unsupported[i]];
+        assert(token->type == VT100_NOTSUPPORT);
+    }
+}
+
+- (void)testWindowManipulationCodes {
+    struct {
+        int p0;
+        VT100TerminalTokenType type;
+    } codes[] = {
+        { 1, XTERMCC_DEICONIFY },
+        { 2, XTERMCC_ICONIFY },
+        { 3, XTERMCC_WINDOWPOS },
+        { 4, XTERMCC_WINDOWSIZE_PIXEL },
+        { 5, XTERMCC_RAISE },
+        { 6, XTERMCC_LOWER },
+        // 7 is not supported (Refresh the window)
+        { 8, XTERMCC_WINDOWSIZE },
+        // 9 is not supported (Various maximize window actions)
+        // 10 is not supported (Various full-screen actions)
+        { 11, XTERMCC_REPORT_WIN_STATE },
+        // 12 is not defined
+        { 13, XTERMCC_REPORT_WIN_POS },
+        { 14, XTERMCC_REPORT_WIN_PIX_SIZE },
+        // 15, 16, and 17 are not defined
+        { 18, XTERMCC_REPORT_WIN_SIZE },
+        { 19, XTERMCC_REPORT_SCREEN_SIZE },
+        { 20, XTERMCC_REPORT_ICON_TITLE },
+        { 21, XTERMCC_REPORT_WIN_TITLE },
+        { 22, XTERMCC_PUSH_TITLE },
+        { 23, XTERMCC_POP_TITLE },
+        // 24+ is not supported (resize to Ps lines - DECSLPP)
+    };
+    int n = sizeof(codes) / sizeof(*codes);
+    for (int i = 0; i < n; i++) {
+        VT100Token *token = [self tokenForDataWithFormat:@"%c[%dt", VT100CC_ESC, codes[i].p0];
+        assert(token->type == codes[i].type);
+        assert(token.csi->p[0] == codes[i].p0);
+    }
+}
 @end

sources/VT100ControlParser.m

@@ -23,17 +23,17 @@ void ParseControl(unsigned char *datap,
                   NSStringEncoding encoding,
                   int tmuxCodeWrapCount,
                   NSMutableDictionary *savedState) {
-    if (tmuxCodeWrapCount && datalen >= 2 && datap[0] == ESC && datap[1] == '\\') {
+    if (tmuxCodeWrapCount && datalen >= 2 && datap[0] == VT100CC_ESC && datap[1] == '\\') {
         token->type = DCS_END_TMUX_CODE_WRAP;
         *rmlen = 2;
         return;
     }
     if (isCSI(datap, datalen)) {
-        [VT100CSIParser decodeBytes:datap
-                             length:datalen
-                          bytesUsed:rmlen
-                        incidentals:incidentals
-                              token:token];
+        iTermParserContext context = iTermParserContextMake(datap, datalen);
+        [VT100CSIParser decodeFromContext:&context
+                              incidentals:incidentals
+                                    token:token];
+        *rmlen = context.rmlen;
     } else if (isXTERM(datap, datalen)) {
         iTermParserContext context = iTermParserContextMake(datap, datalen);
         [VT100XtermParser decodeFromContext:&context

sources/VT100DCSParser.h

@@ -17,7 +17,7 @@ typedef enum {
 } DcsTermcapTerminfoRequestName;
 NS_INLINE BOOL isDCS(unsigned char *code, int len) {
-    return (len >= 2 && code[0] == ESC && code[1] == 'P');
+    return (len >= 2 && code[0] == VT100CC_ESC && code[1] == 'P');
 }

sources/VT100DCSParser.m

@@ -26,7 +26,7 @@
             datap += 2;
             datalen -= 2;
             *rmlen += 2;
-            char st[3] = { ESC, '\\', '\0' };
+            char st[3] = { VT100CC_ESC, '\\', '\0' };
             char *positionOfST = strnstr((const char *)datap, st, datalen);
             if (!positionOfST) {
                 return;
@@ -61,7 +61,7 @@
             *rmlen += 5;
         } else if (datalen >= 6 &&
                    !strncmp((char *)datap, "tmux;", 5) &&
-                   datap[5] == ESC) {
+                   datap[5] == VT100CC_ESC) {
             result->type = DCS_BEGIN_TMUX_CODE_WRAP;
             *rmlen += 6;
         } else {

sources/VT100OtherParser.m

@@ -17,7 +17,7 @@
            encoding:(NSStringEncoding)encoding {
     int c1, c2;
     
-    NSCParameterAssert(datap[0] == ESC);
+    NSCParameterAssert(datap[0] == VT100CC_ESC);
     NSCParameterAssert(datalen > 1);
     
     c1 = (datalen >= 2 ? datap[1]: -1);
@@ -160,9 +160,9 @@
                 for (i = 2; i < datalen; i++) {
                     BOOL isTerminator = NO;
                     int length = i - 2;
-                    if (datap[i] == ESC && i + 1 == datalen) {
+                    if (datap[i] == VT100CC_ESC && i + 1 == datalen) {
                         break;
-                    } else if (datap[i] == ESC && datap[i + 1] == '\\') {
+                    } else if (datap[i] == VT100CC_ESC && datap[i + 1] == '\\') {
                         i++;  // cause the backslash to be consumed below
                         isTerminator = YES;
                     } else if (datap[i] == '\n' || datap[i] == '\r') {

sources/VT100Output.h

@@ -51,6 +51,7 @@ typedef enum {
 - (NSData *)reportDeviceAttribute;
 - (NSData *)reportSecondaryDeviceAttribute;
 - (NSData *)reportColor:(NSColor *)color atIndex:(int)index;
+- (NSData *)reportChecksum:(int)checksum withIdentifier:(int)identifier;
 - (void)setTermTypeIsValid:(BOOL)termTypeIsValid;

sources/VT100Output.m

@@ -554,6 +554,13 @@ typedef enum {
     return [string dataUsingEncoding:NSUTF8StringEncoding];
 }
+- (NSData *)reportChecksum:(int)checksum withIdentifier:(int)identifier {
+    // DCS Pid ! ~ D..D ST
+    NSString *string =
+        [NSString stringWithFormat:@"%cP%d!~%04x%c\\", ESC, identifier, (short)checksum, ESC];
+    return [string dataUsingEncoding:NSUTF8StringEncoding];
+}
+
 #pragma mark - Private
 - (NSData *)specialKey:(int)terminfo

sources/VT100Screen.m

@@ -2497,6 +2497,12 @@ static NSString *const kInlineFileBase64String = @"base64 string";  // NSMutable
 }
 - (void)terminalSetRows:(int)rows andColumns:(int)columns {
+    if (rows == -1) {
+        rows = self.height;
+    }
+    if (columns == -1) {
+        columns = self.width;
+    }
     if ([delegate_ screenShouldInitiateWindowResize] &&
         ![delegate_ screenWindowIsFullscreen]) {
         [delegate_ screenResizeToWidth:columns
@@ -2604,6 +2610,8 @@ static NSString *const kInlineFileBase64String = @"base64 string";  // NSMutable
 - (NSString *)terminalIconTitle {
     if (allowTitleReporting_) {
+        // TODO: Should be something like screenRawName (which doesn't exist yet but would return
+        // [self rawName]), not screenWindowTitle, right?
         return [delegate_ screenWindowTitle] ? [delegate_ screenWindowTitle] : [delegate_ screenDefaultName];
     } else {
         return @"";
@@ -3334,6 +3342,29 @@ static NSString *const kInlineFileBase64String = @"base64 string";  // NSMutable
     return [delegate_ screenProfileName];
 }
+- (VT100GridRect)terminalScrollRegion {
+    return currentGrid_.scrollRegionRect;
+}
+
+- (int)terminalChecksumInRectangle:(VT100GridRect)rect {
+    int result = 0;
+    for (int y = rect.origin.y; y < rect.origin.y + rect.size.height; y++) {
+        screen_char_t *theLine = [self getLineAtScreenIndex:y];
+        for (int x = rect.origin.x; x < rect.origin.x + rect.size.width; x++) {
+            unichar code = theLine[x].code;
+            BOOL isPrivate = (code < ITERM2_PRIVATE_BEGIN &&
+                              code > ITERM2_PRIVATE_END);
+            if (code && !isPrivate) {
+                NSString *s = ScreenCharToStr(&theLine[x]);
+                for (int i = 0; i < s.length; i++) {
+                    result += (int)[s characterAtIndex:i];
+                }
+            }
+        }
+    }
+    return result;
+}
+
 #pragma mark - Private
 - (VT100GridCoordRange)commandRange {

sources/VT100Terminal.m

@@ -917,6 +917,89 @@ static const int kMaxScreenRows = 4096;
     }
 }
+- (VT100GridRect)rectangleInToken:(VT100Token *)token
+                  startingAtIndex:(int)index
+                 defaultRectangle:(VT100GridRect)defaultRectangle {
+    CSIParam *csi = token.csi;
+    VT100GridCoord defaultMax = VT100GridRectMax(defaultRectangle);
+
+    // First, construct a coord range from the passed-in parameters. They may be -1 for default
+    // values.
+    int top = csi->p[index];
+    int left = csi->p[index + 1];
+    int bottom = csi->p[index + 2];
+    int right = csi->p[index + 3];
+    VT100GridCoordRange coordRange = VT100GridCoordRangeMake(left, top, right, bottom);
+
+    // If in origin mode, offset non-default coordinates by the origin of the scroll region.
+    if (self.originMode) {
+        VT100GridRect region = [delegate_ terminalScrollRegion];
+        if (coordRange.start.x >= 0) {
+            coordRange.start.x -= region.origin.x + 1;
+        }
+        if (coordRange.start.y >= 0) {
+            coordRange.start.y -= region.origin.y + 1;
+        }
+        if (coordRange.end.x >= 0) {
+            coordRange.end.x -= region.origin.x + 1;
+        }
+        if (coordRange.end.y >= 0) {
+            coordRange.end.y -= region.origin.y + 1;
+        }
+    }
+
+    // Replace default values with the passed-in defaults.
+    if (coordRange.start.x < 0) {
+        coordRange.start.x = defaultRectangle.origin.x + 1;
+    }
+    if (coordRange.start.y < 0) {
+        coordRange.start.y = defaultRectangle.origin.y + 1;
+    }
+    if (coordRange.end.x < 0) {
+        coordRange.end.x = defaultMax.x + 1;
+    }
+    if (coordRange.end.y < 0) {
+        coordRange.end.y = defaultMax.y + 1;
+    }
+
+    // Convert the coordRange to a 0-based rect (all coords are 1-based so far) and return it.
+    return VT100GridRectMake(coordRange.start.x - 1,
+                             coordRange.start.y - 1,
+                             coordRange.end.x - coordRange.start.x + 1,
+                             coordRange.end.y - coordRange.start.y + 1);
+}
+
+- (BOOL)rectangleIsValid:(VT100GridRect)rect {
+    if (self.originMode) {
+        VT100GridRect scrollRegion = [delegate_ terminalScrollRegion];
+        if (rect.origin.y < scrollRegion.origin.y ||
+            rect.origin.x < scrollRegion.origin.x ||
+            VT100GridRectMax(rect).y > VT100GridRectMax(scrollRegion).y ||
+            VT100GridRectMax(rect).x > VT100GridRectMax(scrollRegion).x) {
+            return NO;
+        }
+    }
+    return (rect.size.width >= 0 &&
+            rect.size.height >= 0);
+}
+
+- (void)sendChecksumReportWithId:(int)identifier
+                       rectangle:(VT100GridRect)rect {
+    if (![delegate_ terminalShouldSendReport]) {
+        return;
+    }
+    if (identifier < 0) {
+        return;
+    }
+    if (![self rectangleIsValid:rect]) {
+        return;
+    }
+    // TODO: Respect origin mode
+    int checksum = [delegate_ terminalChecksumInRectangle:rect];
+    // DCS Pid ! ~ D..D ST
+    [delegate_ terminalSendReport:[self.output reportChecksum:checksum withIdentifier:identifier]];
+}
+
 - (NSString *)decodedBase64PasteCommand:(NSString *)commandString {
     //
     // - write access
@@ -1031,11 +1114,11 @@ static const int kMaxScreenRows = 4096;
         case VT100CSI_DECSLRM_OR_ANSICSI_SCP:
             if ([delegate_ terminalUseColumnScrollRegion]) {
                 token->type = VT100CSI_DECSLRM;
-                SET_PARAM_DEFAULT(token.csi, 0, 1);
-                SET_PARAM_DEFAULT(token.csi, 1, 1);
+                iTermParserSetCSIParameterIfDefault(token.csi, 0, 1);
+                iTermParserSetCSIParameterIfDefault(token.csi, 1, 1);
             } else {
                 token->type = ANSICSI_SCP;
-                SET_PARAM_DEFAULT(token.csi, 0, 0);
+                iTermParserSetCSIParameterIfDefault(token.csi, 0, 0);
             }
             break;
@@ -1062,8 +1145,9 @@ static const int kMaxScreenRows = 4096;
         case VT100_NOTSUPPORT:
             break;
-            //  VT100 CC
+        //  VT100 CC
         case VT100CC_ENQ:
+            // TODO: Add support for an answerback string here.
             break;
         case VT100CC_BEL:
             [delegate_ terminalRingBell];
@@ -1139,31 +1223,37 @@ static const int kMaxScreenRows = 4096;
         case VT100CSI_DECID:
         case VT100CSI_DECKPAM:
         case VT100CSI_DECKPNM:
-        case VT100CSI_DECLL:
             break;
         case VT100CSI_DECRC:
             [self restoreTextAttributes];
             [delegate_ terminalRestoreCursor];
             [delegate_ terminalRestoreCharsetFlags];
             break;
-        case VT100CSI_DECREPTPARM:
-        case VT100CSI_DECREQTPARM:
-            break;
         case VT100CSI_DECSC:
             [self saveTextAttributes];
             [delegate_ terminalSaveCursor];
             [delegate_ terminalSaveCharsetFlags];
             break;
         case VT100CSI_DECSTBM:
-            [delegate_ terminalSetScrollRegionTop:token.csi->p[0] == 0 ? 0 : token.csi->p[0] - 1
-                                           bottom:token.csi->p[1] == 0 ? [delegate_ terminalHeight] - 1 : token.csi->p[1] - 1];
-            break;
-        case VT100CSI_DECSWL:
-        case VT100CSI_DECTST:
+            [delegate_ terminalSetScrollRegionTop:token.csi->p[0] == -1 ? 0 : token.csi->p[0] - 1
+                                           bottom:token.csi->p[1] == -1 ? [delegate_ terminalHeight] - 1 : token.csi->p[1] - 1];
             break;
         case VT100CSI_DSR:
             [self handleDeviceStatusReportWithToken:token withQuestion:NO];
             break;
+        case VT100CSI_DECRQCRA: {
+            VT100GridRect defaultRectangle = VT100GridRectMake(0,
+                                                               0,
+                                                               [delegate_ terminalWidth],
+                                                               [delegate_ terminalHeight]);
+            // xterm incorrectly uses the second parameter for the Pid. Since I use this mostly to
+            // test xterm compatibility, it's handy to be bugwards-compatible.
+            [self sendChecksumReportWithId:token.csi->p[1]
+                                 rectangle:[self rectangleInToken:token
+                                                  startingAtIndex:2
+                                                 defaultRectangle:defaultRectangle]];
+            break;
+        }
         case VT100CSI_DECDSR:
             [self handleDeviceStatusReportWithToken:token withQuestion:YES];
             break;
@@ -1420,7 +1510,7 @@ static const int kMaxScreenRows = 4096;
         case XTERMCC_ICON_TITLE:
             [delegate_ terminalSetIconTitle:[token.string stringByReplacingControlCharsWithQuestionMark]];
             break;
-        case XTERMCC_INSBLNK:
+        case VT100CSI_ICH:
             [delegate_ terminalInsertEmptyCharsAtCursor:token.csi->p[0]];
             break;
         case XTERMCC_INSLN:
@@ -1460,7 +1550,9 @@ static const int kMaxScreenRows = 4096;
             [delegate_ terminalScrollUp:token.csi->p[0]];
             break;
         case XTERMCC_SD:
-            [delegate_ terminalScrollDown:token.csi->p[0]];
+            if (token.csi->count == 1) {
+                [delegate_ terminalScrollDown:token.csi->p[0]];
+            }
             break;
         case XTERMCC_REPORT_WIN_STATE: {
             NSString *s = [NSString stringWithFormat:@"\033[%dt",
@@ -1521,6 +1613,7 @@ static const int kMaxScreenRows = 4096;
                 case 2:
                     [delegate_ terminalPushCurrentTitleForWindow:YES];
                     break;
+                // TODO: Support 3 (UTF-8)
             }
             break;
         }

sources/VT100TerminalDelegate.h

@@ -89,9 +89,15 @@ typedef enum {
 // Returns the cursor's position relative to the scroll region's origin. 1-based.
 - (int)terminalRelativeCursorY;
-// Set the top/bottom scrollr egion.
+// Set the top/bottom scroll region.
 - (void)terminalSetScrollRegionTop:(int)top bottom:(int)bottom;
+// Returns the scroll region, or the whole screen if none is set.
+- (VT100GridRect)terminalScrollRegion;
+
+// Sums the visible characters in the given rectangle onscreen.
+- (int)terminalChecksumInRectangle:(VT100GridRect)rect;
+
 // Erase all characters before the cursor and/or after the cursor.
 - (void)terminalEraseInDisplayBeforeCursor:(BOOL)before afterCursor:(BOOL)after;

sources/VT100Token.h

 #import <Foundation/Foundation.h>
 #import "iTermObjectPool.h"
+#import "iTermParser.h"
 #import "ScreenChar.h"
-#define ESC 0x1b
-
-#define VT100CSIPARAM_MAX 16  // Maximum number of CSI parameters in VT100Token.csi->p.
-#define VT100CSISUBPARAM_MAX 16  // Maximum number of CSI sub-parameters in VT100Token.csi->p.
-
-// If the n'th parameter has a negative (default) value, replace it with |value|.
-// CSI parameter values are all initialized to -1 before parsing, so this has the effect of setting
-// a value iff it hasn't already been set.
-// If there aren't yet n+1 parameters, increase the count to n+1.
-#define SET_PARAM_DEFAULT(csiParam, n, value) \
-    (((csiParam)->p[(n)] = (csiParam)->p[(n)] < 0 ? (value) : (csiParam)->p[(n)]), \
-    ((csiParam)->count = (csiParam)->count > (n) + 1 ? (csiParam)->count : (n) + 1 ))
-
 typedef enum {
     // Any control character between 0-0x1f inclusive can by a token type. For these, the value
     // matters. Make sure to update the -codeName method when changing this enum.
@@ -76,16 +64,11 @@ typedef enum {
     VT100CSI_DECID,                 // Identify Terminal
     VT100CSI_DECKPAM,               // Keypad Application Mode
     VT100CSI_DECKPNM,               // Keypad Numeric Mode
-    VT100CSI_DECLL,                 // Load LEDS
     VT100CSI_DECRC,                 // Restore Cursor
-    VT100CSI_DECREPTPARM,           // Report Terminal Parameters
-    VT100CSI_DECREQTPARM,           // Request Terminal Parameters
     VT100CSI_DECRST,
     VT100CSI_DECSC,                 // Save Cursor
     VT100CSI_DECSET,
     VT100CSI_DECSTBM,               // Set Top and Bottom Margins
-    VT100CSI_DECSWL,                // Single-width Line
-    VT100CSI_DECTST,                // Invoke Confidence Test
     VT100CSI_DSR,                   // Device Status Report
     VT100CSI_ED,                    // Erase In Display
     VT100CSI_EL,                    // Erase In Line
@@ -110,12 +93,13 @@ typedef enum {
     VT100CSI_SET_MODIFIERS,         // CSI > Ps; Pm m (Whether to set modifiers for different kinds of key presses; no official name)
     VT100CSI_RESET_MODIFIERS,       // CSI > Ps n (Set all modifiers values to -1, disabled)
     VT100CSI_DECSLRM,               // Set left-right margin
+    VT100CSI_DECRQCRA,              // Request Checksum of Rectangular Area
     // some xterm extensions
     XTERMCC_WIN_TITLE,            // Set window title
     XTERMCC_ICON_TITLE,
     XTERMCC_WINICON_TITLE,
-    XTERMCC_INSBLNK,              // Insert blank
+    VT100CSI_ICH,                 // Insert blank
     XTERMCC_INSLN,                // Insert lines
     XTERMCC_DELCH,                // delete blank
     XTERMCC_DELLN,                // delete lines
@@ -197,14 +181,6 @@ typedef enum {
     ISO2022_SELECT_UTF_8
 } VT100TerminalTokenType;
-typedef struct {
-    int p[VT100CSIPARAM_MAX];
-    int count;
-    int cmd;
-    int sub[VT100CSIPARAM_MAX][VT100CSISUBPARAM_MAX];
-    int subCount[VT100CSIPARAM_MAX];
-} CSIParam;
-
 // A preinitialized array of screen_char_t. When ASCII data is present, it will have the codes
 // populated and all other fields zeroed out.
 #define kStaticScreenCharsCount 16

sources/VT100Token.m

@@ -130,16 +130,12 @@ static iTermObjectPool *gPool;
                           @(VT100CSI_DECID):                  @"VT100CSI_DECID",
                           @(VT100CSI_DECKPAM):                @"VT100CSI_DECKPAM",
                           @(VT100CSI_DECKPNM):                @"VT100CSI_DECKPNM",
-                          @(VT100CSI_DECLL):                  @"VT100CSI_DECLL",
                           @(VT100CSI_DECRC):                  @"VT100CSI_DECRC",
-                          @(VT100CSI_DECREPTPARM):            @"VT100CSI_DECREPTPARM",
-                          @(VT100CSI_DECREQTPARM):            @"VT100CSI_DECREQTPARM",
+                          @(VT100CSI_DECRQCRA):               @"VT100CSI_DECRQCRA",
                           @(VT100CSI_DECRST):                 @"VT100CSI_DECRST",
                           @(VT100CSI_DECSC):                  @"VT100CSI_DECSC",
                           @(VT100CSI_DECSET):                 @"VT100CSI_DECSET",
                           @(VT100CSI_DECSTBM):                @"VT100CSI_DECSTBM",
-                          @(VT100CSI_DECSWL):                 @"VT100CSI_DECSWL",
-                          @(VT100CSI_DECTST):                 @"VT100CSI_DECTST",
                           @(VT100CSI_DSR):                    @"VT100CSI_DSR",
                           @(VT100CSI_ED):                     @"VT100CSI_ED",
                           @(VT100CSI_EL):                     @"VT100CSI_EL",
@@ -167,7 +163,7 @@ static iTermObjectPool *gPool;
                           @(XTERMCC_WIN_TITLE):               @"XTERMCC_WIN_TITLE",
                           @(XTERMCC_ICON_TITLE):              @"XTERMCC_ICON_TITLE",
                           @(XTERMCC_WINICON_TITLE):           @"XTERMCC_WINICON_TITLE",
-                          @(XTERMCC_INSBLNK):                 @"XTERMCC_INSBLNK",
+                          @(VT100CSI_ICH):                    @"VT100CSI_ICH",
                           @(XTERMCC_INSLN):                   @"XTERMCC_INSLN",
                           @(XTERMCC_DELCH):                   @"XTERMCC_DELCH",
                           @(XTERMCC_DELLN):                   @"XTERMCC_DELLN",

sources/VT100XtermParser.h

@@ -12,7 +12,7 @@
 #import "VT100Token.h"
 NS_INLINE BOOL isXTERM(unsigned char *code, int len) {
-    return (len >= 2 && code[0] == ESC && (code[1] == ']'));
+    return (len >= 2 && code[0] == VT100CC_ESC && (code[1] == ']'));
 }
 @interface VT100XtermParser : NSObject