Ticket #7423: terminal-utf8-spaces-v3.patch

src/apps/terminal/UTF8Char.h

@@ -7 +7 @@
 
 #include <ctype.h>
 #include <string.h>
+#include <stdio.h>
 
-
 struct UTF8Char {
     char    bytes[4];
 
+    
     UTF8Char()
     {
     }
+    
 
     UTF8Char(char c)
     {
         bytes[0] = c;
     }
+    
+    
+    UTF8Char(const char* c)
+    {
+        SetTo(c, ByteCount(*c));
+    }
+    
 
     UTF8Char(const char* c, int32 count)
     {
         SetTo(c, count);
     }
 
+
     void SetTo(const char* c, int32 count)
     {
         bytes[0] = c[0];
@@ -39 +49 @@
         }
     }
 
+
     static int32 ByteCount(char firstChar)
     {
         // Note, this does not recognize invalid chars
@@ -50 +61 @@
         return c < 0xf0 ? 3 : 4;
     }
 
+
     int32 ByteCount() const
     {
         return ByteCount(bytes[0]);
     }
 
+
     bool IsFullWidth() const
     {
         // TODO: Implement!
         return false;
     }
 
+
     bool IsSpace() const
     {
-        // TODO: Support multi-byte chars!
-        return ByteCount() == 1 ? isspace(bytes[0]) : false;
+        if (ByteCount() == 1) {
+            return isspace(bytes[0]);
+        } else if (ByteCount() == 2) {
+            uchar b0 = bytes[0];
+            uchar b1 = bytes[1];
+            
+            // U+00A0 (no-break space), U+0085 (next line)
+            if (b0 == 0xc2)
+                return (b1 == 0xa0 || b1 == 0x85);
+            
+            return false;
+        } else if (ByteCount() == 3) {
+            uchar b0 = bytes[0];
+            uchar b1 = bytes[1];
+            uchar b2 = bytes[2];
+            
+            // most multibyte space chars are all in one range (0x2000-0x200B)
+            if (b0 == 0xe2 && b1 == 0x80)
+                return (b2 >= 0x80 && b2 <= 0x8f);
+            
+            // U+1680 (ogham space mark)
+            if (b0 == 0xe1 && b1 == 0x9a)
+                return (b2 == 0x80);
+            
+            // U+2028 (line separator), U+2029 (paragraph separator),
+            // U+202F (narrow no-break space)
+            if (b0 == 0xe2 && b1 == 0x80)
+                return (b2 == 0xa8 || b2 == 0xa9 || b2 == 0xaf);
+            
+            // U+205F (medium mathematical space)
+            if (b0 == 0xe2 && b1 == 0x81)
+                return (b2 == 0x9f);
+            
+            // U+3000 (ideographic space)
+            if (b0 == 0xe3 && b1 == 0x80)
+                return (b2 == 0x80);
+            
+            // U+FEFF (zero-width non-breaking space)
+            if (b0 == 0xef && b1 == 0xbb)
+                return (b2 == 0xbf);
+            
+            return false;
+        } else {
+            return false;
+        }
     }
 
     UTF8Char ToLower() const
@@ -76 +133 @@
         return UTF8Char((char)tolower(bytes[0]));
     }
 
+
     bool operator==(const UTF8Char& other) const
     {
         int32 byteCount = ByteCount();
@@ -91 +149 @@
         return equals;
     }
 
+
     bool operator!=(const UTF8Char& other) const
     {
         return !(*this == other);

src/apps/terminal/TermView.cpp

@@ -405 +405 @@
 
     virtual int Classify(const char* character)
     {
-        // TODO: Deal correctly with non-ASCII chars.
+        UTF8Char u8c(character);
+        
+        if (u8c.IsSpace())
+            return CHAR_TYPE_SPACE;
+        
         char c = *character;
-        if (UTF8Char::ByteCount(c) > 1)
+        if (u8c.ByteCount() > 1 || isalnum(c) || strchr(fSpecialWordChars, c) != NULL)
             return CHAR_TYPE_WORD_CHAR;
 
-        if (isspace(c))
-            return CHAR_TYPE_SPACE;
-        if (isalnum(c) || strchr(fSpecialWordChars, c) != NULL)
-            return CHAR_TYPE_WORD_CHAR;
-
         return CHAR_TYPE_WORD_DELIMITER;
     }