[Qemacs-commit] qemacs qe.c qe.h variables.c clang.c list.c she...

[Charlie Gordon]

CVSROOT:        /cvsroot/qemacs
Module name:    qemacs
Changes by:     Charlie Gordon <chqrlie>        08/03/29 18:06:42

Modified files:
        .              : qe.c qe.h variables.c clang.c list.c shell.c 
                         tty.c 

Log message:
        fixed crash bug on forward_block in text mode:
         - renamed get_colorized_line() to generic_get_colorized_line
         - added get_non_colorized_line() for uncolored modes
         - renamed EditState::get_colorized_line_func to get_colorized_line
           make this function pointer always non NULL
        
        fixed bug in global-set-key: current bindings could not be overloaded
        
        better tty charset detection and handling:
         - always use utf8 when output is not a tty
         - allow override with -c CHARSET command line option
         - added state variable show-unicode to display non latin1 as \uXXXX
         - added command line argument -c CHARSET (--ttycharset) to specify the 
           charset of the tty. tty charset is reported in *messages* buffer at 
startup
         - display unsupported glyphs as half-width or full-width question mark 
pairs
        
        improved tty output: use kill_end_of_line escape sequence

CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/qemacs/qe.c?cvsroot=qemacs&r1=1.71&r2=1.72
http://cvs.savannah.gnu.org/viewcvs/qemacs/qe.h?cvsroot=qemacs&r1=1.68&r2=1.69
http://cvs.savannah.gnu.org/viewcvs/qemacs/variables.c?cvsroot=qemacs&r1=1.2&r2=1.3
http://cvs.savannah.gnu.org/viewcvs/qemacs/clang.c?cvsroot=qemacs&r1=1.32&r2=1.33
http://cvs.savannah.gnu.org/viewcvs/qemacs/list.c?cvsroot=qemacs&r1=1.8&r2=1.9
http://cvs.savannah.gnu.org/viewcvs/qemacs/shell.c?cvsroot=qemacs&r1=1.43&r2=1.44
http://cvs.savannah.gnu.org/viewcvs/qemacs/tty.c?cvsroot=qemacs&r1=1.36&r2=1.37

Patches:
Index: qe.c
===================================================================
RCS file: /cvsroot/qemacs/qemacs/qe.c,v
retrieving revision 1.71
retrieving revision 1.72
diff -u -b -r1.71 -r1.72
--- qe.c        28 Mar 2008 08:35:06 -0000      1.71
+++ qe.c        29 Mar 2008 18:06:41 -0000      1.72
@@ -180,16 +180,16 @@
     for (i = 0; i < nb_keys; i++) {
         p->keys[i] = keys[i];
     }
-    /* find position: mode keys should be before generic keys */
-    if (m == NULL) {
+    /* find position: mode keys should be before generic keys, but
+     * bindings must be prepended to override previous bindings
+     */
         lp = &qs->first_key;
-        while (*lp != NULL) lp = &(*lp)->next;
-        *lp = p;
-        p->next = NULL;
-    } else {
-        p->next = qs->first_key;
-        qs->first_key = p;
+    if (m == NULL) {
+        while (*lp != NULL && (*lp)->mode != NULL)
+            lp = &(*lp)->next;
     }
+    p->next = *lp;
+    *lp = p;
     return 0;
 }
 
@@ -3052,7 +3052,7 @@
 
 #define COLORIZED_LINE_PREALLOC_SIZE 64
 
-int get_colorized_line(EditState *s, unsigned int *buf, int buf_size,
+int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size,
                        int *offsetp, int line_num)
 {
     int len, l, line, col, offset;
@@ -3129,12 +3129,12 @@
     s->colorize_nb_lines = 0;
     s->colorize_nb_valid_lines = 0;
     s->colorize_max_valid_offset = INT_MAX;
-    s->get_colorized_line_func = NULL;
+    s->get_colorized_line = get_non_colorized_line;
     s->colorize_func = NULL;
 
     if (colorize_func) {
         eb_add_callback(s->b, colorize_callback, s);
-        s->get_colorized_line_func = get_colorized_line;
+        s->get_colorized_line = generic_get_colorized_line;
         s->colorize_func = colorize_func;
     }
 }
@@ -3143,10 +3143,22 @@
 
 void set_colorize_func(EditState *s, ColorizeFunc colorize_func)
 {
+    s->get_colorized_line = get_non_colorized_line;
 }
 
 #endif /* CONFIG_TINY */
 
+int get_non_colorized_line(EditState *s, unsigned int *buf, int buf_size,
+                           int *offsetp, int line_num)
+{
+    /* compute line color */
+    int len = eb_get_line(s->b, buf, buf_size, offsetp);
+    // XXX: should force \0 instead of \n
+    buf[len] = '\n';
+
+    return len;
+}
+
 #define RLE_EMBEDDINGS_SIZE    128
 #define COLORED_MAX_LINE_SIZE  1024
 
@@ -3161,7 +3173,7 @@
     int char_index, colored_nb_chars;
 
     line_num = 0; /* avoid warning */
-    if (s->line_numbers || s->get_colorized_line_func) {
+    if (s->line_numbers || s->get_colorized_line != get_non_colorized_line) {
         eb_get_pos(s->b, &line_num, &col_num, offset);
     }
 
@@ -3209,8 +3221,8 @@
     /* colorize */
     colored_nb_chars = 0;
     offset0 = offset;
-    if (s->get_colorized_line_func) {
-        colored_nb_chars = s->get_colorized_line_func(s, colored_chars,
+    if (s->get_colorized_line != get_non_colorized_line) {
+        colored_nb_chars = s->get_colorized_line(s, colored_chars,
                                                       countof(colored_chars),
                                                       &offset0, line_num);
     }
@@ -3218,7 +3230,7 @@
 #if 1
     /* colorize regions */
     if (s->curline_style || s->region_style) {
-        if (!s->get_colorized_line_func) {
+        if (s->get_colorized_line == get_non_colorized_line) {
             offset0 = offset;
             colored_nb_chars = eb_get_line(s->b, colored_chars,
                 countof(colored_chars), &offset0);
@@ -3288,7 +3300,7 @@
                 /* currently, we cannot display these chars */
                 display_printf(ds, offset0, offset, "\\U%08x", c);
             } else
-            if (c >= 256 && s->screen->charset != &charset_utf8) {
+            if (c >= 256 && s->qe_state->show_unicode == 1) {
                 display_printf(ds, offset0, offset, "\\u%04x", c);
             } else {
                 display_char_bidir(ds, offset0, offset, embedding_level, c);
@@ -7449,11 +7461,19 @@
             "/usr/lib/qe");
 }
 
+void set_tty_charset(const char *name)
+{
+    qe_free(&qe_state.tty_charset);
+    qe_state.tty_charset = qe_strdup(name);
+}
+
 static CmdOptionDef cmd_options[] = {
     { "help", "h", NULL, 0, "display this help message and exit",
       { .func_noarg = show_usage }},
     { "no-init-file", "q", NULL, CMD_OPT_BOOL, "do not load config files",
       { .int_ptr = &no_init_file }},
+    { "ttycharset", "c", "CHARSET", CMD_OPT_ARG, "specify tty charset",
+      { .func_arg = set_tty_charset }},
     { "user", "u", "USER", CMD_OPT_ARG, "load ~USER/.qe/config instead of your 
own",
       { .func_arg = set_user_option }},
     { "version", "V", NULL, 0, "display version information and exit",

Index: qe.h
===================================================================
RCS file: /cvsroot/qemacs/qemacs/qe.h,v
retrieving revision 1.68
retrieving revision 1.69
diff -u -b -r1.68 -r1.69
--- qe.h        28 Mar 2008 12:21:39 -0000      1.68
+++ qe.h        29 Mar 2008 18:06:41 -0000      1.69
@@ -940,7 +940,7 @@
     int mouse_force_highlight; /* if true, mouse can force highlight
                                   (list mode only) */
     /* low level colorization function */
-    GetColorizedLineFunc get_colorized_line_func;
+    GetColorizedLineFunc get_colorized_line;
     /* colorization function */
     ColorizeFunc colorize_func;
     /* default text style */
@@ -1153,6 +1153,13 @@
     int hide_status; /* true if status should be hidden */
     int complete_refresh;
     int is_full_screen;
+    /* select display aspect for non-latin1 characters:
+     * 0 (auto) -> display as unicode on utf-8 capable ttys and x11
+     * 1 (nc) -> display as ? or ?? non character symbols
+     * 2 (escape) -> display as \uXXXX escape sequence
+     */
+    int show_unicode;
+    
     /* commands */
     int flag_split_window_change_focus;
     int backspace_is_control_h;
@@ -1171,6 +1178,7 @@
     int yank_current;
     int argc;  /* command line arguments */
     char **argv;
+    char *tty_charset;
     char res_path[1024];        /* exported as QEPATH */
     char status_shadow[MAX_SCREEN_WIDTH];
     QErrorContext ec;
@@ -1535,7 +1543,9 @@
 int text_display(EditState *s, DisplayState *ds, int offset);
 
 void set_colorize_func(EditState *s, ColorizeFunc colorize_func);
-int get_colorized_line(EditState *s, unsigned int *buf, int buf_size,
+int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size,
+                               int *offsetp, int line_num);
+int get_non_colorized_line(EditState *s, unsigned int *buf, int buf_size,
                        int *offsetp, int line_num);
 
 void do_char(EditState *s, int key, int argval);
@@ -1670,6 +1680,7 @@
 int parse_config_file(EditState *s, const char *filename);
 int parse_command_line(int argc, char **argv);
 void set_user_option(const char *user);
+void set_tty_charset(const char *name);
 
 /* hex.c */
 

Index: variables.c
===================================================================
RCS file: /cvsroot/qemacs/qemacs/variables.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -b -r1.2 -r1.3
--- variables.c 18 Jan 2008 17:04:58 -0000      1.2
+++ variables.c 29 Mar 2008 18:06:41 -0000      1.3
@@ -34,6 +34,8 @@
     S_VAR( "ignore-spaces", ignore_spaces, VAR_NUMBER, VAR_RW )
     S_VAR( "hilite-region", hilite_region, VAR_NUMBER, VAR_RW )
     S_VAR( "mmap-threshold", mmap_threshold, VAR_NUMBER, VAR_RW )
+    S_VAR( "show-unicode", show_unicode, VAR_NUMBER, VAR_RW )
+
 
     //B_VAR( "screen-charset", charset, VAR_NUMBER, VAR_RW )
 

Index: clang.c
===================================================================
RCS file: /cvsroot/qemacs/qemacs/clang.c,v
retrieving revision 1.32
retrieving revision 1.33
diff -u -b -r1.32 -r1.33
--- clang.c     28 Mar 2008 12:23:53 -0000      1.32
+++ clang.c     29 Mar 2008 18:06:41 -0000      1.33
@@ -379,7 +379,7 @@
         line_num--;
         offsetl = eb_prev_line(s->b, offsetl);
         offset1 = offsetl;
-        len = get_colorized_line(s, buf, countof(buf), &offset1, line_num);
+        len = s->get_colorized_line(s, buf, countof(buf), &offset1, line_num);
         /* store indent position */
         pos1 = find_indent1(s, buf);
         p = buf + len;
@@ -497,7 +497,7 @@
  end_parse:
     /* compute special cases which depend on the chars on the current line */
     offset1 = offset;
-    len = get_colorized_line(s, buf, countof(buf), &offset1, line_num1);
+    len = s->get_colorized_line(s, buf, countof(buf), &offset1, line_num1);
 
     if (stack_ptr == 0) {
         if (!pos && lpos >= 0) {
@@ -605,7 +605,7 @@
     eb_get_pos(s->b, &line_num, &col_num, s->offset);
     offset = eb_goto_bol2(s->b, s->offset, &pos);
     offset1 = offset;
-    len = get_colorized_line(s, buf, countof(buf), &offset1, line_num);
+    len = s->get_colorized_line(s, buf, countof(buf), &offset1, line_num);
     style = buf[pos] >> STYLE_SHIFT;
     level = 0;
 
@@ -617,7 +617,7 @@
                 line_num--;
                 offset = eb_prev_line(s->b, offset);
                 offset1 = offset;
-                pos = get_colorized_line(s, buf, countof(buf), &offset1, 
line_num);
+                pos = s->get_colorized_line(s, buf, countof(buf), &offset1, 
line_num);
                 continue;
             }
             c = buf[--pos];
@@ -670,7 +670,7 @@
                 if (offset >= s->b->total_size)
                     break;
                 offset1 = offset;
-                len = get_colorized_line(s, buf, countof(buf), &offset1, 
line_num);
+                len = s->get_colorized_line(s, buf, countof(buf), &offset1, 
line_num);
                 continue;
             }
             c = buf[pos];

Index: list.c
===================================================================
RCS file: /cvsroot/qemacs/qemacs/list.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -b -r1.8 -r1.9
--- list.c      9 Jan 2008 13:41:41 -0000       1.8
+++ list.c      29 Mar 2008 18:06:41 -0000      1.9
@@ -80,7 +80,7 @@
 {
     s->wrap = WRAP_TRUNCATE;
     s->interactive = 1;
-    s->get_colorized_line_func = list_get_colorized_line;
+    s->get_colorized_line = list_get_colorized_line;
     return 0;
 }
 

Index: shell.c
===================================================================
RCS file: /cvsroot/qemacs/qemacs/shell.c,v
retrieving revision 1.43
retrieving revision 1.44
diff -u -b -r1.43 -r1.44
--- shell.c     25 Mar 2008 11:04:59 -0000      1.43
+++ shell.c     29 Mar 2008 18:06:42 -0000      1.44
@@ -1436,7 +1436,7 @@
     s->wrap = WRAP_TRUNCATE;
     s->interactive = 1;
     set_colorize_func(s, NULL);
-    s->get_colorized_line_func = shell_get_colorized_line;
+    s->get_colorized_line = shell_get_colorized_line;
     return 0;
 }
 

Index: tty.c
===================================================================
RCS file: /cvsroot/qemacs/qemacs/tty.c,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -b -r1.36 -r1.37
--- tty.c       11 Jan 2008 11:29:29 -0000      1.36
+++ tty.c       29 Mar 2008 18:06:42 -0000      1.37
@@ -35,7 +35,9 @@
 
 typedef unsigned int TTYChar;
 #define TTYCHAR(ch,fg,bg)   ((ch) | ((fg) << 16) | ((bg) << 24))
+#define TTYCHAR2(ch,col)    ((ch) | ((col) << 16))
 #define TTYCHAR_GETCH(cc)   ((cc) & 0xFFFF)
+#define TTYCHAR_GETCOL(cc)  (((cc) >> 16) & 0xFFFF)
 #define TTYCHAR_GETFG(cc)   (((cc) >> 16) & 0xFF)
 #define TTYCHAR_GETBG(cc)   (((cc) >> 24) & 0xFF)
 #define TTYCHAR_DEFAULT     TTYCHAR(' ', 0, 0)
@@ -174,21 +176,29 @@
                );
 #endif
 
-    s->charset = &charset_vt100;
+    /* Get charset from command line option */
+    s->charset = find_charset(qe_state.tty_charset);
 
-    /* test UTF8 support by looking at the cursor position (idea from
-       Ricardas Cepas <address@hidden>). Since uClibc actually tests
-       to ensure that the format string is a valid multibyte sequence
-       in the current locale (ANSI/ISO C99), use a format specifier of
-       %s to avoid printf() failing with EILSEQ. */
-    /* CG: Should also have a command line switch */
+    if (!s->charset && !isatty(fileno(s->STDOUT)))
+        s->charset = &charset_utf8;
+
+    if (!s->charset) {
+        s->charset = &charset_8859_1;
+
+        /* Test UTF8 support by looking at the cursor position (idea
+         * from Ricardas Cepas <address@hidden>). Since uClibc actually
+         * tests to ensure that the format string is a valid multibyte
+         * sequence in the current locale (ANSI/ISO C99), use a format
+         * specifier of %s to avoid printf() failing with EILSEQ.
+         */
     if (tty_state.term_code != TERM_CYGWIN) {
         int y, x, n;
 
         /*               ^X  ^Z    ^M   \170101  */
         //printf("%s", "\030\032" "\r\xEF\x81\x81" "\033[6n\033D");
         /* Just print utf-8 encoding for eacute and check cursor position */
-        TTY_FPRINTF(s->STDOUT, "%s", "\030\032" "\r\xC3\xA9" "\033[6n\033D");
+            TTY_FPRINTF(s->STDOUT, "%s",
+                        "\030\032" "\r\xC3\xA9" "\033[6n\033D");
         fflush(s->STDOUT);
         n = fscanf(s->STDIN, "\033[%u;%u", &y, &x);  /* get cursor position */
         TTY_FPRINTF(s->STDOUT, "\r   \r");        /* go back, erase 3 chars */
@@ -196,6 +206,9 @@
             s->charset = &charset_utf8;
         }
     }
+    }
+    put_status(NULL, "tty charset: %s", s->charset->name);
+
     atexit(tty_term_exit);
 
     sig.sa_handler = tty_resize;
@@ -707,9 +720,8 @@
 static void tty_term_flush(QEditScreen *s)
 {
     TTYState *ts = s->private;
-    TTYChar *ptr, *ptr1, *ptr2, cc;
-    int y, shadow, ch, bgcolor, fgcolor, shifted, nc;
-    char buf[10];
+    TTYChar *ptr, *ptr1, *ptr2, *ptr3, *ptr4, cc, blankcc;
+    int y, shadow, ch, bgcolor, fgcolor, shifted;
 
     bgcolor = -1;
     fgcolor = -1;
@@ -739,6 +751,12 @@
             ptr2[shadow] = cc;
 
             /* ptr1 points to first difference on row */
+            /* find the last non blank char on row */
+            ptr3 = ptr2;
+            blankcc = TTYCHAR2(' ', TTYCHAR_GETCOL(ptr2[-1]));
+            while (ptr3 > ptr1 && ptr3[-1] == blankcc) {
+                --ptr3;
+            }
             /* scan for last difference on row: */
             while (ptr2 > ptr1 && ptr2[-1] == ptr2[shadow - 1]) {
                 --ptr2;
@@ -748,8 +766,11 @@
 
             TTY_FPRINTF(s->STDOUT, "\033[%d;%dH", y + 1, ptr1 - ptr + 1);
 
-            /* CG: should scan for sequences of blanks */
-            while (ptr1 < ptr2) {
+            ptr4 = ptr2;
+            if (ptr2 > ptr3 + 3)
+                ptr4 = ptr3;
+
+            while (ptr1 < ptr4) {
                 cc = *ptr1;
                 ptr1[shadow] = cc;
                 ptr1++;
@@ -791,7 +812,7 @@
                         // was in qemacs-0.3.1.g2.gw/tty.c:
                         // if (cc == 0x2500)
                         //    printf("\016x\017");
-                        /* s->charset is either vt100 or utf-8 */
+                        /* s->charset is either latin1 or utf-8 */
                         if (shifted) {
                             TTY_FPUTS("\033(B", s->STDOUT);
                             shifted = 0;
@@ -810,9 +831,28 @@
                         //    }
                         //} else
                         {
-                            nc = unicode_to_charset(buf, cc, s->charset);
+                            u8 buf[10], *q;
+                            int nc;
+
+                            //nc = unicode_to_charset(buf, ch, s->charset);
+
+                            q = s->charset->encode_func(s->charset, buf, ch);
+                            if (!q) {
+                                if (s->charset == &charset_8859_1) {
+                                    /* upside down question */
+                                    buf[0] = 0xBF;
+                                } else {
+                                    buf[0] = '?';
+                                }
+                                q = buf + 1;
+                                if (tty_term_glyph_width(s, ch) == 2) {
+                                    *q++ = '?';
+                                }
+                            }
+
+                            nc = q - buf;
                             if (nc == 1) {
-                                TTY_PUTC(*(u8 *)buf, s->STDOUT);
+                                TTY_PUTC(*buf, s->STDOUT);
                             } else
                             {
                                 TTY_FWRITE(buf, 1, nc, s->STDOUT);
@@ -825,6 +865,19 @@
                 TTY_FPUTS("\033(B", s->STDOUT);
                 shifted = 0;
             }
+            if (ptr1 < ptr2) {
+                /* More differences to synch in shadow, erase eol */
+                cc = *ptr1;
+                if (bgcolor != (int)TTYCHAR_GETBG(cc)) {
+                    bgcolor = TTYCHAR_GETBG(cc);
+                    TTY_FPRINTF(s->STDOUT, "\033[%dm", 40 + bgcolor);
+                }
+                TTY_FPUTS("\033[K", s->STDOUT);
+                while (ptr1 < ptr2) {
+                    ptr1[shadow] = cc;
+                    ptr1++;
+                }
+            }
         }
     }