>From 40f39973fc0399aa45775ca26564948f77678ab5 Mon Sep 17 00:00:00 2001 From: Bruno Haible Date: Sat, 4 Jan 2020 02:31:39 +0100 Subject: [PATCH 3/5] mbrtowc: Refactor to share code with mbrtoc32. * lib/mbrtowc-impl.h: New file, extracted from lib/mbrtowc.c. * lib/mbrtowc-impl-utf8.h: Likewise. * lib/mbrtowc.c (mbrtowc): Define macro FITS_IN_CHAR_TYPE. Include mbrtowc-impl.h. * modules/mbrtowc (Files): Add the new files. --- ChangeLog | 9 ++ lib/mbrtowc-impl-utf8.h | 138 ++++++++++++++++++ lib/mbrtowc-impl.h | 262 ++++++++++++++++++++++++++++++++++ lib/mbrtowc.c | 364 ++---------------------------------------------- modules/mbrtowc | 2 + 5 files changed, 420 insertions(+), 355 deletions(-) create mode 100644 lib/mbrtowc-impl-utf8.h create mode 100644 lib/mbrtowc-impl.h diff --git a/ChangeLog b/ChangeLog index c5ae7b7..f8c6793 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +2020-01-03 Bruno Haible + + mbrtowc: Refactor to share code with mbrtoc32. + * lib/mbrtowc-impl.h: New file, extracted from lib/mbrtowc.c. + * lib/mbrtowc-impl-utf8.h: Likewise. + * lib/mbrtowc.c (mbrtowc): Define macro FITS_IN_CHAR_TYPE. Include + mbrtowc-impl.h. + * modules/mbrtowc (Files): Add the new files. + 2020-01-03 Jim Meyering doc: fix time.texi wording diff --git a/lib/mbrtowc-impl-utf8.h b/lib/mbrtowc-impl-utf8.h new file mode 100644 index 0000000..a826b1b --- /dev/null +++ b/lib/mbrtowc-impl-utf8.h @@ -0,0 +1,138 @@ +/* Convert multibyte character to wide character. + Copyright (C) 1999-2002, 2005-2020 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* Written by Bruno Haible , 2008. */ + +/* This file contains the part of the body of the mbrtowc and mbrtoc32 functions + that handles the special case of the UTF-8 encoding. */ + + /* Cf. unistr/u8-mbtouc.c. */ + unsigned char c = (unsigned char) p[0]; + + if (c < 0x80) + { + if (pwc != NULL) + *pwc = c; + res = (c == 0 ? 0 : 1); + goto success; + } + if (c >= 0xc2) + { + if (c < 0xe0) + { + if (m == 1) + goto incomplete; + else /* m >= 2 */ + { + unsigned char c2 = (unsigned char) p[1]; + + if ((c2 ^ 0x80) < 0x40) + { + if (pwc != NULL) + *pwc = ((unsigned int) (c & 0x1f) << 6) + | (unsigned int) (c2 ^ 0x80); + res = 2; + goto success; + } + } + } + else if (c < 0xf0) + { + if (m == 1) + goto incomplete; + else + { + unsigned char c2 = (unsigned char) p[1]; + + if ((c2 ^ 0x80) < 0x40 + && (c >= 0xe1 || c2 >= 0xa0) + && (c != 0xed || c2 < 0xa0)) + { + if (m == 2) + goto incomplete; + else /* m >= 3 */ + { + unsigned char c3 = (unsigned char) p[2]; + + if ((c3 ^ 0x80) < 0x40) + { + unsigned int wc = + (((unsigned int) (c & 0x0f) << 12) + | ((unsigned int) (c2 ^ 0x80) << 6) + | (unsigned int) (c3 ^ 0x80)); + + if (FITS_IN_CHAR_TYPE (wc)) + { + if (pwc != NULL) + *pwc = wc; + res = 3; + goto success; + } + } + } + } + } + } + else if (c <= 0xf4) + { + if (m == 1) + goto incomplete; + else + { + unsigned char c2 = (unsigned char) p[1]; + + if ((c2 ^ 0x80) < 0x40 + && (c >= 0xf1 || c2 >= 0x90) + && (c < 0xf4 || (c == 0xf4 && c2 < 0x90))) + { + if (m == 2) + goto incomplete; + else + { + unsigned char c3 = (unsigned char) p[2]; + + if ((c3 ^ 0x80) < 0x40) + { + if (m == 3) + goto incomplete; + else /* m >= 4 */ + { + unsigned char c4 = (unsigned char) p[3]; + + if ((c4 ^ 0x80) < 0x40) + { + unsigned int wc = + (((unsigned int) (c & 0x07) << 18) + | ((unsigned int) (c2 ^ 0x80) << 12) + | ((unsigned int) (c3 ^ 0x80) << 6) + | (unsigned int) (c4 ^ 0x80)); + + if (FITS_IN_CHAR_TYPE (wc)) + { + if (pwc != NULL) + *pwc = wc; + res = 4; + goto success; + } + } + } + } + } + } + } + } + } + goto invalid; diff --git a/lib/mbrtowc-impl.h b/lib/mbrtowc-impl.h new file mode 100644 index 0000000..c970439 --- /dev/null +++ b/lib/mbrtowc-impl.h @@ -0,0 +1,262 @@ +/* Convert multibyte character to wide character. + Copyright (C) 1999-2002, 2005-2020 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* Written by Bruno Haible , 2008. */ + +/* This file contains the body of the mbrtowc and mbrtoc32 functions, + when GNULIB_defined_mbstate_t is defined. */ + + char *pstate = (char *)ps; + + if (s == NULL) + { + pwc = NULL; + s = ""; + n = 1; + } + + if (n == 0) + return (size_t)(-2); + + /* Here n > 0. */ + + if (pstate == NULL) + pstate = internal_state; + + { + size_t nstate = pstate[0]; + char buf[4]; + const char *p; + size_t m; + enc_t enc; + int res; + + switch (nstate) + { + case 0: + p = s; + m = n; + break; + case 3: + buf[2] = pstate[3]; + FALLTHROUGH; + case 2: + buf[1] = pstate[2]; + FALLTHROUGH; + case 1: + buf[0] = pstate[1]; + p = buf; + m = nstate; + buf[m++] = s[0]; + if (n >= 2 && m < 4) + { + buf[m++] = s[1]; + if (n >= 3 && m < 4) + buf[m++] = s[2]; + } + break; + default: + errno = EINVAL; + return (size_t)(-1); + } + + /* Here m > 0. */ + + enc = locale_encoding_classification (); + + if (enc == enc_utf8) /* UTF-8 */ + { + /* Achieve + - multi-thread safety and + - the ability to produce wide character values > WCHAR_MAX + by not calling mbtowc() at all. */ +#include "mbrtowc-impl-utf8.h" + } + else + { + /* The hidden internal state of mbtowc would make this function not + multi-thread safe. Achieve multi-thread safety through a lock. */ + wchar_t wc; + res = mbtowc_with_lock (&wc, p, m); + + if (res >= 0) + { + if ((wc == 0) != (res == 0)) + abort (); + if (pwc != NULL) + *pwc = wc; + goto success; + } + + /* mbtowc does not distinguish between invalid and incomplete multibyte + sequences. But mbrtowc needs to make this distinction. + There are two possible approaches: + - Use iconv() and its return value. + - Use built-in knowledge about the possible encodings. + Given the low quality of implementation of iconv() on the systems + that lack mbrtowc(), we use the second approach. + The possible encodings are: + - 8-bit encodings, + - EUC-JP, EUC-KR, GB2312, EUC-TW, BIG5, GB18030, SJIS, + - UTF-8 (already handled above). + Use specialized code for each. */ + if (m >= 4 || m >= MB_CUR_MAX) + goto invalid; + /* Here MB_CUR_MAX > 1 and 0 < m < 4. */ + switch (enc) + { + /* As a reference for this code, you can use the GNU libiconv + implementation. Look for uses of the RET_TOOFEW macro. */ + + case enc_eucjp: /* EUC-JP */ + { + if (m == 1) + { + unsigned char c = (unsigned char) p[0]; + + if ((c >= 0xa1 && c < 0xff) || c == 0x8e || c == 0x8f) + goto incomplete; + } + if (m == 2) + { + unsigned char c = (unsigned char) p[0]; + + if (c == 0x8f) + { + unsigned char c2 = (unsigned char) p[1]; + + if (c2 >= 0xa1 && c2 < 0xff) + goto incomplete; + } + } + goto invalid; + } + + case enc_94: /* EUC-KR, GB2312, BIG5 */ + { + if (m == 1) + { + unsigned char c = (unsigned char) p[0]; + + if (c >= 0xa1 && c < 0xff) + goto incomplete; + } + goto invalid; + } + + case enc_euctw: /* EUC-TW */ + { + if (m == 1) + { + unsigned char c = (unsigned char) p[0]; + + if ((c >= 0xa1 && c < 0xff) || c == 0x8e) + goto incomplete; + } + else /* m == 2 || m == 3 */ + { + unsigned char c = (unsigned char) p[0]; + + if (c == 0x8e) + goto incomplete; + } + goto invalid; + } + + case enc_gb18030: /* GB18030 */ + { + if (m == 1) + { + unsigned char c = (unsigned char) p[0]; + + if ((c >= 0x90 && c <= 0xe3) || (c >= 0xf8 && c <= 0xfe)) + goto incomplete; + } + else /* m == 2 || m == 3 */ + { + unsigned char c = (unsigned char) p[0]; + + if (c >= 0x90 && c <= 0xe3) + { + unsigned char c2 = (unsigned char) p[1]; + + if (c2 >= 0x30 && c2 <= 0x39) + { + if (m == 2) + goto incomplete; + else /* m == 3 */ + { + unsigned char c3 = (unsigned char) p[2]; + + if (c3 >= 0x81 && c3 <= 0xfe) + goto incomplete; + } + } + } + } + goto invalid; + } + + case enc_sjis: /* SJIS */ + { + if (m == 1) + { + unsigned char c = (unsigned char) p[0]; + + if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea) + || (c >= 0xf0 && c <= 0xf9)) + goto incomplete; + } + goto invalid; + } + + default: + /* An unknown multibyte encoding. */ + goto incomplete; + } + } + + success: + /* res >= 0 is the corrected return value of + mbtowc_with_lock (&wc, p, m). */ + if (nstate >= (res > 0 ? res : 1)) + abort (); + res -= nstate; + pstate[0] = 0; + return res; + + incomplete: + { + size_t k = nstate; + /* Here 0 <= k < m < 4. */ + pstate[++k] = s[0]; + if (k < m) + { + pstate[++k] = s[1]; + if (k < m) + pstate[++k] = s[2]; + } + if (k != m) + abort (); + } + pstate[0] = m; + return (size_t)(-2); + + invalid: + errno = EILSEQ; + /* The conversion state is undefined, says POSIX. */ + return (size_t)(-1); + } diff --git a/lib/mbrtowc.c b/lib/mbrtowc.c index fdef8f9..6cb5267 100644 --- a/lib/mbrtowc.c +++ b/lib/mbrtowc.c @@ -20,13 +20,9 @@ /* Specification. */ #include -#if MBRTOWC_IN_C_LOCALE_MAYBE_EILSEQ -# include "hard-locale.h" -# include -#endif - #if GNULIB_defined_mbstate_t -/* Implement mbrtowc() on top of mbtowc(). */ +/* Implement mbrtowc() on top of mbtowc() for the non-UTF-8 locales + and directly for the UTF-8 locales. */ # include # include @@ -72,360 +68,18 @@ static char internal_state[4]; size_t mbrtowc (wchar_t *pwc, const char *s, size_t n, mbstate_t *ps) { - char *pstate = (char *)ps; - - if (s == NULL) - { - pwc = NULL; - s = ""; - n = 1; - } - - if (n == 0) - return (size_t)(-2); - - /* Here n > 0. */ - - if (pstate == NULL) - pstate = internal_state; - - { - size_t nstate = pstate[0]; - char buf[4]; - const char *p; - size_t m; - enc_t enc; - int res; - - switch (nstate) - { - case 0: - p = s; - m = n; - break; - case 3: - buf[2] = pstate[3]; - FALLTHROUGH; - case 2: - buf[1] = pstate[2]; - FALLTHROUGH; - case 1: - buf[0] = pstate[1]; - p = buf; - m = nstate; - buf[m++] = s[0]; - if (n >= 2 && m < 4) - { - buf[m++] = s[1]; - if (n >= 3 && m < 4) - buf[m++] = s[2]; - } - break; - default: - errno = EINVAL; - return (size_t)(-1); - } - - /* Here m > 0. */ - - enc = locale_encoding_classification (); - - if (enc == enc_utf8) /* UTF-8 */ - { - /* Achieve multi-thread safety by not calling mbtowc() at all. */ - /* Cf. unistr/u8-mbtouc.c. */ - unsigned char c = (unsigned char) p[0]; - - if (c < 0x80) - { - if (pwc != NULL) - *pwc = c; - res = (c == 0 ? 0 : 1); - goto success; - } - if (c >= 0xc2) - { - if (c < 0xe0) - { - if (m == 1) - goto incomplete; - else /* m >= 2 */ - { - unsigned char c2 = (unsigned char) p[1]; - - if ((c2 ^ 0x80) < 0x40) - { - if (pwc != NULL) - *pwc = ((unsigned int) (c & 0x1f) << 6) - | (unsigned int) (c2 ^ 0x80); - res = 2; - goto success; - } - } - } - else if (c < 0xf0) - { - if (m == 1) - goto incomplete; - else - { - unsigned char c2 = (unsigned char) p[1]; - - if ((c2 ^ 0x80) < 0x40 - && (c >= 0xe1 || c2 >= 0xa0) - && (c != 0xed || c2 < 0xa0)) - { - if (m == 2) - goto incomplete; - else /* m >= 3 */ - { - unsigned char c3 = (unsigned char) p[2]; - - if ((c3 ^ 0x80) < 0x40) - { - unsigned int wc - = (((unsigned int) (c & 0x0f) << 12) - | ((unsigned int) (c2 ^ 0x80) << 6) - | (unsigned int) (c3 ^ 0x80)); - if (wc <= WCHAR_MAX) - { - if (pwc != NULL) - *pwc = wc; - res = 3; - goto success; - } - } - } - } - } - } - else if (c <= 0xf4) - { - if (m == 1) - goto incomplete; - else - { - unsigned char c2 = (unsigned char) p[1]; - - if ((c2 ^ 0x80) < 0x40 - && (c >= 0xf1 || c2 >= 0x90) - && (c < 0xf4 || (c == 0xf4 && c2 < 0x90))) - { - if (m == 2) - goto incomplete; - else - { - unsigned char c3 = (unsigned char) p[2]; - - if ((c3 ^ 0x80) < 0x40) - { - if (m == 3) - goto incomplete; - else /* m >= 4 */ - { - unsigned char c4 = (unsigned char) p[3]; - - if ((c4 ^ 0x80) < 0x40) - { - unsigned int wc - = (((unsigned int) (c & 0x07) << 18) - | ((unsigned int) (c2 ^ 0x80) - << 12) - | ((unsigned int) (c3 ^ 0x80) << 6) - | (unsigned int) (c4 ^ 0x80)); - if (wc <= WCHAR_MAX) - { - if (pwc != NULL) - *pwc = wc; - res = 4; - goto success; - } - } - } - } - } - } - } - } - } - goto invalid; - } - else - { - /* The hidden internal state of mbtowc would make this function not - multi-thread safe. Achieve multi-thread safety through a lock. */ - res = mbtowc_with_lock (pwc, p, m); - - if (res >= 0) - { - if (pwc != NULL && ((*pwc == 0) != (res == 0))) - abort (); - goto success; - } - - /* mbtowc does not distinguish between invalid and incomplete multibyte - sequences. But mbrtowc needs to make this distinction. - There are two possible approaches: - - Use iconv() and its return value. - - Use built-in knowledge about the possible encodings. - Given the low quality of implementation of iconv() on the systems - that lack mbrtowc(), we use the second approach. - The possible encodings are: - - 8-bit encodings, - - EUC-JP, EUC-KR, GB2312, EUC-TW, BIG5, GB18030, SJIS, - - UTF-8 (already handled above). - Use specialized code for each. */ - if (m >= 4 || m >= MB_CUR_MAX) - goto invalid; - /* Here MB_CUR_MAX > 1 and 0 < m < 4. */ - switch (enc) - { - /* As a reference for this code, you can use the GNU libiconv - implementation. Look for uses of the RET_TOOFEW macro. */ - - case enc_eucjp: /* EUC-JP */ - { - if (m == 1) - { - unsigned char c = (unsigned char) p[0]; - - if ((c >= 0xa1 && c < 0xff) || c == 0x8e || c == 0x8f) - goto incomplete; - } - if (m == 2) - { - unsigned char c = (unsigned char) p[0]; - - if (c == 0x8f) - { - unsigned char c2 = (unsigned char) p[1]; - - if (c2 >= 0xa1 && c2 < 0xff) - goto incomplete; - } - } - goto invalid; - } - - case enc_94: /* EUC-KR, GB2312, BIG5 */ - { - if (m == 1) - { - unsigned char c = (unsigned char) p[0]; - - if (c >= 0xa1 && c < 0xff) - goto incomplete; - } - goto invalid; - } - - case enc_euctw: /* EUC-TW */ - { - if (m == 1) - { - unsigned char c = (unsigned char) p[0]; - - if ((c >= 0xa1 && c < 0xff) || c == 0x8e) - goto incomplete; - } - else /* m == 2 || m == 3 */ - { - unsigned char c = (unsigned char) p[0]; - - if (c == 0x8e) - goto incomplete; - } - goto invalid; - } - - case enc_gb18030: /* GB18030 */ - { - if (m == 1) - { - unsigned char c = (unsigned char) p[0]; - - if ((c >= 0x90 && c <= 0xe3) || (c >= 0xf8 && c <= 0xfe)) - goto incomplete; - } - else /* m == 2 || m == 3 */ - { - unsigned char c = (unsigned char) p[0]; - - if (c >= 0x90 && c <= 0xe3) - { - unsigned char c2 = (unsigned char) p[1]; - - if (c2 >= 0x30 && c2 <= 0x39) - { - if (m == 2) - goto incomplete; - else /* m == 3 */ - { - unsigned char c3 = (unsigned char) p[2]; - - if (c3 >= 0x81 && c3 <= 0xfe) - goto incomplete; - } - } - } - } - goto invalid; - } - - case enc_sjis: /* SJIS */ - { - if (m == 1) - { - unsigned char c = (unsigned char) p[0]; - - if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea) - || (c >= 0xf0 && c <= 0xf9)) - goto incomplete; - } - goto invalid; - } - - default: - /* An unknown multibyte encoding. */ - goto incomplete; - } - } - - success: - /* res >= 0 is the corrected return value of mbtowc (pwc, p, m). */ - if (nstate >= (res > 0 ? res : 1)) - abort (); - res -= nstate; - pstate[0] = 0; - return res; - - incomplete: - { - size_t k = nstate; - /* Here 0 <= k < m < 4. */ - pstate[++k] = s[0]; - if (k < m) - { - pstate[++k] = s[1]; - if (k < m) - pstate[++k] = s[2]; - } - if (k != m) - abort (); - } - pstate[0] = m; - return (size_t)(-2); - - invalid: - errno = EILSEQ; - /* The conversion state is undefined, says POSIX. */ - return (size_t)(-1); - } +# define FITS_IN_CHAR_TYPE(wc) ((wc) <= WCHAR_MAX) +# include "mbrtowc-impl.h" } #else /* Override the system's mbrtowc() function. */ +# if MBRTOWC_IN_C_LOCALE_MAYBE_EILSEQ +# include "hard-locale.h" +# include +# endif + # undef mbrtowc size_t diff --git a/modules/mbrtowc b/modules/mbrtowc index 22afc96..ee2e649 100644 --- a/modules/mbrtowc +++ b/modules/mbrtowc @@ -3,6 +3,8 @@ mbrtowc() function: convert multibyte character to wide character. Files: lib/mbrtowc.c +lib/mbrtowc-impl.h +lib/mbrtowc-impl-utf8.h lib/lc-charset-dispatch.h lib/lc-charset-dispatch.c lib/mbtowc-lock.h -- 2.7.4