[PATCH] mbrtowc: port better to narrow-wchar_t platforms

[Paul Eggert]

* lib/mbrtowc.c (mbrtowc): On platforms like AIX 7.2, where
wchar_t is too narrow to represent all the Unicode characters,
consider a byte sequence for an out-of-wchar_t-range character to
be an encoding error.  This fixes grep’s surrogate-pair test
failure on AIX 7.2.
---
 ChangeLog     |  9 +++++++++
 lib/mbrtowc.c | 37 ++++++++++++++++++++++++-------------
 2 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 40f74abdd..70b46625f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2019-12-26  Paul Eggert  <address@hidden>
+
+       mbrtowc: port better to narrow-wchar_t platforms
+       * lib/mbrtowc.c (mbrtowc): On platforms like AIX 7.2, where
+       wchar_t is too narrow to represent all the Unicode characters,
+       consider a byte sequence for an out-of-wchar_t-range character to
+       be an encoding error.  This fixes grep’s surrogate-pair test
+       failure on AIX 7.2.
+
 2019-12-24  Bruno Haible  <address@hidden>
 
        localcharset: Avoid referencing rpl_setlocale on native Windows.
diff --git a/lib/mbrtowc.c b/lib/mbrtowc.c
index 1aad22cd5..38301dfba 100644
--- a/lib/mbrtowc.c
+++ b/lib/mbrtowc.c
@@ -214,12 +214,17 @@ mbrtowc (wchar_t *pwc, const char *s, size_t n, mbstate_t 
*ps)
 
                             if ((c3 ^ 0x80) < 0x40)
                               {
-                                if (pwc != NULL)
-                                  *pwc = ((unsigned int) (c & 0x0f) << 12)
-                                         | ((unsigned int) (c2 ^ 0x80) << 6)
-                                         | (unsigned int) (c3 ^ 0x80);
-                                res = 3;
-                                goto success;
+                                unsigned int wc
+                                  = (((unsigned int) (c & 0x0f) << 12)
+                                     | ((unsigned int) (c2 ^ 0x80) << 6)
+                                     | (unsigned int) (c3 ^ 0x80));
+                                if (wc <= WCHAR_MAX)
+                                  {
+                                    if (pwc != NULL)
+                                      *pwc = wc;
+                                    res = 3;
+                                    goto success;
+                                  }
                               }
                           }
                       }
@@ -253,13 +258,19 @@ mbrtowc (wchar_t *pwc, const char *s, size_t n, mbstate_t 
*ps)
 
                                     if ((c4 ^ 0x80) < 0x40)
                                       {
-                                        if (pwc != NULL)
-                                          *pwc = ((unsigned int) (c & 0x07) << 
18)
-                                                 | ((unsigned int) (c2 ^ 0x80) 
<< 12)
-                                                 | ((unsigned int) (c3 ^ 0x80) 
<< 6)
-                                                 | (unsigned int) (c4 ^ 0x80);
-                                        res = 4;
-                                        goto success;
+                                        unsigned int wc
+                                          = (((unsigned int) (c & 0x07) << 18)
+                                             | ((unsigned int) (c2 ^ 0x80)
+                                                << 12)
+                                             | ((unsigned int) (c3 ^ 0x80) << 
6)
+                                             | (unsigned int) (c4 ^ 0x80));
+                                        if (wc <= WCHAR_MAX)
+                                          {
+                                            if (pwc != NULL)
+                                              *pwc = wc;
+                                            res = 4;
+                                            goto success;
+                                          }
                                       }
                                   }
                               }
-- 
2.17.1