[freetype2] master 5ad8457: [cache] Fix a possible overflow by signed integer comparison.

[suzuki toshiya]

branch: master
commit 5ad845771a7389e72e4868dd1d4f0986f812b705
Author: suzuki toshiya <address@hidden>
Commit: suzuki toshiya <address@hidden>

    [cache] Fix a possible overflow by signed integer comparison.
    
    Improve the code by 5d3ff05615dda6d1325ed612381a17a0df04c975 ,
    issues are found by Behdad Esfahbod and Werner Lemberg.
    
    * src/cache/ftcbasic.c (FTC_ImageCache_Lookup): Replace
    a subtraction to check higher bit by a bit operation,
    and cpp-conditionalize for appropriate systems.  Add better
    documentation to the comment.
    (FTC_ImageCache_LookupScaler): Ditto.
    (FTC_SBitCache_Lookup): Ditto.
    (FTC_SBitCache_LookupScaler): Ditto.
---
 ChangeLog            | 15 +++++++++++++++
 src/cache/ftcbasic.c | 42 ++++++++++++++++++++++++++++++++++++++----
 2 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index b25a822..62618f8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+2017-09-13  suzuki toshiya  <address@hidden>
+
+       [cache] Fix a possible overflow by signed integer comparison.
+
+       Improve the code by 5d3ff05615dda6d1325ed612381a17a0df04c975 ,
+       issues are found by Behdad Esfahbod and Werner Lemberg.
+
+       * src/cache/ftcbasic.c (FTC_ImageCache_Lookup): Replace
+       a subtraction to check higher bit by a bit operation,
+       and cpp-conditionalize for appropriate systems.  Add better
+       documentation to the comment.
+       (FTC_ImageCache_LookupScaler): Ditto.
+       (FTC_SBitCache_Lookup): Ditto.
+       (FTC_SBitCache_LookupScaler): Ditto.
+
 2017-09-13  Werner Lemberg  <address@hidden>
 
        [autofit] Really fix #41334 (#52000).
diff --git a/src/cache/ftcbasic.c b/src/cache/ftcbasic.c
index 289bd5c..59221f2 100644
--- a/src/cache/ftcbasic.c
+++ b/src/cache/ftcbasic.c
@@ -304,10 +304,19 @@
     if ( anode )
       *anode  = NULL;
 
-    if ( (FT_ULong)( type->flags - FT_INT_MIN ) > FT_UINT_MAX )
+/*
+ * internal FTC_BasicAttr->load_flags is typed FT_UInt,
+ * but public FT_ImageType->flags is typed FT_Int32
+ *
+ * On 16bit systems, higher bits of type->flags
+ * could not be handled.
+ */
+#if 0xFFFFFFFFUL > FT_UINT_MAX
+    if ( (type->flags & (FT_ULong)FT_UINT_MAX) )
       FT_TRACE1(( "FTC_ImageCache_Lookup:"
                   " higher bits in load_flags 0x%x are dropped\n",
                   (FT_ULong)type->flags & ~((FT_ULong)FT_UINT_MAX) ));
+#endif
 
     query.attrs.scaler.face_id = type->face_id;
     query.attrs.scaler.width   = type->width;
@@ -377,11 +386,19 @@
     if ( anode )
       *anode  = NULL;
 
-    /* `FT_Load_Glyph' and `FT_Load_Char' take FT_UInt flags */
+/*
+ * internal FTC_BasicAttr->load_flags is typed FT_UInt,
+ * but public FT_Face->face_flags is typed FT_Long.
+ *
+ * On long > int systems, higher bits of load_flags
+ * could not be handled.
+ */
+#if FT_ULONG_MAX > FT_UINT_MAX
     if ( load_flags > FT_UINT_MAX )
       FT_TRACE1(( "FTC_ImageCache_LookupScaler:"
                   " higher bits in load_flags 0x%x are dropped\n",
                   load_flags & ~((FT_ULong)FT_UINT_MAX) ));
+#endif
 
     query.attrs.scaler     = scaler[0];
     query.attrs.load_flags = (FT_UInt)load_flags;
@@ -487,10 +504,19 @@
 
     *ansbit = NULL;
 
-    if ( (FT_ULong)( type->flags - FT_INT_MIN ) > FT_UINT_MAX )
+/*
+ * internal FTC_BasicAttr->load_flags is typed FT_UInt,
+ * but public FT_ImageType->flags is typed FT_Int32
+ *
+ * On 16bit systems, higher bits of type->flags
+ * could not be handled.
+ */
+#if 0xFFFFFFFFUL > FT_UINT_MAX
+    if ( (type->flags & (FT_ULong)FT_UINT_MAX) )
       FT_TRACE1(( "FTC_ImageCache_Lookup:"
                   " higher bits in load_flags 0x%x are dropped\n",
                   (FT_ULong)type->flags & ~((FT_ULong)FT_UINT_MAX) ));
+#endif
 
     query.attrs.scaler.face_id = type->face_id;
     query.attrs.scaler.width   = type->width;
@@ -562,11 +588,19 @@
 
     *ansbit = NULL;
 
-    /* `FT_Load_Glyph' and `FT_Load_Char' take FT_UInt flags */
+/*
+ * internal FTC_BasicAttr->load_flags is typed FT_UInt,
+ * but public FT_Face->face_flags is typed FT_Long.
+ *
+ * On long > int systems, higher bits of load_flags
+ * could not be handled.
+ */
+#if FT_ULONG_MAX > FT_UINT_MAX
     if ( load_flags > FT_UINT_MAX )
       FT_TRACE1(( "FTC_ImageCache_LookupScaler:"
                   " higher bits in load_flags 0x%x are dropped\n",
                   load_flags & ~((FT_ULong)FT_UINT_MAX) ));
+#endif
 
     query.attrs.scaler     = scaler[0];
     query.attrs.load_flags = (FT_UInt)load_flags;