[PATCH 2/4] AIX: Do not rely on OBJECT_MODE env var being set.

[Michael Haubenwallner]

To compile 64-bit, AIX toolchain provides support to set the environment
variable OBJECT_MODE=64, to avoid the need for changing anything else in
the build environment. However, gcc ignores OBJECT_MODE, and does not
set it while building gcc itself during its multi-lib build either.
Also, CC may be some wrapper to operate in 64-bit mode. So we cannot
rely on OBJECT_MODE being set to properly support 64-bit builds.

Instead, we better identify the object mode in use by looking at the
file type of an object file generated by CC, to properly set the
commandline flags for additional tools used within libtool. While there
is similar code for LD in _LT_ENABLE_LOCK, this is too late for NM.

* m4/libtool.m4 (_LT_CACHE_CHECK_OBJECT_FILE_TYPE): New. Provides object
file type identified using /usr/bin/file.
(_LT_PREPARE_TOOLS_ABI_FLAGS): New. Defines func_set_tool_abi_flag.
Define AIX specific ABI flags for LD, AR, NM, STRIP, RANLIB.
Set AIX_OBJECT_MODE as well.
(_LT_SET_TOOL_ABI_FLAG): New. Uses func_set_tool_abi_flag to set host
specific ABI flags into tool commands.
(_LT_PROG_AR): Use _LT_SET_TOOL_ABI_FLAG.
(LT_PATH_LD): Ditto.
(LT_PATH_NM): Ditto.
(_LT_CMD_OLD_ARCHIVE): Use _LT_SET_TOOL_ABI_FLAG for STRIP, RANLIB.
(_LT_LINKER_SHLIBS): Use AIX_OBJECT_MODE.
(_LT_LANG_CXX_CONFIG): Ditto.
* m4/ltoptions.m4 (_LT_WITH_AIX_SONAME): Ditto.
---
 m4/libtool.m4   | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 m4/ltoptions.m4 |  3 +-
 2 files changed, 84 insertions(+), 4 deletions(-)

diff --git a/m4/libtool.m4 b/m4/libtool.m4
index 7b8b591..2c0e657 100644
--- a/m4/libtool.m4
+++ b/m4/libtool.m4
@@ -1177,6 +1177,80 @@ fi
 ])# _LT_SYS_MODULE_PATH_AIX
 
 
+# _LT_CACHE_CHECK_OBJECT_FILE_TYPE
+# --------------------------------
+# Compiles a minimal program and identifies the file type of that object file.
+# All object files linked together in one build configuration have to be of
+# the same object file type.  As we always need the C compiler anyway, there
+# is no point in checking object file types per TAG.
+m4_defun([_LT_CACHE_CHECK_OBJECT_FILE_TYPE],
+[AC_REQUIRE([AC_PROG_CC])dnl
+m4_require([_LT_DECL_SED])dnl
+m4_require([_LT_PROG_ECHO_BACKSLASH])dnl
+AC_CACHE_CHECK([$CC object file type], [lt_cv_object_file_type],
+  [AC_COMPILE_IFELSE([AC_LANG_PROGRAM],
+    [echo "$as_me:$LINENO: LC_ALL=C /usr/bin/file conftest.$ac_objext" 
>&AS_MESSAGE_LOG_FD
+    lt_cv_object_file_type=`LC_ALL=C /usr/bin/file conftest.$ac_objext 
2>&AS_MESSAGE_LOG_FD`
+    ac_status=$?
+    func_echo_all "$lt_cv_object_file_type" >&AS_MESSAGE_LOG_FD
+    func_echo_all "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD
+    lt_cv_object_file_type=`func_echo_all "$lt_cv_object_file_type" | $SED -e 
"s/^conftest\.$ac_objext:  *//"`
+    ])
+  ])
+]) # _LT_CACHE_CHECK_OBJECT_FILE_TYPE
+
+
+# _LT_PREPARE_TOOLS_ABI_FLAGS
+# ---------------------------
+m4_defun([_LT_PREPARE_TOOLS_ABI_FLAGS],
+[AC_REQUIRE([AC_CANONICAL_HOST])dnl
+AC_REQUIRE([AC_PROG_CC])dnl
+# func_set_tool_abi_flag TOOL [TOOLVAR]
+# Ensure that TOOL-specific command in $TOOL (or $TOOLVAR if specified) does
+# contain $lt_abi_flag_TOOL (if defined below) as standalone argument.
+# Does nothing if $TOOL (or $TOOLVAR) is empty.
+func_set_tool_abi_flag ()
+{
+  eval "AS_CASE([\" address@hidden|@@S|@address@hidden|@1}  \"],
+  ['   '],
+    [# command is empty
+    ],
+  [*\" address@hidden|@address@hidden|@1 \"*],
+    [# standalone flag found, or flag is empty
+    ],
+    
address@hidden|@address@hidden|@1}=\"address@hidden|@@S|@address@hidden|@1} 
address@hidden|@address@hidden|@1\"]
+  )"
+}
+
+# PORTME Define flags for various tools to accept the ABI selected by
+# '$CC $CFLAGS $CPPFLAGS'.
+case $host in
+power*-*-aix4.[[3-9]]* | power*-aix[[5-9]]*)
+  _LT_CACHE_CHECK_OBJECT_FILE_TYPE
+  case $ac_status:$lt_cv_object_file_type in
+  0:*64-bit*)             AIX_OBJECT_MODE=64 ;;
+  0:*"RISC System/6000"*) AIX_OBJECT_MODE=32 ;;
+  *) AC_MSG_FAILURE([failed to check $CC generated ABI]) ;;
+  esac
+  lt_abi_flag_LD=-b$AIX_OBJECT_MODE
+  lt_abi_flag_AR=-X$AIX_OBJECT_MODE
+  lt_abi_flag_NM=-X$AIX_OBJECT_MODE
+  lt_abi_flag_STRIP=-X$AIX_OBJECT_MODE
+  lt_abi_flag_RANLIB=-X$AIX_OBJECT_MODE
+  ;;
+esac
+]) # _LT_PREPARE_TOOLS_ABI_FLAGS
+
+
+# _LT_SET_TOOL_ABI_FLAG(TOOL, [TOOLVAR])
+# ----------------------------------------
+# Ensure to be defined, and call: func_set_tool_abi_flag TOOL TOOLVAR
+m4_defun([_LT_SET_TOOL_ABI_FLAG],
+[m4_require([_LT_PREPARE_TOOLS_ABI_FLAGS])dnl
+func_set_tool_abi_flag $1 $2
+]) # _LT_SET_TOOL_ABI_FLAG
+
+
 # _LT_SHELL_INIT(ARG)
 # -------------------
 m4_define([_LT_SHELL_INIT],
@@ -1285,7 +1359,7 @@ m4_defun([_LT_ENABLE_LOCK],
 test no = "$enable_libtool_lock" || enable_libtool_lock=yes
 
 # Some flags need to be propagated to the compiler or linker for good
-# libtool support.
+# libtool support. Should move to _LT_PREPARE_TOOLS_ABI_FLAGS.
 case $host in
 ia64-*-hpux*)
   # Find out what ABI is being produced by ac_compile, and set mode
@@ -1493,6 +1567,7 @@ need_locks=$enable_libtool_lock
 m4_defun([_LT_PROG_AR],
 [AC_CHECK_TOOLS(AR, [ar], false)
 : ${AR=ar}
+_LT_SET_TOOL_ABI_FLAG([AR])
 _LT_DECL([], [AR], [1], [The archiver])
 
 # Use ARFLAGS variable as AR's operation code to sync the variable naming with
@@ -1545,10 +1620,12 @@ m4_defun([_LT_CMD_OLD_ARCHIVE],
 
 AC_CHECK_TOOL(STRIP, strip, :)
 test -z "$STRIP" && STRIP=:
+_LT_SET_TOOL_ABI_FLAG([STRIP])
 _LT_DECL([], [STRIP], [1], [A symbol stripping program])
 
 AC_CHECK_TOOL(RANLIB, ranlib, :)
 test -z "$RANLIB" && RANLIB=:
+_LT_SET_TOOL_ABI_FLAG([RANLIB])
 _LT_DECL([], [RANLIB], [1],
     [Commands used to install an old-style archive])
 
@@ -3342,6 +3419,7 @@ else
 fi
 test -z "$LD" && AC_MSG_ERROR([no acceptable ld found in \$PATH])
 _LT_PATH_LD_GNU
+_LT_SET_TOOL_ABI_FLAG([LD])
 AC_SUBST([LD])
 
 _LT_TAGDECL([], [LD], [1], [The linker used to build libraries])
@@ -3752,6 +3830,7 @@ else
   fi
 fi
 test -z "$NM" && NM=nm
+_LT_SET_TOOL_ABI_FLAG([NM])
 AC_SUBST([NM])
 _LT_DECL([], [NM], [1], [A BSD- or MS-compatible name lister])dnl
 
@@ -5548,7 +5627,7 @@ _LT_EOF
            _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_aix' -o 
$output_objdir/$realname.d/$soname $libobjs $deplibs $wl-bnoentry 
'$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$AR 
$AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$realname.d/$soname'
          fi
          if test aix != "$with_aix_soname"; then
-           _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_svr4' -o 
$output_objdir/$realname.d/$shared_archive_member_spec.o $libobjs $deplibs 
$wl-bnoentry 
'$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$STRIP -e 
$output_objdir/$realname.d/$shared_archive_member_spec.o~( func_echo_all "#! 
$soname($shared_archive_member_spec.o)"; if test shr_64 = 
"$shared_archive_member_spec"; then func_echo_all "# 64"; else func_echo_all "# 
32"; fi; cat $export_symbols ) > 
$output_objdir/$realname.d/$shared_archive_member_spec.imp~$AR $AR_FLAGS 
$output_objdir/$soname $output_objdir/$realname.d/$shared_archive_member_spec.o 
$output_objdir/$realname.d/$shared_archive_member_spec.imp'
+           _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_svr4' -o 
$output_objdir/$realname.d/$shared_archive_member_spec.o $libobjs $deplibs 
$wl-bnoentry 
'$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$STRIP -e 
$output_objdir/$realname.d/$shared_archive_member_spec.o~( func_echo_all "#! 
$soname($shared_archive_member_spec.o)"; func_echo_all "# '$AIX_OBJECT_MODE'"; 
cat $export_symbols ) > 
$output_objdir/$realname.d/$shared_archive_member_spec.imp~$AR $AR_FLAGS 
$output_objdir/$soname $output_objdir/$realname.d/$shared_archive_member_spec.o 
$output_objdir/$realname.d/$shared_archive_member_spec.imp'
          else
            # used by -dlpreopen to get the symbols
            _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$MV  
$output_objdir/$realname.d/$soname $output_objdir'
@@ -6623,7 +6702,7 @@ if test yes != "$_lt_caught_CXX_error"; then
              _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_aix' -o 
$output_objdir/$realname.d/$soname $libobjs $deplibs $wl-bnoentry 
'$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$AR 
$AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$realname.d/$soname'
            fi
            if test aix != "$with_aix_soname"; then
-             _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_svr4' -o 
$output_objdir/$realname.d/$shared_archive_member_spec.o $libobjs $deplibs 
$wl-bnoentry 
'$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$STRIP -e 
$output_objdir/$realname.d/$shared_archive_member_spec.o~( func_echo_all "#! 
$soname($shared_archive_member_spec.o)"; if test shr_64 = 
"$shared_archive_member_spec"; then func_echo_all "# 64"; else func_echo_all "# 
32"; fi; cat $export_symbols ) > 
$output_objdir/$realname.d/$shared_archive_member_spec.imp~$AR $AR_FLAGS 
$output_objdir/$soname $output_objdir/$realname.d/$shared_archive_member_spec.o 
$output_objdir/$realname.d/$shared_archive_member_spec.imp'
+             _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_svr4' -o 
$output_objdir/$realname.d/$shared_archive_member_spec.o $libobjs $deplibs 
$wl-bnoentry 
'$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$STRIP -e 
$output_objdir/$realname.d/$shared_archive_member_spec.o~( func_echo_all "#! 
$soname($shared_archive_member_spec.o)"; func_echo_all "# '$AIX_OBJECT_MODE'"; 
cat $export_symbols ) > 
$output_objdir/$realname.d/$shared_archive_member_spec.imp~$AR $AR_FLAGS 
$output_objdir/$soname $output_objdir/$realname.d/$shared_archive_member_spec.o 
$output_objdir/$realname.d/$shared_archive_member_spec.imp'
            else
              # used by -dlpreopen to get the symbols
              _LT_TAGVAR(archive_expsym_cmds, 
$1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$MV  
$output_objdir/$realname.d/$soname $output_objdir'
diff --git a/m4/ltoptions.m4 b/m4/ltoptions.m4
index b34dd2f..5026816 100644
--- a/m4/ltoptions.m4
+++ b/m4/ltoptions.m4
@@ -328,6 +328,7 @@ dnl AC_DEFUN([AM_DISABLE_FAST_INSTALL], [])
 # is either `aix', `both' or `svr4'.  If omitted, it defaults to `aix'.
 m4_define([_LT_WITH_AIX_SONAME],
 [m4_define([_LT_WITH_AIX_SONAME_DEFAULT], [m4_if($1, svr4, svr4, m4_if($1, 
both, both, aix))])dnl
+m4_require([_LT_PREPARE_TOOLS_ABI_FLAGS])dnl
 shared_archive_member_spec=
 case $host,$enable_shared in
 power*-*-aix[[5-9]]*,yes)
@@ -353,7 +354,7 @@ power*-*-aix[[5-9]]*,yes)
     # and 'shr.imp' or 'shr_64.imp', respectively, for the Import File.
     # Even when GNU compilers ignore OBJECT_MODE but need '-maix64' flag,
     # the AIX toolchain works better with OBJECT_MODE set (default 32).
-    if test 64 = "${OBJECT_MODE-32}"; then
+    if test 64 = "$AIX_OBJECT_MODE"; then
       shared_archive_member_spec=shr_64
     else
       shared_archive_member_spec=shr
-- 
2.4.6