[gpsd-dev] [PATCH 10/12] Fixes fake.py and gpsfake for Python 3.

[Fred Wright]

This incorporates the following changes:

1) Adds definitions for polystr(), polybytes(), and make_std_wrapper()
to misc.py (the most logical place for code needed by multiple
modules), using dummy definitions in the Python 2 case.  For more
info, see the Practical Python Porting guide.

2) Reworks the logfile->daemon data path in fake.py to use 'bytes'
consistently.

3) Uses polybytes() in fake.py to construct control-socket commands as
'bytes', as expected by the socket I/O.

4) Uses make_std_wrapper() in gpsfake, to ensure that binary data is
correctly written to stdout.

5) Adds 'division' to the future imports in gpsfake for consistency,
though it doesn't actually matter in practice.

Also updates the compatibility comments in all three files, and fixes
a minor typo in misc.py.

TESTED:
Ran "scons build-all check" with Python 2.7, and ran all daemon
regression tests with Python 2.6 and with Python 3.2-3.5 (with
appropriately built extensions; not yet a build option).
---
 gps/fake.py | 32 +++++++++++++++++---------------
 gps/misc.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 gpsfake     |  8 ++++++--
 3 files changed, 80 insertions(+), 20 deletions(-)

diff --git a/gps/fake.py b/gps/fake.py
index 144edd9..88336d3 100644
--- a/gps/fake.py
+++ b/gps/fake.py
@@ -67,13 +67,14 @@ To allow for adding and removing clients while the test is 
running,
 run in threaded mode by calling the start() method.  This simply calls
 the run method in a subthread, with locking of critical regions.
 """
-# This code run compatibly under Python 2 and 3.x for x >= 3.
+# This code runs compatibly under Python 2 and 3.x for x >= 2.
 # Preserve this property!
 from __future__ import absolute_import, print_function, division
 
 import os, sys, time, signal, pty, termios  # fcntl, array, struct
 import threading, socket, select
 import gps
+from gps import polybytes
 from . import packet as sniffer
 import stat
 
@@ -130,7 +131,7 @@ class TestLoad:
     def __init__(self, logfp, predump=False, slow=False, oneshot=False):
         self.sentences = []  # This is the interesting part
         if type(logfp) == type(""):
-            logfp = open(logfp, "r")
+            logfp = open(logfp, "rb")
         self.name = logfp.name
         self.logfp = logfp
         self.predump = predump
@@ -141,34 +142,35 @@ class TestLoad:
         self.delimiter = None
         # Stash away a copy in case we need to resplit
         text = logfp.read()
-        logfp = open(logfp.name)
+        logfp = open(logfp.name, 'rb')
         # Grab the packets in the normal way
         getter = sniffer.new()
         # gps.packet.register_report(reporter)
         type_latch = None
         commentlen = 0
         while True:
+            # Note that packet data is bytes rather than str
             (plen, ptype, packet, _counter) = getter.get(logfp.fileno())
             if plen <= 0:
                 break
             elif ptype == sniffer.COMMENT_PACKET:
                 commentlen += len(packet)
                 # Some comments are magic
-                if "Serial:" in packet:
+                if b"Serial:" in packet:
                     # Change serial parameters
                     packet = packet[1:].strip()
                     try:
                         (_xx, baud, params) = packet.split()
                         baud = int(baud)
-                        if params[0] in ('7', '8'):
+                        if params[0] in (b'7', b'8'):
                             databits = int(params[0])
                         else:
                             raise ValueError
-                        if params[1] in ('N', 'O', 'E'):
+                        if params[1] in (b'N', b'O', b'E'):
                             parity = params[1]
                         else:
                             raise ValueError
-                        if params[2] in ('1', '2'):
+                        if params[2] in (b'1', b'2'):
                             stopbits = int(params[2])
                         else:
                             raise ValueError
@@ -176,12 +178,12 @@ class TestLoad:
                         raise TestLoadError("bad serial-parameter spec in %s" %
                                             self.name)
                     self.serial = (baud, databits, parity, stopbits)
-                elif "Transport: UDP" in packet:
+                elif b"Transport: UDP" in packet:
                     self.sourcetype = "UDP"
-                elif "Transport: TCP" in packet:
+                elif b"Transport: TCP" in packet:
                     self.sourcetype = "TCP"
-                elif "Delay-Cookie:" in packet:
-                    if packet.startswith("#"):
+                elif b"Delay-Cookie:" in packet:
+                    if packet.startswith(b"#"):
                         packet = packet[1:]
                     try:
                         (_dummy, self.delimiter, delay) = 
packet.strip().split()
@@ -210,7 +212,7 @@ class TestLoad:
             self.sentences = text[commentlen:].split(self.delimiter)
         # Do we want single-shot operation?
         if oneshot:
-            self.sentences.append("# EOF\n")
+            self.sentences.append(b"# EOF\n")
 
 
 class PacketError(BaseException):
@@ -235,7 +237,7 @@ class FakeGPS:
     def feed(self):
         "Feed a line from the contents of the GPS log to the daemon."
         line = self.testload.sentences[self.index % 
len(self.testload.sentences)]
-        if "%Delay:" in line:
+        if b"%Delay:" in line:
             # Delay specified number of seconds
             delay = line.split()[1]
             time.sleep(int(delay))
@@ -525,14 +527,14 @@ class DaemonInstance:
     def add_device(self, path):
         "Add a device to the daemon's internal search list."
         if self.__get_control_socket():
-            self.sock.sendall("+%s\r\n\x00" % path)
+            self.sock.sendall(polybytes("+%s\r\n\x00" % path))
             self.sock.recv(12)
             self.sock.close()
 
     def remove_device(self, path):
         "Remove a device from the daemon's internal search list."
         if self.__get_control_socket():
-            self.sock.sendall("-%s\r\n\x00" % path)
+            self.sock.sendall(polybytes("-%s\r\n\x00" % path))
             self.sock.recv(12)
             self.sock.close()
 
diff --git a/gps/misc.py b/gps/misc.py
index e1c01fe..f1fcadf 100644
--- a/gps/misc.py
+++ b/gps/misc.py
@@ -3,11 +3,11 @@
 # This file is Copyright (c) 2010 by the GPSD project
 # BSD terms apply: see the file COPYING in the distribution root for details.
 
-# This code run compatibly under Python 2 and 3.x for x >= 3.
+# This code runs compatibly under Python 2 and 3.x for x >= 2.
 # Preserve this property!
 from __future__ import absolute_import, print_function, division
 
-import time, calendar, math
+import time, calendar, math, io
 
 # Determine a single class for testing "stringness"
 try:
@@ -15,6 +15,60 @@ try:
 except NameError:
     STR_CLASS = str         # In Python 3, 'str' is the base class
 
+# We need to be able to handle data which may be a mixture of text and binary
+# data.  The text in this context is known to be limited to US-ASCII, so
+# there aren't any issues regarding character sets, but we need to ensure
+# that binary data is preserved.  In Python 2, this happens naturally with
+# "strings" and the 'str' and 'bytes' types are synonyms.  But in Python 3,
+# these are distinct types (with 'str' being based on Unicode), and conversions
+# are encoding-sensitive.  The most straightforward encoding to use in this
+# context is 'latin-1' (a.k.a.'iso-8859-1'), which directly maps all 256
+# 8-bit character values to Unicode page 0.  Thus, if we can enforce the use
+# of 'latin-1' encoding, we can preserve arbitrary binary data while correctly
+# mapping any actual text to the proper characters.
+
+BINARY_ENCODING = 'latin-1'
+
+if bytes is str:  # In Python 2 these functions can be null transformations
+
+    polystr = str
+    polybytes = bytes
+
+    def make_std_wrapper(stream):
+        "Dummy stdio wrapper function."
+        return stream
+
+else:  # Otherwise we do something real
+
+    def polystr(o):
+        "Convert bytes or str to str with proper encoding."
+        if isinstance(o, str):
+            return o
+        if isinstance(o, bytes):
+            return str(o, encoding=BINARY_ENCODING)
+        raise ValueError
+
+    def polybytes(o):
+        "Convert bytes or str to bytes with proper encoding."
+        if isinstance(o, bytes):
+            return o
+        if isinstance(o, str):
+            return bytes(o, encoding=BINARY_ENCODING)
+        raise ValueError
+
+    def make_std_wrapper(stream):
+        "Standard input/output wrapper factory function"
+        # This ensures that the encoding of standard output and standard
+        # error on Python 3 matches the binary encoding we use to turn
+        # bytes to Unicode in polystr above.
+        #
+        # newline="\n" ensures that Python 3 won't mangle line breaks
+        # line_buffering=True ensures that interactive command sessions
+        # work as expected
+        return io.TextIOWrapper(stream.buffer, encoding=BINARY_ENCODING,
+                                newline="\n", line_buffering=True)
+
+
 # some multipliers for interpreting GPS output
 METERS_TO_FEET = 3.2808399     # Meters to U.S./British feet
 METERS_TO_MILES        = 0.00062137119 # Meters to miles
@@ -35,7 +89,7 @@ def Deg2Rad(x):
 
 
 def Rad2Deg(x):
-    "Radians to degress."
+    "Radians to degrees."
     return x * (180 / math.pi)
 
 
diff --git a/gpsfake b/gpsfake
index 4be8047..f687438 100755
--- a/gpsfake
+++ b/gpsfake
@@ -9,8 +9,9 @@
 # This file is Copyright (c) 2010 by the GPSD project
 # BSD terms apply: see the file COPYING in the distribution root for details.
 
-# This code runs under both Python 2 and Python 3. Preserve this property!
-from __future__ import print_function
+# This code runs compatibly under Python 2 and 3.x for x >= 2.
+# Preserve this property!
+from __future__ import print_function, division
 
 import getopt
 import gps
@@ -27,6 +28,9 @@ try:
 except NameError:
     my_input = input
 
+# Make sure stdout can accept binary data in Python 3
+sys.stdout = gps.make_std_wrapper(sys.stdout)
+
 class Baton:
     "Ship progress indications to stderr."
     # By setting this > 1 we reduce the frequency of the twirl
-- 
2.8.0