Merge pull request #1195 from calbach/python-json-struct

Manually down-integrate python JSON struct support from internal code base.

.gitignore

@@ -58,6 +58,7 @@ python/.eggs/
 python/.tox
 python/build/
 python/google/protobuf/compiler/
+python/google/protobuf/util/
 
 src/protoc
 src/unittest_proto_middleman

python/google/protobuf/internal/well_known_types.py

@@ -34,6 +34,7 @@ This files defines well known classes which need extra maintenance including:
   - Any
   - Duration
   - FieldMask
+  - Struct
   - Timestamp
 """
 
@@ -41,6 +42,7 @@ __author__ = 'jieluo@google.com (Jie Luo)'
 
 from datetime import datetime
 from datetime import timedelta
+import six
 
 from google.protobuf.descriptor import FieldDescriptor
 
@@ -64,9 +66,12 @@ class ParseError(Error):
 class Any(object):
   """Class for Any Message type."""
 
-  def Pack(self, msg):
+  def Pack(self, msg, type_url_prefix='type.googleapis.com/'):
     """Packs the specified message into current Any message."""
-    self.type_url = 'type.googleapis.com/%s' % msg.DESCRIPTOR.full_name
+    if len(type_url_prefix) < 1 or type_url_prefix[-1] != '/':
+      self.type_url = '%s/%s' % (type_url_prefix, msg.DESCRIPTOR.full_name)
+    else:
+      self.type_url = '%s%s' % (type_url_prefix, msg.DESCRIPTOR.full_name)
     self.value = msg.SerializeToString()
 
   def Unpack(self, msg):
@@ -614,9 +619,102 @@ def _AddFieldPaths(node, prefix, field_mask):
     _AddFieldPaths(node[name], child_path, field_mask)
 
 
+_INT_OR_FLOAT = six.integer_types + (float,)
+
+
+def _SetStructValue(struct_value, value):
+  if value is None:
+    struct_value.null_value = 0
+  elif isinstance(value, bool):
+    # Note: this check must come before the number check because in Python
+    # True and False are also considered numbers.
+    struct_value.bool_value = value
+  elif isinstance(value, six.string_types):
+    struct_value.string_value = value
+  elif isinstance(value, _INT_OR_FLOAT):
+    struct_value.number_value = value
+  else:
+    raise ValueError('Unexpected type')
+
+
+def _GetStructValue(struct_value):
+  which = struct_value.WhichOneof('kind')
+  if which == 'struct_value':
+    return struct_value.struct_value
+  elif which == 'null_value':
+    return None
+  elif which == 'number_value':
+    return struct_value.number_value
+  elif which == 'string_value':
+    return struct_value.string_value
+  elif which == 'bool_value':
+    return struct_value.bool_value
+  elif which == 'list_value':
+    return struct_value.list_value
+  elif which is None:
+    raise ValueError('Value not set')
+
+
+class Struct(object):
+  """Class for Struct message type."""
+
+  __slots__ = []
+
+  def __getitem__(self, key):
+    return _GetStructValue(self.fields[key])
+
+  def __setitem__(self, key, value):
+    _SetStructValue(self.fields[key], value)
+
+  def get_or_create_list(self, key):
+    """Returns a list for this key, creating if it didn't exist already."""
+    return self.fields[key].list_value
+
+  def get_or_create_struct(self, key):
+    """Returns a struct for this key, creating if it didn't exist already."""
+    return self.fields[key].struct_value
+
+  # TODO(haberman): allow constructing/merging from dict.
+
+
+class ListValue(object):
+  """Class for ListValue message type."""
+
+  def __len__(self):
+    return len(self.values)
+
+  def append(self, value):
+    _SetStructValue(self.values.add(), value)
+
+  def extend(self, elem_seq):
+    for value in elem_seq:
+      self.append(value)
+
+  def __getitem__(self, index):
+    """Retrieves item by the specified index."""
+    return _GetStructValue(self.values.__getitem__(index))
+
+  def __setitem__(self, index, value):
+    _SetStructValue(self.values.__getitem__(index), value)
+
+  def items(self):
+    for i in range(len(self)):
+      yield self[i]
+
+  def add_struct(self):
+    """Appends and returns a struct value as the next value in the list."""
+    return self.values.add().struct_value
+
+  def add_list(self):
+    """Appends and returns a list value as the next value in the list."""
+    return self.values.add().list_value
+
+
 WKTBASES = {
     'google.protobuf.Any': Any,
     'google.protobuf.Duration': Duration,
     'google.protobuf.FieldMask': FieldMask,
+    'google.protobuf.ListValue': ListValue,
+    'google.protobuf.Struct': Struct,
     'google.protobuf.Timestamp': Timestamp,
 }

python/google/protobuf/internal/well_known_types_test.py

@@ -41,13 +41,17 @@ try:
 except ImportError:
   import unittest
 
+from google.protobuf import any_pb2
 from google.protobuf import duration_pb2
 from google.protobuf import field_mask_pb2
+from google.protobuf import struct_pb2
 from google.protobuf import timestamp_pb2
 from google.protobuf import unittest_pb2
+from google.protobuf.internal import any_test_pb2
 from google.protobuf.internal import test_util
 from google.protobuf.internal import well_known_types
 from google.protobuf import descriptor
+from google.protobuf import text_format
 
 
 class TimeUtilTestBase(unittest.TestCase):
@@ -509,5 +513,124 @@ class FieldMaskTest(unittest.TestCase):
     self.assertEqual(1, len(nested_dst.payload.repeated_int32))
     self.assertEqual(1234, nested_dst.payload.repeated_int32[0])
 
+
+class StructTest(unittest.TestCase):
+
+  def testStruct(self):
+    struct = struct_pb2.Struct()
+    struct_class = struct.__class__
+
+    struct['key1'] = 5
+    struct['key2'] = 'abc'
+    struct['key3'] = True
+    struct.get_or_create_struct('key4')['subkey'] = 11.0
+    struct_list = struct.get_or_create_list('key5')
+    struct_list.extend([6, 'seven', True, False, None])
+    struct_list.add_struct()['subkey2'] = 9
+
+    self.assertTrue(isinstance(struct, well_known_types.Struct))
+    self.assertEquals(5, struct['key1'])
+    self.assertEquals('abc', struct['key2'])
+    self.assertIs(True, struct['key3'])
+    self.assertEquals(11, struct['key4']['subkey'])
+    inner_struct = struct_class()
+    inner_struct['subkey2'] = 9
+    self.assertEquals([6, 'seven', True, False, None, inner_struct],
+                      list(struct['key5'].items()))
+
+    serialized = struct.SerializeToString()
+
+    struct2 = struct_pb2.Struct()
+    struct2.ParseFromString(serialized)
+
+    self.assertEquals(struct, struct2)
+
+    self.assertTrue(isinstance(struct2, well_known_types.Struct))
+    self.assertEquals(5, struct2['key1'])
+    self.assertEquals('abc', struct2['key2'])
+    self.assertIs(True, struct2['key3'])
+    self.assertEquals(11, struct2['key4']['subkey'])
+    self.assertEquals([6, 'seven', True, False, None, inner_struct],
+                      list(struct2['key5'].items()))
+
+    struct_list = struct2['key5']
+    self.assertEquals(6, struct_list[0])
+    self.assertEquals('seven', struct_list[1])
+    self.assertEquals(True, struct_list[2])
+    self.assertEquals(False, struct_list[3])
+    self.assertEquals(None, struct_list[4])
+    self.assertEquals(inner_struct, struct_list[5])
+
+    struct_list[1] = 7
+    self.assertEquals(7, struct_list[1])
+
+    struct_list.add_list().extend([1, 'two', True, False, None])
+    self.assertEquals([1, 'two', True, False, None],
+                      list(struct_list[6].items()))
+
+    text_serialized = str(struct)
+    struct3 = struct_pb2.Struct()
+    text_format.Merge(text_serialized, struct3)
+    self.assertEquals(struct, struct3)
+
+    struct.get_or_create_struct('key3')['replace'] = 12
+    self.assertEquals(12, struct['key3']['replace'])
+
+
+class AnyTest(unittest.TestCase):
+
+  def testAnyMessage(self):
+    # Creates and sets message.
+    msg = any_test_pb2.TestAny()
+    msg_descriptor = msg.DESCRIPTOR
+    all_types = unittest_pb2.TestAllTypes()
+    all_descriptor = all_types.DESCRIPTOR
+    all_types.repeated_string.append(u'\u00fc\ua71f')
+    # Packs to Any.
+    msg.value.Pack(all_types)
+    self.assertEqual(msg.value.type_url,
+                     'type.googleapis.com/%s' % all_descriptor.full_name)
+    self.assertEqual(msg.value.value,
+                     all_types.SerializeToString())
+    # Tests Is() method.
+    self.assertTrue(msg.value.Is(all_descriptor))
+    self.assertFalse(msg.value.Is(msg_descriptor))
+    # Unpacks Any.
+    unpacked_message = unittest_pb2.TestAllTypes()
+    self.assertTrue(msg.value.Unpack(unpacked_message))
+    self.assertEqual(all_types, unpacked_message)
+    # Unpacks to different type.
+    self.assertFalse(msg.value.Unpack(msg))
+    # Only Any messages have Pack method.
+    try:
+      msg.Pack(all_types)
+    except AttributeError:
+      pass
+    else:
+      raise AttributeError('%s should not have Pack method.' %
+                           msg_descriptor.full_name)
+
+  def testPackWithCustomTypeUrl(self):
+    submessage = any_test_pb2.TestAny()
+    submessage.int_value = 12345
+    msg = any_pb2.Any()
+    # Pack with a custom type URL prefix.
+    msg.Pack(submessage, 'type.myservice.com')
+    self.assertEqual(msg.type_url,
+                     'type.myservice.com/%s' % submessage.DESCRIPTOR.full_name)
+    # Pack with a custom type URL prefix ending with '/'.
+    msg.Pack(submessage, 'type.myservice.com/')
+    self.assertEqual(msg.type_url,
+                     'type.myservice.com/%s' % submessage.DESCRIPTOR.full_name)
+    # Pack with an empty type URL prefix.
+    msg.Pack(submessage, '')
+    self.assertEqual(msg.type_url,
+                     '/%s' % submessage.DESCRIPTOR.full_name)
+    # Test unpacking the type.
+    unpacked_message = any_test_pb2.TestAny()
+    self.assertTrue(msg.Unpack(unpacked_message))
+    self.assertEqual(submessage, unpacked_message)
+
+
 if __name__ == '__main__':
   unittest.main()