Fix Any json encoding/decoding in php (#5496)

* Fix Any json encoding/decoding in php

* Fix comments

conformance/failure_list_php_c.txt

@@ -22,26 +22,8 @@ Required.DurationProtoInputTooLarge.JsonOutput
 Required.DurationProtoInputTooSmall.JsonOutput
 Required.TimestampProtoInputTooLarge.JsonOutput
 Required.TimestampProtoInputTooSmall.JsonOutput
-Required.Proto3.JsonInput.Any.JsonOutput
-Required.Proto3.JsonInput.Any.ProtobufOutput
-Required.Proto3.JsonInput.AnyNested.JsonOutput
-Required.Proto3.JsonInput.AnyNested.ProtobufOutput
-Required.Proto3.JsonInput.AnyUnorderedTypeTag.JsonOutput
-Required.Proto3.JsonInput.AnyUnorderedTypeTag.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithDuration.JsonOutput
-Required.Proto3.JsonInput.AnyWithDuration.ProtobufOutput
 Required.Proto3.JsonInput.AnyWithFieldMask.JsonOutput
 Required.Proto3.JsonInput.AnyWithFieldMask.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithInt32ValueWrapper.JsonOutput
-Required.Proto3.JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithStruct.JsonOutput
-Required.Proto3.JsonInput.AnyWithStruct.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithTimestamp.JsonOutput
-Required.Proto3.JsonInput.AnyWithTimestamp.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithValueForInteger.JsonOutput
-Required.Proto3.JsonInput.AnyWithValueForInteger.ProtobufOutput
-Required.Proto3.JsonInput.AnyWithValueForJsonObject.JsonOutput
-Required.Proto3.JsonInput.AnyWithValueForJsonObject.ProtobufOutput
 Required.Proto3.JsonInput.BoolMapField.JsonOutput
 Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.JsonOutput
 Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput

php/ext/google/protobuf/encode_decode.c

@@ -1095,7 +1095,10 @@ static const upb_json_parsermethod *msgdef_jsonparsermethod(Descriptor* desc) {
 static void putmsg(zval* msg, const Descriptor* desc, upb_sink* sink,
                    int depth, bool is_json TSRMLS_DC);
 static void putrawmsg(MessageHeader* msg, const Descriptor* desc,
-                      upb_sink* sink, int depth, bool is_json TSRMLS_DC);
+                      upb_sink* sink, int depth, bool is_json,
+                      bool open_msg TSRMLS_DC);
+static void putjsonany(MessageHeader* msg, const Descriptor* desc,
+                       upb_sink* sink, int depth TSRMLS_DC);
 
 static void putstr(zval* str, const upb_fielddef* f, upb_sink* sink,
                    bool force_default);
@@ -1245,15 +1248,114 @@ static void putmap(zval* map, const upb_fielddef* f, upb_sink* sink,
 static void putmsg(zval* msg_php, const Descriptor* desc, upb_sink* sink,
                    int depth, bool is_json TSRMLS_DC) {
   MessageHeader* msg = UNBOX(MessageHeader, msg_php);
-  putrawmsg(msg, desc, sink, depth, is_json TSRMLS_CC);
+  putrawmsg(msg, desc, sink, depth, is_json, true TSRMLS_CC);
+}
+
+static const upb_handlers* msgdef_json_serialize_handlers(
+    Descriptor* desc, bool preserve_proto_fieldnames);
+
+static void putjsonany(MessageHeader* msg, const Descriptor* desc,
+                       upb_sink* sink, int depth TSRMLS_DC) {
+  upb_status status;
+  const upb_fielddef* type_field = upb_msgdef_itof(desc->msgdef, UPB_ANY_TYPE);
+  const upb_fielddef* value_field = upb_msgdef_itof(desc->msgdef, UPB_ANY_VALUE);
+
+  uint32_t type_url_offset;
+  zval* type_url_php_str;
+  const upb_msgdef *payload_type = NULL;
+
+  upb_sink_startmsg(sink);
+
+  /* Handle type url */
+  type_url_offset = desc->layout->fields[upb_fielddef_index(type_field)].offset;
+  type_url_php_str = CACHED_PTR_TO_ZVAL_PTR(
+      DEREF(message_data(msg), type_url_offset, CACHED_VALUE*));
+  if (Z_STRLEN_P(type_url_php_str) > 0) {
+    putstr(type_url_php_str, type_field, sink, false);
+  }
+
+  {
+    const char* type_url_str = Z_STRVAL_P(type_url_php_str);
+    size_t type_url_len = Z_STRLEN_P(type_url_php_str);
+    if (type_url_len <= 20 ||
+        strncmp(type_url_str, "type.googleapis.com/", 20) != 0) {
+      zend_error(E_ERROR, "Invalid type url: %s", type_url_str);
+    }
+
+    /* Resolve type url */
+    type_url_str += 20;
+    type_url_len -= 20;
+
+    payload_type = upb_symtab_lookupmsg2(
+        generated_pool->symtab, type_url_str, type_url_len);
+    if (payload_type == NULL) {
+      zend_error(E_ERROR, "Unknown type: %s", type_url_str);
+      return;
+    }
+  }
+
+  {
+    uint32_t value_offset;
+    zval* value_php_str;
+    const char* value_str;
+    size_t value_len;
+
+    value_offset = desc->layout->fields[upb_fielddef_index(value_field)].offset;
+    value_php_str = CACHED_PTR_TO_ZVAL_PTR(
+        DEREF(message_data(msg), value_offset, CACHED_VALUE*));
+    value_str = Z_STRVAL_P(value_php_str);
+    value_len = Z_STRLEN_P(value_php_str);
+
+    if (value_len > 0) {
+      Descriptor* payload_desc =
+          UNBOX_HASHTABLE_VALUE(Descriptor, get_def_obj((void*)payload_type));
+      zend_class_entry* payload_klass = payload_desc->klass;
+      zval val;
+      upb_sink subsink;
+      bool is_wellknown;
+
+      /* Create message of the payload type. */
+      ZVAL_OBJ(&val, payload_klass->create_object(payload_klass TSRMLS_CC));
+      MessageHeader* intern = UNBOX(MessageHeader, &val);
+      custom_data_init(payload_klass, intern PHP_PROTO_TSRMLS_CC);
+
+      merge_from_string(value_str, value_len, payload_desc, intern);
+
+      is_wellknown =
+          upb_msgdef_wellknowntype(payload_desc->msgdef) !=
+              UPB_WELLKNOWN_UNSPECIFIED;
+      if (is_wellknown) {
+        upb_sink_startstr(sink, getsel(value_field, UPB_HANDLER_STARTSTR), 0,
+                          &subsink);
+      }
+
+      subsink.handlers =
+          msgdef_json_serialize_handlers(payload_desc, true);
+      subsink.closure = sink->closure;
+      putrawmsg(intern, payload_desc, &subsink, depth, true,
+                is_wellknown TSRMLS_CC);
+
+      zval_dtor(&val);
+    }
+  }
+
+  upb_sink_endmsg(sink, &status);
 }
 
 static void putrawmsg(MessageHeader* msg, const Descriptor* desc,
-                      upb_sink* sink, int depth, bool is_json TSRMLS_DC) {
+                      upb_sink* sink, int depth, bool is_json,
+                      bool open_msg TSRMLS_DC) {
   upb_msg_field_iter i;
   upb_status status;
 
-  upb_sink_startmsg(sink);
+  if (is_json && upb_msgdef_wellknowntype(desc->msgdef) == UPB_WELLKNOWN_ANY) {
+    putjsonany(msg, desc, sink, depth TSRMLS_CC);
+    return;
+  }
+
+  if (open_msg) {
+    upb_sink_startmsg(sink);
+  }
 
   // Protect against cycles (possible because users may freely reassign message
   // and repeated fields) by imposing a maximum recursion depth.
@@ -1343,7 +1445,9 @@ static void putrawmsg(MessageHeader* msg, const Descriptor* desc,
     upb_sink_putunknown(sink, unknown->ptr, unknown->len);
   }
 
-  upb_sink_endmsg(sink, &status);
+  if (open_msg) {
+    upb_sink_endmsg(sink, &status);
+  }
 }
 
 static void putstr(zval* str, const upb_fielddef *f,
@@ -1409,7 +1513,7 @@ static void putrawsubmsg(MessageHeader* submsg, const upb_fielddef* f,
       UNBOX_HASHTABLE_VALUE(Descriptor, get_def_obj(upb_fielddef_msgsubdef(f)));
 
   upb_sink_startsubmsg(sink, getsel(f, UPB_HANDLER_STARTSUBMSG), &subsink);
-  putrawmsg(submsg, subdesc, &subsink, depth + 1, is_json TSRMLS_CC);
+  putrawmsg(submsg, subdesc, &subsink, depth + 1, is_json, true TSRMLS_CC);
   upb_sink_endsubmsg(sink, getsel(f, UPB_HANDLER_ENDSUBMSG));
 }
 
@@ -1633,7 +1737,8 @@ PHP_METHOD(Message, mergeFromJsonString) {
     stackenv_init(&se, "Error occurred during parsing: %s");
 
     upb_sink_reset(&sink, get_fill_handlers(desc), msg);
-    parser = upb_json_parser_create(&se.env, method, &sink, ignore_json_unknown);
+    parser = upb_json_parser_create(&se.env, method, generated_pool->symtab,
+                                    &sink, ignore_json_unknown);
     upb_bufsrc_putbuf(data, data_len, upb_json_parser_input(parser));
 
     stackenv_uninit(&se);

php/tests/encode_decode_test.php

@@ -5,12 +5,14 @@ require_once('test_util.php');
 
 use Google\Protobuf\RepeatedField;
 use Google\Protobuf\GPBType;
+use Foo\TestAny;
 use Foo\TestEnum;
 use Foo\TestMessage;
 use Foo\TestMessage\Sub;
 use Foo\TestPackedMessage;
 use Foo\TestRandomFieldOrder;
 use Foo\TestUnpackedMessage;
+use Google\Protobuf\Any;
 use Google\Protobuf\DoubleValue;
 use Google\Protobuf\FloatValue;
 use Google\Protobuf\Int32Value;
@@ -915,4 +917,143 @@ class EncodeDecodeTest extends TestBase
         $this->assertSame("{\"a\":1.5}", $m->serializeToJsonString());
     }
 
+    public function testDecodeTopLevelAny()
+    {
+        // Make sure packed message has been created at least once.
+        $packed = new TestMessage();
+
+        $m1 = new Any();
+        $m1->mergeFromJsonString(
+            "{\"optionalInt32\": 1, " .
+            "\"@type\":\"type.googleapis.com/foo.TestMessage\"}");
+        $this->assertSame("type.googleapis.com/foo.TestMessage",
+                          $m1->getTypeUrl());
+        $this->assertSame("0801", bin2hex($m1->getValue()));
+
+        $m2 = new Any();
+        $m2->mergeFromJsonString(
+            "{\"@type\":\"type.googleapis.com/foo.TestMessage\", " .
+            "\"optionalInt32\": 1}");
+        $this->assertSame("type.googleapis.com/foo.TestMessage",
+                          $m2->getTypeUrl());
+        $this->assertSame("0801", bin2hex($m2->getValue()));
+
+        $m3 = new Any();
+        $m3->mergeFromJsonString(
+            "{\"optionalInt32\": 1, " .
+            "\"@type\":\"type.googleapis.com/foo.TestMessage\", " .
+            "\"optionalInt64\": 2}");
+        $this->assertSame("type.googleapis.com/foo.TestMessage",
+                          $m3->getTypeUrl());
+        $this->assertSame("08011002", bin2hex($m3->getValue()));
+    }
+
+    public function testDecodeAny()
+    {
+        // Make sure packed message has been created at least once.
+        $packed = new TestMessage();
+
+        $m1 = new TestAny();
+        $m1->mergeFromJsonString(
+            "{\"any\": {\"optionalInt32\": 1, " .
+            "\"@type\":\"type.googleapis.com/foo.TestMessage\"}}");
+        $this->assertSame("type.googleapis.com/foo.TestMessage",
+                          $m1->getAny()->getTypeUrl());
+        $this->assertSame("0801", bin2hex($m1->getAny()->getValue()));
+
+        $m2 = new TestAny();
+        $m2->mergeFromJsonString(
+            "{\"any\":{\"@type\":\"type.googleapis.com/foo.TestMessage\", " .
+            "\"optionalInt32\": 1}}");
+        $this->assertSame("type.googleapis.com/foo.TestMessage",
+                          $m2->getAny()->getTypeUrl());
+        $this->assertSame("0801", bin2hex($m2->getAny()->getValue()));
+
+        $m3 = new TestAny();
+        $m3->mergeFromJsonString(
+            "{\"any\":{\"optionalInt32\": 1, " .
+            "\"@type\":\"type.googleapis.com/foo.TestMessage\", " .
+            "\"optionalInt64\": 2}}");
+        $this->assertSame("type.googleapis.com/foo.TestMessage",
+                          $m3->getAny()->getTypeUrl());
+        $this->assertSame("08011002", bin2hex($m3->getAny()->getValue()));
+    }
+
+    public function testDecodeAnyWithWellKnownPacked()
+    {
+        // Make sure packed message has been created at least once.
+        $packed = new Int32Value();
+
+        $m1 = new TestAny();
+        $m1->mergeFromJsonString(
+            "{\"any\":" .
+            "  {\"@type\":\"type.googleapis.com/google.protobuf.Int32Value\"," .
+            "   \"value\":1}}");
+        $this->assertSame("type.googleapis.com/google.protobuf.Int32Value",
+                          $m1->getAny()->getTypeUrl());
+        $this->assertSame("0801", bin2hex($m1->getAny()->getValue()));
+    }
+
+    /**
+     * @expectedException Exception
+     */
+    public function testDecodeAnyWithUnknownPacked()
+    {
+        $m = new TestAny();
+        $m->mergeFromJsonString(
+            "{\"any\":" .
+            "  {\"@type\":\"type.googleapis.com/unknown\"," .
+            "   \"value\":1}}");
+    }
+
+    public function testEncodeTopLevelAny()
+    {
+        // Test a normal message.
+        $packed = new TestMessage();
+        $packed->setOptionalInt32(123);
+        $packed->setOptionalString("abc");
+
+        $m = new Any();
+        $m->pack($packed);
+        $expected1 =
+            "{\"@type\":\"type.googleapis.com/foo.TestMessage\"," .
+            "\"optional_int32\":123,\"optional_string\":\"abc\"}";
+        $expected2 =
+            "{\"@type\":\"type.googleapis.com/foo.TestMessage\"," .
+            "\"optionalInt32\":123,\"optionalString\":\"abc\"}";
+        $result = $m->serializeToJsonString();
+        $this->assertTrue($expected1 === $result || $expected2 === $result);
+
+        // Test a well known message.
+        $packed = new Int32Value();
+        $packed->setValue(123);
+
+        $m = new Any();
+        $m->pack($packed);
+        $this->assertSame(
+            "{\"@type\":\"type.googleapis.com/google.protobuf.Int32Value\"," .
+            "\"value\":123}",
+            $m->serializeToJsonString());
+
+        // Test an Any message.
+        $outer = new Any();
+        $outer->pack($m);
+        $this->assertSame(
+            "{\"@type\":\"type.googleapis.com/google.protobuf.Any\"," .
+            "\"value\":{\"@type\":\"type.googleapis.com/google.protobuf.Int32Value\"," .
+            "\"value\":123}}",
+            $outer->serializeToJsonString());
+
+        // Test a Timestamp message.
+        $packed = new Google\Protobuf\Timestamp();
+        $packed->setSeconds(946684800);
+        $packed->setNanos(123456789);
+        $m = new Any();
+        $m->pack($packed);
+        $this->assertSame(
+            "{\"@type\":\"type.googleapis.com/google.protobuf.Timestamp\"," .
+            "\"value\":\"2000-01-01T00:00:00.123456789Z\"}",
+            $m->serializeToJsonString());
+    }
+
 }

php/tests/gdb_test.sh

@@ -11,7 +11,8 @@ php -i | grep "Configuration"
 # gdb --args php -dextension=../ext/google/protobuf/modules/protobuf.so `which
 # phpunit` --bootstrap autoload.php tmp_test.php
 #
-gdb --args php -dextension=../ext/google/protobuf/modules/protobuf.so `which phpunit` --bootstrap autoload.php generated_class_test.php
+# gdb --args php -dextension=../ext/google/protobuf/modules/protobuf.so `which phpunit` --bootstrap autoload.php generated_class_test.php
+gdb --args php -dextension=../ext/google/protobuf/modules/protobuf.so `which phpunit` --bootstrap autoload.php encode_decode_test.php
 #
 # gdb --args php -dextension=../ext/google/protobuf/modules/protobuf.so memory_leak_test.php
 #

php/tests/memory_leak_test.php

@@ -19,6 +19,7 @@ require_once('generated/Bar/TestLegacyMessage/NestedEnum.php');
 require_once('generated/Bar/TestLegacyMessage/NestedMessage.php');
 require_once('generated/Foo/PBARRAY.php');
 require_once('generated/Foo/PBEmpty.php');
+require_once('generated/Foo/TestAny.php');
 require_once('generated/Foo/TestEnum.php');
 require_once('generated/Foo/TestIncludeNamespaceMessage.php');
 require_once('generated/Foo/TestIncludePrefixMessage.php');
@@ -49,6 +50,7 @@ require_once('test_util.php');
 
 use Google\Protobuf\Internal\RepeatedField;
 use Google\Protobuf\Internal\GPBType;
+use Foo\TestAny;
 use Foo\TestMessage;
 use Foo\TestMessage\Sub;
 
@@ -191,3 +193,16 @@ $to = new TestMessage();
 TestUtil::setTestMessage($from);
 $to->mergeFrom($from);
 TestUtil::assertTestMessage($to);
+
+// Test decode Any
+// Make sure packed message has been created at least once.
+$packed = new TestMessage();
+
+$m = new TestAny();
+$m->mergeFromJsonString(
+    "{\"any\":" .
+    "  {\"@type\":\"type.googleapis.com/foo.TestMessage\"," .
+    "   \"optionalInt32\":1}}");
+assert("type.googleapis.com/foo.TestMessage" ===
+       $m->getAny()->getTypeUrl());
+assert("0801" === bin2hex($m->getAny()->getValue()));

php/tests/proto/test.proto

 syntax = "proto3";
 
+import 'google/protobuf/any.proto';
 import 'proto/test_include.proto';
 import 'proto/test_no_namespace.proto';
 import 'proto/test_php_namespace.proto';
@@ -201,3 +202,7 @@ message testLowerCaseMessage {
 enum testLowerCaseEnum {
   VALUE = 0;
 }
+
+message TestAny {
+  google.protobuf.Any any = 1;
+}