WIP.

js/commonjs_export.js

+/**
+ * @fileoverview Export symbols needed by generated code in CommonJS style.
+ */
+
+exports = {
+  Message: jspb.Message,
+  BinaryReader: jspb.BinaryReader,
+  BinaryWriter: jspb.BinaryWriter,
+  ExtensionFieldInfo: jspb.ExtensionFieldInfo,
+};

js/gulpfile.js

 var gulp = require('gulp');
 var exec = require('child_process').exec;
 
-gulp.task('genproto', function (cb) {
+gulp.task('genproto_closure', function (cb) {
   exec('../src/protoc --js_out=library=testproto_libs,binary:. -I ../src -I . *.proto ../src/google/protobuf/descriptor.proto',
        function (err, stdout, stderr) {
     console.log(stdout);
     console.log(stderr);
     cb(err);
   });
-})
+});
+
+gulp.task('genproto_commonjs', function (cb) {
+  exec('mkdir -p commonjs_out && ../src/protoc --js_out=import_style=commonjs,binary:commonjs_out -I ../src -I . *.proto ../src/google/protobuf/descriptor.proto',
+       function (err, stdout, stderr) {
+    console.log(stdout);
+    console.log(stderr);
+    cb(err);
+  });
+});
 
-gulp.task('deps', ['genproto'], function (cb) {
+gulp.task('dist', function (cb) {
+  // TODO(haberman): minify this more aggressively.
+  // Will require proper externs/exports.
+  exec('./node_modules/google-closure-library/closure/bin/calcdeps.py -i message.js -i binary/reader.js -i binary/writer.js -p . -p node_modules/google-closure-library/closure -o compiled --compiler_jar node_modules/google-closure-compiler/compiler.jar > google-protobuf.js',
+       function (err, stdout, stderr) {
+    console.log(stdout);
+    console.log(stderr);
+    cb(err);
+  });
+});
+
+gulp.task('deps', ['genproto_closure'], function (cb) {
   exec('./node_modules/google-closure-library/closure/bin/build/depswriter.py *.js binary/*.js > deps.js',
        function (err, stdout, stderr) {
     console.log(stdout);
     console.log(stderr);
     cb(err);
   });
-})
+});
 
-gulp.task('test', ['genproto', 'deps'], function (cb) {
+gulp.task('test_closure', ['genproto_closure', 'deps'], function (cb) {
   exec('JASMINE_CONFIG_PATH=jasmine.json ./node_modules/.bin/jasmine',
        function (err, stdout, stderr) {
     console.log(stdout);
@@ -27,3 +47,12 @@ gulp.task('test', ['genproto', 'deps'], function (cb) {
     cb(err);
   });
 });
+
+gulp.task('test_commonjs', ['genproto_commonjs', 'dist'], function (cb) {
+  exec('JASMINE_CONFIG_PATH=jasmine.json cp jasmine_commonjs.json commonjs_out/jasmine.json && cd commonjs_out && ../node_modules/.bin/jasmine',
+       function (err, stdout, stderr) {
+    console.log(stdout);
+    console.log(stderr);
+    cb(err);
+  });
+});

js/jasmine_commonjs.json

+{
+    "spec_dir": "",
+    "spec_files": [
+        "*_test.js"
+    ],
+    "helpers": [
+        "node_modules/google-closure-library/closure/goog/bootstrap/nodejs.js",
+        "node_loader.js",
+        "deps.js"
+    ]
+}

js/package.json

@@ -2,13 +2,15 @@
   "name": "google-protobuf",
   "version": "3.0.0-alpha.5",
   "description": "Protocol Buffers for JavaScript",
-  "main": "debug.js",
+  "main": "google-protobuf.js",
   "dependencies": {
     "google-closure-library": "~20160125.0.0",
     "gulp": "~3.9.0",
     "jasmine": "~2.4.1"
   },
-  "devDependencies": {},
+  "devDependencies": {
+    "google-closure-compiler": "~20151216.2.0"
+  },
   "scripts": {
     "test": "./node_modules/gulp/bin/gulp.js test"
   },

src/google/protobuf/compiler/js/js_generator.cc

@@ -134,12 +134,37 @@ bool IsReserved(const string& ident) {
 
 // Returns a copy of |filename| with any trailing ".protodevel" or ".proto
 // suffix stripped.
+// TODO(robinson): Unify with copy in compiler/cpp/internal/helpers.cc.
 string StripProto(const string& filename) {
   const char* suffix = HasSuffixString(filename, ".protodevel")
       ? ".protodevel" : ".proto";
   return StripSuffixString(filename, suffix);
 }
 
+// Given a filename like foo/bar/baz.proto, returns the correspoding JavaScript
+// file foo/bar/baz.js.
+string GetJSFilename(const string& filename) {
+  const char* suffix = HasSuffixString(filename, ".protodevel")
+      ? ".protodevel" : ".proto";
+  return StripSuffixString(filename, suffix) + "_pb.js";
+}
+
+// Returns the alias we assign to the module of the given .proto filename
+// when importing.
+string ModuleAlias(const string& filename) {
+  // This scheme could technically cause problems if a file includes any 2 of:
+  //   foo/bar_baz.proto
+  //   foo_bar_baz.proto
+  //   foo_bar/baz.proto
+  //
+  // We'll worry about this problem if/when we actually see it.  This name isn't
+  // exposed to users so we can change it later if we need to.
+  string basename = StripProto(filename);
+  StripString(&basename, "-", '$');
+  StripString(&basename, "/", '_');
+  return basename + "_pb";
+}
+
 // Returns the fully normalized JavaScript path for the given
 // file descriptor's package.
 string GetPath(const GeneratorOptions& options,
@@ -215,6 +240,26 @@ string GetPath(const GeneratorOptions& options,
       value_descriptor->type()) + "." + value_descriptor->name();
 }
 
+string MaybeCrossFileRef(const GeneratorOptions& options,
+                         const FileDescriptor* from_file,
+                         const Descriptor* to_message) {
+  if (options.import_style == GeneratorOptions::IMPORT_COMMONJS &&
+      from_file != to_message->file()) {
+    // Cross-file ref in CommonJS needs to use the module alias instead of
+    // the global name.
+    return ModuleAlias(to_message->file()->name()) + "." + to_message->name();
+  } else {
+    // Within a single file we use a full name.
+    return GetPath(options, to_message);
+  }
+}
+
+string SubmessageTypeRef(const GeneratorOptions& options,
+                         const FieldDescriptor* field) {
+  GOOGLE_CHECK(field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE);
+  return MaybeCrossFileRef(options, field->file(), field->message_type());
+}
+
 // - Object field name: LOWER_UNDERSCORE -> LOWER_CAMEL, except for group fields
 // (UPPER_CAMEL -> LOWER_CAMEL), with "List" (or "Map") appended if appropriate,
 // and with reserved words triggering a "pb_" prefix.
@@ -952,11 +997,13 @@ string RelativeTypeName(const FieldDescriptor* field) {
 }
 
 string JSExtensionsObjectName(const GeneratorOptions& options,
+                              const FileDescriptor* from_file,
                               const Descriptor* desc) {
   if (desc->full_name() == "google.protobuf.bridge.MessageSet") {
+    // TODO(haberman): fix this for the IMPORT_COMMONJS case.
     return "jspb.Message.messageSetExtensions";
   } else {
-    return GetPath(options, desc) + ".extensions";
+    return MaybeCrossFileRef(options, from_file, desc) + ".extensions";
   }
 }
 
@@ -1113,19 +1160,24 @@ void Generator::GenerateHeader(const GeneratorOptions& options,
                  "\n");
 }
 
+void Generator::FindProvidesForFile(const GeneratorOptions& options,
+                                    io::Printer* printer,
+                                    const FileDescriptor* file,
+                                    std::set<string>* provided) const {
+  for (int i = 0; i < file->message_type_count(); i++) {
+    FindProvidesForMessage(options, printer, file->message_type(i), provided);
+  }
+  for (int i = 0; i < file->enum_type_count(); i++) {
+    FindProvidesForEnum(options, printer, file->enum_type(i), provided);
+  }
+}
+
 void Generator::FindProvides(const GeneratorOptions& options,
                              io::Printer* printer,
                              const vector<const FileDescriptor*>& files,
                              std::set<string>* provided) const {
   for (int i = 0; i < files.size(); i++) {
-    for (int j = 0; j < files[i]->message_type_count(); j++) {
-      FindProvidesForMessage(options, printer, files[i]->message_type(j),
-                                 provided);
-    }
-    for (int j = 0; j < files[i]->enum_type_count(); j++) {
-      FindProvidesForEnum(options, printer, files[i]->enum_type(j),
-                              provided);
-    }
+    FindProvidesForFile(options, printer, files[i], provided);
   }
 
   printer->Print("\n");
@@ -1204,6 +1256,10 @@ void Generator::GenerateRequires(const GeneratorOptions& options,
                                  io::Printer* printer,
                                  const vector<const FileDescriptor*>& files,
                                  std::set<string>* provided) const {
+  if (options.import_style == GeneratorOptions::IMPORT_BROWSER) {
+    return;
+  } else if (options.import_style == GeneratorOptions::IMPORT_CLOSURE) {
+    // For Closure imports we need to import every message type individually.
     std::set<string> required;
     std::set<string> forwards;
     bool have_extensions = false;
@@ -1236,6 +1292,9 @@ void Generator::GenerateRequires(const GeneratorOptions& options,
     GenerateRequiresImpl(options, printer, &required, &forwards, provided,
                          /* require_jspb = */ have_message,
                          /* require_extension = */ have_extensions);
+  } else if (options.import_style == GeneratorOptions::IMPORT_COMMONJS) {
+    // CommonJS imports are based on files
+  }
 }
 
 void Generator::GenerateRequires(const GeneratorOptions& options,
@@ -1406,6 +1465,19 @@ void Generator::GenerateClass(const GeneratorOptions& options,
     if (IsExtendable(desc) && desc->full_name() != "google.protobuf.bridge.MessageSet") {
       GenerateClassExtensionFieldInfo(options, printer, desc);
     }
+
+    if (options.import_style == GeneratorOptions::IMPORT_COMMONJS) {
+      printer->Print(
+          "exports.$name$ = $fullName$;\n",
+          "name", desc->name(),
+          "fullName", GetPath(options, desc));
+    }
+
+    if (options.import_style != GeneratorOptions:: IMPORT_CLOSURE) {
+      for (int i = 0; i < desc->extension_count(); i++) {
+        GenerateExtension(options, printer, desc->extension(i));
+      }
+    }
   }
 
   // Recurse on nested types.
@@ -1623,7 +1695,7 @@ void Generator::GenerateClassToObject(const GeneratorOptions& options,
         "obj,\n"
         "      $extObject$, $class$.prototype.getExtension,\n"
         "      includeInstance);\n",
-        "extObject", JSExtensionsObjectName(options, desc),
+        "extObject", JSExtensionsObjectName(options, desc->file(), desc),
         "class", GetPath(options, desc));
   }
 
@@ -1652,13 +1724,13 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options,
         printer->Print("jspb.Message.toObjectList(msg.get$getter$(),\n"
                        "    $type$.toObject, includeInstance)",
                        "getter", JSGetterName(field),
-                       "type", GetPath(options, field->message_type()));
+                       "type", SubmessageTypeRef(options, field));
       }
     } else {
       printer->Print("(f = msg.get$getter$()) && "
                      "$type$.toObject(includeInstance, f)",
                      "getter", JSGetterName(field),
-                     "type", GetPath(options, field->message_type()));
+                     "type", SubmessageTypeRef(options, field));
     }
   } else {
     // Simple field (singular or repeated).
@@ -1723,7 +1795,7 @@ void Generator::GenerateClassFieldFromObject(
             "      }));\n",
             "name", JSObjectFieldName(field),
             "index", JSFieldIndex(field),
-            "fieldclass", GetPath(options, field->message_type()));
+            "fieldclass", SubmessageTypeRef(options, field));
       }
     } else {
       printer->Print(
@@ -1731,7 +1803,7 @@ void Generator::GenerateClassFieldFromObject(
           "      msg, $index$, $fieldclass$.fromObject(obj.$name$));\n",
           "name", JSObjectFieldName(field),
           "index", JSFieldIndex(field),
-          "fieldclass", GetPath(options, field->message_type()));
+          "fieldclass", SubmessageTypeRef(options, field));
     }
   } else {
     // Simple (primitive) field.
@@ -1815,7 +1887,7 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
                                       /* always_singular = */ false),
         "rpt", (field->is_repeated() ? "Repeated" : ""),
         "index", JSFieldIndex(field),
-        "wrapperclass", GetPath(options, field->message_type()),
+        "wrapperclass", SubmessageTypeRef(options, field),
         "required", (field->label() == FieldDescriptor::LABEL_REQUIRED ?
                      ", 1" : ""));
     printer->Print(
@@ -2043,7 +2115,7 @@ void Generator::GenerateClassDeserializeBinary(const GeneratorOptions& options,
         "        $class$.prototype.getExtension,\n"
         "        $class$.prototype.setExtension);\n"
         "      break;\n",
-        "extobj", JSExtensionsObjectName(options, desc),
+        "extobj", JSExtensionsObjectName(options, desc->file(), desc),
         "class", GetPath(options, desc));
   } else {
     printer->Print(
@@ -2073,7 +2145,7 @@ void Generator::GenerateClassDeserializeBinaryField(
         "      var value = new $fieldclass$;\n"
         "      reader.read$msgOrGroup$($grpfield$value,"
         "$fieldclass$.deserializeBinaryFromReader);\n",
-        "fieldclass", GetPath(options, field->message_type()),
+        "fieldclass", SubmessageTypeRef(options, field),
         "msgOrGroup", (field->type() == FieldDescriptor::TYPE_GROUP) ?
                       "Group" : "Message",
         "grpfield", (field->type() == FieldDescriptor::TYPE_GROUP) ?
@@ -2149,7 +2221,7 @@ void Generator::GenerateClassSerializeBinary(const GeneratorOptions& options,
     printer->Print(
         "  jspb.Message.serializeBinaryExtensions(this, writer, $extobj$,\n"
         "    $class$.prototype.getExtension);\n",
-        "extobj", JSExtensionsObjectName(options, desc),
+        "extobj", JSExtensionsObjectName(options, desc->file(), desc),
         "class", GetPath(options, desc));
   }
 
@@ -2222,7 +2294,7 @@ void Generator::GenerateClassSerializeBinaryField(
     printer->Print(
         ",\n"
         "      $submsg$.serializeBinaryToWriter\n",
-        "submsg", GetPath(options, field->message_type()));
+        "submsg", SubmessageTypeRef(options, field));
   } else {
     printer->Print("\n");
   }
@@ -2290,9 +2362,9 @@ void Generator::GenerateExtension(const GeneratorOptions& options,
       "index", SimpleItoa(field->number()),
       "name", JSObjectFieldName(field),
       "ctor", (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ?
-               GetPath(options, field->message_type()) : string("null")),
+               SubmessageTypeRef(options, field) : string("null")),
       "toObject", (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ?
-                   (GetPath(options, field->message_type()) + ".toObject") :
+                   (SubmessageTypeRef(options, field) + ".toObject") :
                    string("null")),
       "repeated", (field->is_repeated() ? "1" : "0"));
 
@@ -2308,11 +2380,11 @@ void Generator::GenerateExtension(const GeneratorOptions& options,
         "binaryWriterFn", JSBinaryWriterMethodName(field),
         "binaryMessageSerializeFn",
         (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) ?
-        (GetPath(options, field->message_type()) +
+        (SubmessageTypeRef(options, field) +
          ".serializeBinaryToWriter") : "null",
         "binaryMessageDeserializeFn",
         (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) ?
-        (GetPath(options, field->message_type()) +
+        (SubmessageTypeRef(options, field) +
          ".deserializeBinaryFromReader") : "null",
         "isPacked", (field->is_packed() ? "true" : "false"));
   } else {
@@ -2324,7 +2396,8 @@ void Generator::GenerateExtension(const GeneratorOptions& options,
       "// toObject() will function correctly.\n"
       "$extendName$[$index$] = $class$.$name$;\n"
       "\n",
-      "extendName", JSExtensionsObjectName(options, field->containing_type()),
+      "extendName", JSExtensionsObjectName(options, field->file(),
+                                           field->containing_type()),
       "index", SimpleItoa(field->number()),
       "class", extension_scope,
       "name", JSObjectFieldName(field));
@@ -2364,6 +2437,19 @@ bool GeneratorOptions::ParseFromOptions(
       namespace_prefix = options[i].second;
     } else if (options[i].first == "library") {
       library = options[i].second;
+    } else if (options[i].first == "import_style") {
+      if (options[i].second == "closure") {
+        import_style = IMPORT_CLOSURE;
+      } else if (options[i].second == "commonjs") {
+        import_style = IMPORT_COMMONJS;
+      } else if (options[i].second == "browser") {
+        import_style = IMPORT_BROWSER;
+      } else if (options[i].second == "es6") {
+        import_style = IMPORT_ES6;
+      } else {
+        *error = "Unknown import style " + options[i].second + ", expected " +
+                 "one of: closure, commonjs, browser, es6.";
+      }
     } else {
       // Assume any other option is an output directory, as long as it is a bare
       // `key` rather than a `key=value` option.
@@ -2375,6 +2461,11 @@ bool GeneratorOptions::ParseFromOptions(
     }
   }
 
+  if (!library.empty() && import_style != IMPORT_CLOSURE) {
+    *error = "The library option should only be used for "
+             "import_style=closure";
+  }
+
   return true;
 }
 
@@ -2418,6 +2509,47 @@ void Generator::GenerateFileAndDeps(
   }
 }
 
+void Generator::GenerateFile(const GeneratorOptions& options,
+                             io::Printer* printer,
+                             const FileDescriptor* file) const {
+  GenerateHeader(options, printer);
+
+  // Generate "require" statements.
+  if (options.import_style == GeneratorOptions::IMPORT_COMMONJS) {
+    for (int i = 0; i < file->dependency_count(); i++) {
+      const std::string& name = file->dependency(i)->name();
+      printer->Print(
+          "var $alias$ = require('$file$');\n",
+          "alias", ModuleAlias(name),
+          "file", GetJSFilename(name));
+    }
+  }
+
+  // We aren't using Closure's import system, but we use goog.exportSymbol()
+  // to construct the expected tree of objects, eg.
+  //
+  //   goog.exportSymbol('foo.bar.Baz', null, this);
+  //
+  //   // Later generated code expects foo.bar = {} to exist:
+  //   foo.bar.Baz = function() { /* ... */ }
+  std::set<std::string> provided;
+  FindProvidesForFile(options, printer, file, &provided);
+  //FindProvidesForFields(options, printer, extensions, &provided);
+  for (std::set<string>::iterator it = provided.begin();
+       it != provided.end(); ++it) {
+    printer->Print("goog.exportSymbol('$name$', null, this);\n",
+                   "name", *it);
+  }
+
+  GenerateClassesAndEnums(options, printer, file);
+
+  // Extensions nested inside messages are emitted inside
+  // GenerateClassesAndEnums().
+  for (int i = 0; i < file->extension_count(); i++) {
+    GenerateExtension(options, printer, file->extension(i));
+  }
+}
+
 bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
                             const string& parameter,
                             GeneratorContext* context,
@@ -2430,10 +2562,14 @@ bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
   }
 
 
-  // We're either generating a single library file with definitions for message
-  // and enum types in *all* FileDescriptor inputs, or we're generating a single
-  // file for each type.
-  if (options.library != "") {
+  // There are three schemes for where output files go:
+  //
+  // - import_style = IMPORT_CLOSURE, library non-empty: all output in one file
+  // - import_style = IMPORT_CLOSURE, library empty: one output file per type
+  // - import_style != IMPORT_CLOSURE: one output file per .proto file
+  if (options.import_style == GeneratorOptions::IMPORT_CLOSURE &&
+      options.library != "") {
+    // All output should go in a single file.
     string filename = options.output_dir + "/" + options.library + ".js";
     google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(context->Open(filename));
     GOOGLE_CHECK(output.get());
@@ -2469,7 +2605,7 @@ bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
     if (printer.failed()) {
       return false;
     }
-  } else {
+  } else if (options.import_style == GeneratorOptions::IMPORT_CLOSURE) {
     // Collect all types, and print each type to a separate file. Pull out
     // free-floating extensions while we make this pass.
     map< string, vector<const FieldDescriptor*> > extensions_by_namespace;
@@ -2611,6 +2747,24 @@ bool Generator::GenerateAll(const vector<const FileDescriptor*>& files,
         }
       }
     }
+  } else {
+    // Generate one output file per input (.proto) file.
+
+    for (int i = 0; i < files.size(); i++) {
+      const google::protobuf::FileDescriptor* file = files[i];
+
+      string filename = options.output_dir + "/" + GetJSFilename(file->name());
+      google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
+          context->Open(filename));
+      GOOGLE_CHECK(output.get());
+      io::Printer printer(output.get(), '$');
+
+      GenerateFile(options, &printer, file);
+
+      if (printer.failed()) {
+        return false;
+      }
+    }
   }
 
   return true;

src/google/protobuf/compiler/js/js_generator.h

@@ -67,6 +67,13 @@ struct GeneratorOptions {
   bool error_on_name_conflict;
   // Enable binary-format support?
   bool binary;
+  // What style of imports should be used.
+  enum ImportStyle {
+    IMPORT_CLOSURE,    // goog.require()
+    IMPORT_COMMONJS,   // require()
+    IMPORT_BROWSER,    // no import statements
+    IMPORT_ES6,        // import { member } from ''
+  } import_style;
 
   GeneratorOptions()
       : add_require_for_enums(false),
@@ -75,7 +82,8 @@ struct GeneratorOptions {
         namespace_prefix(""),
         library(""),
         error_on_name_conflict(false),
-        binary(false) {}
+        binary(false),
+        import_style(IMPORT_CLOSURE) {}
 
   bool ParseFromOptions(
       const vector< pair< string, string > >& options,
@@ -111,6 +119,10 @@ class LIBPROTOC_EXPORT Generator : public CodeGenerator {
                     io::Printer* printer,
                     const vector<const FileDescriptor*>& file,
                     std::set<string>* provided) const;
+  void FindProvidesForFile(const GeneratorOptions& options,
+                           io::Printer* printer,
+                           const FileDescriptor* file,
+                           std::set<string>* provided) const;
   void FindProvidesForMessage(const GeneratorOptions& options,
                               io::Printer* printer,
                               const Descriptor* desc,
@@ -168,6 +180,10 @@ class LIBPROTOC_EXPORT Generator : public CodeGenerator {
                                 std::set<string>* required,
                                 std::set<string>* forwards) const;
 
+  void GenerateFile(const GeneratorOptions& options,
+                    io::Printer* printer,
+                    const FileDescriptor* file) const;
+
   // Generate definitions for all message classes and enums in all files,
   // processing the files in dependence order.
   void GenerateFilesInDepOrder(const GeneratorOptions& options,