Implement code generator plugins. The Cap'n Proto compiler, when given -o foo,…

Implement code generator plugins.  The Cap'n Proto compiler, when given -o foo, invokes the binary capnp-foo and passes the schema, in Cap'n Proto format, to its stdin.  The binary is executed with the current working directory set to the desired output directory, so all it has to do is write the files.

c++/src/capnproto/message.h

@@ -167,6 +167,12 @@ static typename RootType::Reader readMessageTrusted(const word* data);
 // a message MyMessage, you can read its default value like so:
 //    MyMessage::Reader reader = Message<MyMessage>::ReadTrusted(MyMessage::DEFAULT.words);
 
+template <typename Type>
+static typename Type::Reader defaultValue();
+// Get a default instance of the given struct or list type.
+//
+// TODO(cleanup):  Find a better home for this function?
+
 // =======================================================================================
 
 class SegmentArrayMessageReader: public MessageReader {
@@ -282,6 +288,12 @@ typename RootType::Reader readMessageTrusted(const word* data) {
   return typename RootType::Reader(internal::StructReader::readRootTrusted(data));
 }
 
+template <typename Type>
+static typename Type::Reader defaultValue() {
+  // TODO(soon):  Correctly handle lists.  Maybe primitives too?
+  return typename Type::Reader(internal::StructReader());
+}
+
 }  // namespace capnproto
 
 #endif  // CAPNPROTO_MESSAGE_H_

c++/src/capnproto/type-safety.h

@@ -212,7 +212,19 @@ inline Array<T> newArray(size_t size) {
 
 template <typename T>
 class ArrayBuilder {
-  union Slot { T value; char dummy; };
+  // TODO(cleanup):  This class doesn't work for non-primitive types because Slot is not
+  //   constructable.  Giving Slot a constructor/destructor means arrays of it have to be tagged
+  //   so operator delete can run the destructors.  If we reinterpret_cast the array to an array
+  //   of T and delete it as that type, operator delete gets very upset.
+  //
+  //   Perhaps we should bite the bullet and make the Array family do manual memory allocation,
+  //   bypassing the rather-stupid C++ array new/delete operators which store a redundant copy of
+  //   the size anyway.
+
+  union Slot {
+    T value;
+    char dummy;
+  };
   static_assert(sizeof(Slot) == sizeof(T), "union is bigger than content?");
 
 public:

c++/src/capnproto/util.c++

@@ -51,6 +51,20 @@ STRINGIFY_INT(const void*, "%p");
 
 #undef STRINGIFY_INT
 
+#define HEXIFY_INT(type, format) \
+CappedArray<char, sizeof(type) * 4> hex(type i) { \
+  CappedArray<char, sizeof(type) * 4> result; \
+  result.setSize(sprintf(result.begin(), format, i)); \
+  return result; \
+}
+
+HEXIFY_INT(unsigned short, "%x");
+HEXIFY_INT(unsigned int, "%x");
+HEXIFY_INT(unsigned long, "%lx");
+HEXIFY_INT(unsigned long long, "%llx");
+
+#undef HEXIFY_INT
+
 namespace {
 
 // ----------------------------------------------------------------------

c++/src/capnproto/util.h

@@ -32,6 +32,7 @@
 #include <utility>
 #include <type_traits>
 #include "type-safety.h"
+#include "blob.h"
 #include <string.h>
 
 namespace capnproto {
@@ -82,9 +83,15 @@ public:
   inline const T* begin() const { return content; }
   inline const T* end() const { return content + currentSize; }
 
-  inline operator ArrayPtr<T>() const {
-    return arrayPtr(content, fixedSize);
+  inline operator ArrayPtr<T>() {
+    return arrayPtr(content, currentSize);
   }
+  inline operator ArrayPtr<const T>() const {
+    return arrayPtr(content, currentSize);
+  }
+
+  inline T& operator[](size_t index) { return content[index]; }
+  inline const T& operator[](size_t index) const { return content[index]; }
 
 private:
   size_t currentSize;
@@ -168,6 +175,10 @@ struct Stringifier {
   // anything.
 
   inline ArrayPtr<const char> operator*(ArrayPtr<const char> s) const { return s; }
+  inline ArrayPtr<const char> operator*(const Array<const char>& s) const { return s; }
+  inline ArrayPtr<const char> operator*(const Array<char>& s) const { return s; }
+  template<size_t n>
+  inline ArrayPtr<const char> operator*(const CappedArray<char, n>& s) const { return s; }
   inline ArrayPtr<const char> operator*(const char* s) const { return arrayPtr(s, strlen(s)); }
 
   inline FixedArray<char, 1> operator*(char c) const {
@@ -176,6 +187,10 @@ struct Stringifier {
     return result;
   }
 
+  inline ArrayPtr<const char> operator*(Text::Reader text) const {
+    return arrayPtr(text.data(), text.size());
+  }
+
   CappedArray<char, sizeof(short) * 4> operator*(short i) const;
   CappedArray<char, sizeof(unsigned short) * 4> operator*(unsigned short i) const;
   CappedArray<char, sizeof(int) * 4> operator*(int i) const;
@@ -190,9 +205,16 @@ struct Stringifier {
 
   template <typename T>
   Array<char> operator*(ArrayPtr<T> arr) const;
+  template <typename T>
+  Array<char> operator*(const Array<T>& arr) const;
 };
 static constexpr Stringifier STR;
 
+CappedArray<char, sizeof(unsigned short) * 4> hex(unsigned short i);
+CappedArray<char, sizeof(unsigned int) * 4> hex(unsigned int i);
+CappedArray<char, sizeof(unsigned long) * 4> hex(unsigned long i);
+CappedArray<char, sizeof(unsigned long long) * 4> hex(unsigned long long i);
+
 template <typename... Params>
 Array<char> str(Params&&... params) {
   // Magic function which builds a string from a bunch of arbitrary values.  Example:
@@ -205,7 +227,7 @@ Array<char> str(Params&&... params) {
 }
 
 template <typename T>
-Array<char> strArray(ArrayPtr<T> arr, const char* delim) {
+Array<char> strArray(T&& arr, const char* delim) {
   size_t delimLen = strlen(delim);
   decltype(STR * arr[0]) pieces[arr.size()];
   size_t size = 0;
@@ -232,6 +254,22 @@ inline Array<char> Stringifier::operator*(ArrayPtr<T> arr) const {
   return strArray(arr, ", ");
 }
 
+template <typename T>
+inline Array<char> Stringifier::operator*(const Array<T>& arr) const {
+  return strArray(arr, ", ");
+}
+
+template <typename T, typename Func>
+auto mapArray(T&& arr, Func&& func) -> Array<decltype(func(arr[0]))> {
+  // TODO(cleanup):  Use ArrayBuilder.
+  Array<decltype(func(arr[0]))> result = newArray<decltype(func(arr[0]))>(arr.size());
+  size_t pos = 0;
+  for (auto& element: arr) {
+    result[pos++] = func(element);
+  }
+  return result;
+}
+
 }  // namespace capnproto
 
 #endif  // CAPNPROTO_UTIL_H_

compiler/capnproto-compiler.cabal

@@ -30,7 +30,8 @@ executable capnpc
                    directory,
                    syb,
                    transformers,
-                   entropy
+                   entropy,
+                   process
   ghc-options:     -Wall -fno-warn-missing-signatures
   other-modules:
                    Lexer,

compiler/src/Compiler.hs

@@ -800,6 +800,7 @@ compileDecl scope (ConstantDecl (Located _ name) t annotations (Located valuePos
         compiledAnnotations <- compileAnnotations scope ConstantAnnotation annotations
         return (DescConstant ConstantDesc
             { constantName = name
+            , constantId = childId name Nothing scope
             , constantParent = scope
             , constantType = typeDesc
             , constantValue = valueDesc

compiler/src/CxxGenerator.hs

@@ -449,7 +449,10 @@ hastacheConfig = MuConfig
 generateCxxHeader file = hastacheStr hastacheConfig (encodeStr headerTemplate) (fileContext file)
 generateCxxSource file = hastacheStr hastacheConfig (encodeStr srcTemplate) (fileContext file)
 
-generateCxx file = do
-    header <- generateCxxHeader file
-    source <- generateCxxSource file
-    return [(fileName file ++ ".h", header), (fileName file ++ ".c++", source)]
+generateCxx files _ = do
+    let handleFile file = do
+            header <- generateCxxHeader file
+            source <- generateCxxSource file
+            return [(fileName file ++ ".h", header), (fileName file ++ ".c++", source)]
+    results <- mapM handleFile files
+    return $ concat results

compiler/src/Main.hs

@@ -25,15 +25,17 @@ module Main ( main ) where
 
 import System.Environment
 import System.Console.GetOpt
-import System.Exit(exitFailure, exitSuccess)
-import System.IO(hPutStr, stderr)
+import System.Exit(exitFailure, exitSuccess, ExitCode(..))
+import System.IO(hPutStr, stderr, hSetBinaryMode, hClose)
 import System.FilePath(takeDirectory)
 import System.Directory(createDirectoryIfMissing, doesDirectoryExist, doesFileExist)
 import System.Entropy(getEntropy)
+import System.Process(createProcess, proc, std_in, cwd, StdStream(CreatePipe), waitForProcess)
 import Control.Monad
 import Control.Monad.IO.Class(MonadIO, liftIO)
 import Control.Exception(IOException, catch)
-import Control.Monad.Trans.State(StateT, state, modify, execStateT)
+import Control.Monad.Trans.State(StateT, state, modify, evalStateT)
+import qualified Control.Monad.Trans.State as State
 import Prelude hiding (catch)
 import Compiler
 import Util(delimit)
@@ -43,18 +45,21 @@ import Text.Printf(printf)
 import qualified Data.List as List
 import qualified Data.Map as Map
 import qualified Data.ByteString.Lazy.Char8 as LZ
-import Data.ByteString(unpack)
+import Data.ByteString(unpack, pack, hPut)
 import Data.Word(Word64, Word8)
+import Data.Maybe(fromMaybe, catMaybes)
 import Semantics
+import WireFormat(encodeSchema)
 
 import CxxGenerator(generateCxx)
 
-type GeneratorFn = FileDesc -> IO [(FilePath, LZ.ByteString)]
+type GeneratorFn = [FileDesc] -> [FileDesc] -> IO [(FilePath, LZ.ByteString)]
 
+generatorFns :: Map.Map String GeneratorFn
 generatorFns = Map.fromList [ ("c++", generateCxx) ]
 
 data Opt = SearchPathOpt FilePath
-         | OutputOpt String (Maybe GeneratorFn) FilePath
+         | OutputOpt String GeneratorFn FilePath
          | VerboseOpt
          | HelpOpt
          | GenIdOpt
@@ -78,10 +83,7 @@ main = do
          \Generate source code based on Cap'n Proto definition FILEs.\n"
          optionDescs
     args <- getArgs
-    let (options, files, optErrs) = getOpt Permute optionDescs args
-    let langErrs = map (printf "Unknown output language: %s\n")
-                       [lang | OutputOpt lang Nothing _ <- options]
-    let errs = optErrs ++ langErrs
+    let (options, files, errs) = getOpt Permute optionDescs args
     unless (null errs) (do
         mapM_ (hPutStr stderr) errs
         hPutStr stderr usage
@@ -104,7 +106,7 @@ main = do
         exitSuccess)
 
     let isVerbose = not $ null [opt | opt@VerboseOpt <- options]
-    let outputs = [(fn, dir) | OutputOpt _ (Just fn) dir <- options]
+    let outputs = [(fn, dir) | OutputOpt _ fn dir <- options]
     let searchPath = [dir | SearchPathOpt dir <- options]
 
     let verifyDirectoryExists dir = do
@@ -114,15 +116,49 @@ main = do
             exitFailure)
     mapM_ verifyDirectoryExists [dir | (_, dir) <- outputs]
 
-    CompilerState failed _ <-
-        execStateT (mapM_ (handleFile outputs isVerbose searchPath) files)
+    (failed, requestedFiles, allFiles) <-
+        evalStateT (handleFiles isVerbose searchPath files)
                    (CompilerState False Map.empty)
+
+    mapM_ (doOutput requestedFiles allFiles) outputs
+
     when failed exitFailure
 
+handleFiles isVerbose searchPath files = do
+    requestedFiles <- liftM catMaybes $ mapM (handleFile isVerbose searchPath) files
+    CompilerState failed importMap <- State.get
+    return (failed, requestedFiles, [ file | (_, ImportSucceeded file) <- Map.toList importMap ])
+
 parseOutputArg :: String -> Opt
-parseOutputArg str = case List.elemIndex ':' str of
-    Just i -> let (lang, _:dir) = splitAt i str in OutputOpt lang (Map.lookup lang generatorFns) dir
-    Nothing -> OutputOpt str (Map.lookup str generatorFns) "."
+parseOutputArg str = let
+    generatorFn lang wd = fromMaybe (callPlugin lang wd) $ Map.lookup lang generatorFns
+    in case List.elemIndex ':' str of
+        Just i -> let
+            (lang, _:dir) = splitAt i str
+            in OutputOpt lang (generatorFn lang (Just dir)) dir
+        Nothing -> OutputOpt str (generatorFn str Nothing) "."
+
+pluginName lang = if '/' `elem` lang then lang else "capnpc-" ++ lang
+
+callPlugin lang wd descs transitiveImports = do
+    let schema = encodeSchema descs transitiveImports
+    (Just hin, _, _, p) <- createProcess (proc (pluginName lang) [])
+        { std_in = CreatePipe, cwd = wd }
+    hSetBinaryMode hin True
+    hPut hin (pack schema)
+    hClose hin
+    exitCode <- waitForProcess p
+    case exitCode of
+        ExitFailure 126 -> do
+            _ <- printf "Plugin for language '%s' is not executable.\n" lang
+            exitFailure
+        ExitFailure 127 -> do
+            _ <- printf "No plugin found for language '%s'.\n" lang
+            exitFailure
+        ExitFailure i -> do
+            _ <- printf "Plugin for language '%s' failed with exit code: %d\n" lang i
+            exitFailure
+        ExitSuccess -> return []
 
 -- As always, here I am, writing my own path manipulation routines, because the ones in the
 -- standard lib don't do what I want.
@@ -227,21 +263,23 @@ parseFile isVerbose searchPath filename text = do
             liftIO $ mapM_ printError (List.sortBy compareErrors e)
             return $ Right "File contained errors."
 
-handleFile :: [(GeneratorFn, FilePath)] -> Bool -> [FilePath] -> FilePath -> CompilerMonad ()
-handleFile outputs isVerbose searchPath filename = do
+handleFile :: Bool -> [FilePath] -> FilePath -> CompilerMonad (Maybe FileDesc)
+handleFile isVerbose searchPath filename = do
     result <- importFile isVerbose searchPath filename
 
     case result of
-        Right _ -> return ()
-        Left desc -> do
-            let write dir (name, content) = do
-                    let outFilename = dir ++ "/" ++ name
-                    createDirectoryIfMissing True $ takeDirectory outFilename
-                    LZ.writeFile outFilename content
-                generate (generatorFn, dir) = do
-                    files <- generatorFn desc
-                    mapM_ (write dir) files
-            liftIO $ mapM_ generate outputs
+        Right _ -> return Nothing
+        Left desc -> return $ Just desc
+
+doOutput requestedFiles allFiles output = do
+    let write dir (name, content) = do
+            let outFilename = dir ++ "/" ++ name
+            createDirectoryIfMissing True $ takeDirectory outFilename
+            LZ.writeFile outFilename content
+        generate (generatorFn, dir) = do
+            files <- generatorFn requestedFiles allFiles
+            mapM_ (write dir) files
+    liftIO $ generate output
 
 compareErrors a b = compare (errorPos a) (errorPos b)
 

compiler/src/Semantics.hs

@@ -89,6 +89,7 @@ descId (DescFile      d) = fileId d
 descId (DescEnum      d) = enumId d
 descId (DescStruct    d) = structId d
 descId (DescInterface d) = interfaceId d
+descId (DescConstant  d) = constantId d
 descId (DescAnnotation d) = annotationId d
 descId _ = error "This construct does not have an ID."
 
@@ -363,6 +364,14 @@ typeName _ (InlineDataType s) = printf "InlineData(%d)" s
 --    symbol, and use them if so.  A particularly important case of this is imports -- typically
 --    the import will have a `using` in the file scope.
 descQualifiedName :: Desc -> Desc -> String
+
+-- Builtin descs can be aliased with "using", so we need to support them.
+descQualifiedName _ (DescBuiltinType t) = builtinTypeName t
+descQualifiedName _ DescBuiltinList = "List"
+descQualifiedName _ DescBuiltinInline = "Inline"
+descQualifiedName _ DescBuiltinInlineList = "InlineList"
+descQualifiedName _ DescBuiltinInlineData = "InlineData"
+
 descQualifiedName (DescFile scope) (DescFile desc) =
     if fileName scope == fileName desc
         then ""
@@ -394,6 +403,7 @@ usingRuntimeImports _ = []
 
 data ConstantDesc = ConstantDesc
     { constantName :: String
+    , constantId :: Word64
     , constantParent :: Desc
     , constantType :: TypeDesc
     , constantAnnotations :: AnnotationMap
@@ -544,20 +554,22 @@ descToCode indent self@(DescEnum desc) = printf "%senum %s @0x%016x%s {\n%s%s}\n
 descToCode indent self@(DescEnumerant desc) = printf "%s%s @%d%s;\n" indent
     (enumerantName desc) (enumerantNumber desc)
     (annotationsCode self)
-descToCode indent self@(DescStruct desc) = printf "%sstruct %s @0x%016x%s%s {\n%s%s}\n" indent
-    (structName desc)
-    (structId desc)
-    (if structIsFixedWidth desc
-        then printf " fixed(%s, %d pointers) "
-            (dataSectionSizeString $ structDataSize desc)
-            (structPointerCount desc)
-        else "")
-    (annotationsCode self)
-    (blockCode indent (structMembers desc))
-    indent
-descToCode indent self@(DescField desc) = printf "%s%s@%d%s: %s%s%s;  # %s\n" indent
+descToCode indent self@(DescStruct desc) =
+    printf "%sstruct %s @0x%016x%s%s {  # %d bytes, %d pointers\n%s%s}\n" indent
+        (structName desc)
+        (structId desc)
+        (if structIsFixedWidth desc
+            then printf " fixed(%s, %d pointers) "
+                (dataSectionSizeString $ structDataSize desc)
+                (structPointerCount desc)
+            else "")
+        (annotationsCode self)
+        (div (dataSectionBits $ structDataSize desc) 8)
+        (structPointerCount desc)
+        (blockCode indent (structMembers desc))
+        indent
+descToCode indent self@(DescField desc) = printf "%s%s@%d: %s%s%s;  # %s%s\n" indent
     (fieldName desc) (fieldNumber desc)
-    (case fieldUnion desc of { Nothing -> ""; Just (u, _) -> " in " ++ unionName u})
     (typeName (descParent self) (fieldType desc))
     (case fieldDefaultValue desc of { Nothing -> ""; Just v -> " = " ++ valueString v; })
     (annotationsCode self)
@@ -572,6 +584,8 @@ descToCode indent self@(DescField desc) = printf "%s%s@%d%s: %s%s%s;  # %s\n" in
         DataOffset dataSize offset -> let
             bits = dataSizeInBits dataSize
             in printf "bits[%d, %d)" (offset * bits) ((offset + 1) * bits))
+    (case fieldUnion desc of { Nothing -> ""; Just (_, i) -> printf ", union tag = %d" i})
+
 descToCode indent self@(DescUnion desc) = printf "%sunion %s@%d%s {  # [%d, %d)\n%s%s}\n" indent
     (unionName desc) (unionNumber desc)
     (annotationsCode self)
@@ -596,11 +610,11 @@ descToCode _ self@(DescParam desc) = printf "%s: %s%s%s"
         Just v -> printf " = %s" $ valueString v
         Nothing -> "")
     (annotationsCode self)
-descToCode indent self@(DescAnnotation desc) = printf "%sannotation %s @0x%016x: %s on(%s)%s;\n" indent
+descToCode indent self@(DescAnnotation desc) = printf "%sannotation %s @0x%016x(%s): %s%s;\n" indent
     (annotationName desc)
     (annotationId desc)
-    (typeName (descParent self) (annotationType desc))
     (delimit ", " $ map show $ Set.toList $ annotationTargets desc)
+    (typeName (descParent self) (annotationType desc))
     (annotationsCode self)
 descToCode _ (DescBuiltinType _) = error "Can't print code for builtin type."
 descToCode _ DescBuiltinList = error "Can't print code for builtin type."

doc/language.md

@@ -392,7 +392,7 @@ annotation qux @0xf8a1bedf44c89f00 (field) :Text;
 
 If you omit the ID for a type or annotation, one will be assigned automatically.  This default
 ID is derived by taking the first 8 bytes of the MD5 hash of the parent scope's ID concatenated
-with the declaration's name (where the parent scope means the file for top-level delarations, or
+with the declaration's name (where the "parent scope" is the file for top-level delarations, or
 the outer type for nested declarations).  You can see the automatically-generated IDs by running
 `capnpc -v` on a file.  In general, you would only specify an explicit ID for a declaration if that
 declaration has been renamed or moved and you want the ID to stay the same for