Fixed wrong handling of shared messages

The shared reference count was not shared but copied. msg_t cannot
store the refcnt itsef but has to store a pointer to an externally
allocated (shared) refcnter. The changes to lmsg are reverted to
use content_t again. Howver, this introduces an allocation in v2_decoder
when creating the message which can be avoided. When allocating the reception
buffer, space is allocated for the maximum number of reference counts
(8192 / max_vsm_size = 8192/64 = 128 zmq:atomic_counter objects). This
increases the buffer by 128*sizeof(atomic_counter) = 128*4 = 512 bytes only.
When creating a message, the refcnt member is set to the address of one of the
pre-allocated atomic_counter_t objects. To do so, a new msg_t type zcmsg
is introduced because msg::copy must discriminate between the message types
when releasing memory.

Makefile.am

@@ -643,7 +643,7 @@ check_PROGRAMS = ${test_apps}
 
 #  Run the test cases
 TESTS = $(test_apps)
-XFAIL_TESTS = tests/test_msg_ffn
+XFAIL_TESTS = 
 
 if !ON_LINUX
 XFAIL_TESTS += tests/test_abstract_ipc

src/msg.hpp

@@ -54,6 +54,7 @@ namespace zmq
     class msg_t
     {
     public:
+
         //  Message flags.
         enum
         {
@@ -65,12 +66,17 @@ namespace zmq
         };
 
         bool check ();
-        int init (void *data_, size_t size_, msg_free_fn *ffn_,
-                  void *hint_);
-        int init ();
+        int init();
+
+        int init (void* data, size_t size_,
+                  msg_free_fn* ffn_, void* hint,
+                  zmq::atomic_counter_t* refcnt_ = NULL);
+
         int init_size (size_t size_);
         int init_data (void *data_, size_t size_, msg_free_fn *ffn_,
-            void *hint_);
+                       void *hint_);
+        int init_external_storage(void *data_, size_t size_, zmq::atomic_counter_t* ctr,
+                                  msg_free_fn *ffn_, void *hint_);
         int init_delimiter ();
         int close ();
         int move (msg_t &src_);
@@ -88,9 +94,9 @@ namespace zmq
         bool is_identity () const;
         bool is_credential () const;
         bool is_delimiter () const;
-        bool is_vsm ();
-        bool is_lmsg () const;
-        bool is_cmsg ();
+        bool is_vsm () const;
+        bool is_cmsg () const;
+        bool is_zcmsg() const;
         uint32_t get_routing_id();
         int set_routing_id(uint32_t routing_id_);
 
@@ -102,13 +108,30 @@ namespace zmq
         //  references drops to 0, the message is closed and false is returned.
         bool rm_refs (int refs_);
 
-    private:
-
         //  Size in bytes of the largest message that is still copied around
         //  rather than being reference-counted.
         enum { msg_t_size = 64 };
         enum { max_vsm_size = msg_t_size - (8 + sizeof (metadata_t *) + 3 + sizeof(uint32_t)) };
 
+    private:
+        zmq::atomic_counter_t* refcnt();
+
+        //  Shared message buffer. Message data are either allocated in one
+        //  continuous block along with this structure - thus avoiding one
+        //  malloc/free pair or they are stored in used-supplied memory.
+        //  In the latter case, ffn member stores pointer to the function to be
+        //  used to deallocate the data. If the buffer is actually shared (there
+        //  are at least 2 references to it) refcount member contains number of
+        //  references.
+        struct content_t
+        {
+            void *data;
+            size_t size;
+            msg_free_fn *ffn;
+            void *hint;
+            zmq::atomic_counter_t refcnt;
+        };
+
         //  Different message types.
         enum type_t
         {
@@ -121,10 +144,12 @@ namespace zmq
             type_delimiter = 103,
             //  CMSG messages point to constant data
             type_cmsg = 104,
-            type_max = 104
-        };
 
-        atomic_counter_t& msg_counter();
+            // zero-copy LMSG message for v2_decoder
+            type_zclmsg = 105,
+
+            type_max = 105
+        };
 
         // the file descriptor where this message originated, needs to be 64bit due to alignment
         int64_t file_desc;
@@ -149,36 +174,36 @@ namespace zmq
                 unsigned char flags;
                 uint32_t routing_id;
             } vsm;
-            struct lmsg_t {
+            struct {
+                metadata_t *metadata;
+                content_t *content;
+                unsigned char unused [msg_t_size - (8 + sizeof (metadata_t *)
+                                                      + sizeof (content_t*)
+                                                      + 2
+                                                      + sizeof(uint32_t))];
+                unsigned char type;
+                unsigned char flags;
+                uint32_t routing_id;
+            } lmsg;
+            struct {
                 metadata_t *metadata;
-                //  Shared message buffer. Message data are either allocated in one
-                //  continuous block along with this structure - thus avoiding one
-                //  malloc/free pair or they are stored in used-supplied memory.
-                //  In the latter case, ffn member stores pointer to the function to be
-                //  used to deallocate the data. If the buffer is actually shared (there
-                //  are at least 2 references to it) refcount member contains number of
-                //  references.
                 void *data;
                 size_t size;
                 msg_free_fn *ffn;
                 void *hint;
-                // create an aligned block for an atomic_counter_t object
-                union aligned_atomic_counter_storage {
-                    zmq::atomic_counter_t::integer_t maxAlign;
-                    unsigned char counter[ sizeof(zmq::atomic_counter_t) ];
-                } refcnt;
+                zmq::atomic_counter_t* refcnt;
                 unsigned char unused [msg_t_size - (8 + sizeof (metadata_t *)
-                                                    + sizeof(void*)
-                                                    + sizeof(size_t)
-                                                    + sizeof(msg_free_fn*)
-                                                    + sizeof(void*)
-                                                    + sizeof(aligned_atomic_counter_storage)
-                                                    + 2
-                                                    + sizeof(uint32_t))];
+                                                      + sizeof (void*)
+                                                      + sizeof (size_t)
+                                                      + sizeof (msg_free_fn*)
+                                                      + sizeof (void*)
+                                                      + sizeof (zmq::atomic_counter_t*)
+                                                      + 2
+                                                      + sizeof(uint32_t))];
                 unsigned char type;
                 unsigned char flags;
                 uint32_t routing_id;
-            } lmsg;
+            } zclmsg;
             struct {
                 metadata_t *metadata;
                 void* data;

src/v2_decoder.cpp

@@ -29,6 +29,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <cmath>
 
 #include "platform.hpp"
 #ifdef ZMQ_HAVE_WINDOWS
@@ -44,7 +45,8 @@
 zmq::shared_message_memory_allocator::shared_message_memory_allocator(size_t bufsize_):
     buf(NULL),
     bufsize( 0 ),
-    maxsize( bufsize_ )
+    maxsize( bufsize_ ),
+    msg_refcnt( NULL )
 {
 
 }
@@ -67,9 +69,14 @@ unsigned char* zmq::shared_message_memory_allocator::allocate()
     // @todo aligmnet padding may be needed
     if (!buf)
     {
-        buf = (unsigned char *) malloc(bufsize + sizeof(zmq::atomic_counter_t));
+        // allocate memory for reference counters together with reception buffer
+        size_t const maxCounters = std::ceil( (double)max_size / (double)msg_t::max_vsm_size);
+        size_t const allocationsize = maxsize + sizeof(zmq::atomic_counter_t) + maxCounters * sizeof(zmq::atomic_counter_t);
+
+        buf = (unsigned char *) malloc(allocationsize);
         alloc_assert (buf);
         new(buf) atomic_counter_t(1);
+        msg_refcnt = reinterpret_cast<zmq::atomic_counter_t*>( buf + sizeof(atomic_counter_t) + maxsize );
     }
 
     bufsize = maxsize;
@@ -81,6 +88,7 @@ void zmq::shared_message_memory_allocator::deallocate()
     free(buf);
     buf = NULL;
     bufsize = 0;
+    msg_refcnt = NULL;
 }
 
 unsigned char* zmq::shared_message_memory_allocator::release()
@@ -88,16 +96,11 @@ unsigned char* zmq::shared_message_memory_allocator::release()
     unsigned char* b = buf;
     buf = NULL;
     bufsize = 0;
+    msg_refcnt = NULL;
 
     return b;
 }
 
-void zmq::shared_message_memory_allocator::reset(unsigned char* b)
-{
-    deallocate();
-    buf = b;
-}
-
 void zmq::shared_message_memory_allocator::inc_ref()
 {
     ((zmq::atomic_counter_t*)buf)->add(1);
@@ -204,6 +207,7 @@ int zmq::v2_decoder_t::size_ready(uint64_t msg_size, unsigned char const* read_p
 
     // the current message can exceed the current buffer. We have to copy the buffer
     // data into a new message and complete it in the next receive.
+
     if (unlikely ((unsigned char*)read_pos + msg_size > (data() + size())))
     {
         // a new message has started, but the size would exceed the pre-allocated arena
@@ -214,12 +218,13 @@ int zmq::v2_decoder_t::size_ready(uint64_t msg_size, unsigned char const* read_p
     {
         // construct message using n bytes from the buffer as storage
         // increase buffer ref count
-        rc = in_progress.init( (unsigned char*)read_pos,
-                               msg_size, shared_message_memory_allocator::call_dec_ref,
-                               buffer() );
+        // if the message will be a large message, pass a valid refcnt memory location as well
+        rc = in_progress.init( (unsigned char*)read_pos, msg_size,
+                               shared_message_memory_allocator::call_dec_ref, buffer(),
+                               create_refcnt() );
 
         // For small messages, data has been copied and refcount does not have to be increased
-        if (in_progress.is_lmsg())
+        if (in_progress.is_zcmsg())
         {
             inc_ref();
         }

src/v2_decoder.hpp

@@ -63,8 +63,6 @@ namespace zmq
         // the messages constructed on top of it.
         unsigned char* release();
 
-        void reset(unsigned char* b);
-
         void inc_ref();
 
         static void call_dec_ref(void*, void* buffer);
@@ -85,6 +83,12 @@ namespace zmq
             bufsize = new_size;
         }
 
+        //
+        zmq::atomic_counter_t* create_refcnt()
+        {
+            return msg_refcnt++;
+        }
+
     private:
         unsigned char* buf;
         size_t bufsize;