added OpenPGM sender - ZMQ_PUB

Makefile.am

@@ -17,5 +17,7 @@ endif
 SUBDIRS = src $(DIR_P) $(DIR_R) $(DIR_J) $(DIR_PERF)
 DIST_SUBDIRS = src python ruby java perf
 
+EXTRA_DIST =  $(top_srcdir)/foreign/openpgm/@pgm_basename@.tar.bz2
+
 dist-hook:
-		-rm -rf $(distdir)/third-party/openpgm/$pgm_basename
+		-rm -rf $(distdir)/foreign/openpgm/@pgm_basename@

c/zmq.h

@@ -52,6 +52,8 @@ extern "C" {
 #define ZMQ_IDENTITY 6
 #define ZMQ_SUBSCRIBE 7
 #define ZMQ_UNSUBSCRIBE 8
+#define ZMQ_RATE 9
+#define ZMQ_RECOVERY_IVL 10
 
 //  The operation should be performed in non-blocking mode. I.e. if it cannot
 //  be processed immediately, error should be returned with errno set to EAGAIN.

src/Makefile.am

@@ -62,6 +62,8 @@ libzmq_la_SOURCES = $(pgm_sources) \
     object.hpp \
     options.hpp \
     owned.hpp \
+    pgm_sender.hpp \
+    pgm_socket.hpp \
     pipe.hpp \
     platform.hpp \
     poll.hpp \
@@ -101,6 +103,8 @@ libzmq_la_SOURCES = $(pgm_sources) \
     object.cpp \
     options.cpp \
     owned.cpp \
+    pgm_sender.cpp \
+    pgm_socket.cpp \
     pipe.cpp \
     poll.cpp \
     select.cpp \
@@ -122,7 +126,7 @@ libzmq_la_SOURCES = $(pgm_sources) \
     zmq_listener.cpp \
     zmq_listener_init.cpp
 
-libzmq_la_LDFLAGS = -version-info @LTVER@
+libzmq_la_LDFLAGS = -version-info @LTVER@ @LIBZMQ_EXTRA_LDFLAFS@
 
 if BUILD_PGM
 libzmq_la_CXXFLAGS = -I$(top_srcdir)/foreign/openpgm/@pgm_basename@/openpgm/pgm/include/ -Wall @LIBZMQ_EXTRA_CXXFLAGS@

src/app_thread.cpp

@@ -145,7 +145,7 @@ zmq::socket_base_t *zmq::app_thread_t::create_socket (int type_)
     case ZMQ_PUB:
     case ZMQ_REQ:
     case ZMQ_REP:
-        s = new socket_base_t (this);
+        s = new socket_base_t (this, type_);
         break;
     default:
         //  TODO: This should be EINVAL.

src/config.hpp

@@ -70,8 +70,10 @@ namespace zmq
 
         //  Maximal number of non-accepted connections that can be held by
         //  TCP listener object.
-        tcp_connection_backlog = 10
+        tcp_connection_backlog = 10,
 
+        //  Maximum transport data unit size for PGM (TPDU).
+        pgm_max_tpdu = 1500
     };
 
 }

src/options.cpp

@@ -24,6 +24,8 @@ zmq::options_t::options_t () :
     lwm (0),
     swap (0),
     mask (0),
-    affinity (0)
+    affinity (0),
+    rate (0),
+    recovery_ivl (0)
 {
 }

src/options.hpp

@@ -37,6 +37,12 @@ namespace zmq
         uint64_t mask;
         uint64_t affinity;
         std::string identity;
+
+        //  Maximum tranfer rate [kb/s].
+        uint32_t rate;
+
+        //  Reliability time interval [s].
+        uint32_t recovery_ivl;
     };
 
 }

src/pgm_sender.cpp

+/*
+    Copyright (c) 2007-2009 FastMQ Inc.
+
+    This file is part of 0MQ.
+
+    0MQ is free software; you can redistribute it and/or modify it under
+    the terms of the Lesser GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    0MQ is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    Lesser GNU General Public License for more details.
+
+    You should have received a copy of the Lesser GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "platform.hpp"
+
+#if defined ZMQ_HAVE_OPENPGM
+
+#include <iostream>
+
+#include "io_thread.hpp"
+#include "pgm_sender.hpp"
+#include "err.hpp"
+#include "wire.hpp"
+
+//#define PGM_SENDER_DEBUG
+//#define PGM_SENDER_DEBUG_LEVEL 1
+
+// level 1 = key behaviour
+// level 2 = processing flow
+// level 4 = infos
+
+#ifndef PGM_SENDER_DEBUG
+#   define zmq_log(n, ...)  while (0)
+#else
+#   define zmq_log(n, ...)    do { if ((n) <= PGM_SENDER_DEBUG_LEVEL) \
+        { printf (__VA_ARGS__);}} while (0)
+#endif
+
+#ifdef ZMQ_HAVE_LINUX
+zmq::pgm_sender_t::pgm_sender_t (io_thread_t *parent_, 
+      const options_t &options_, const char *session_name_) :
+    io_object_t (parent_),
+    pgm_socket (false, options_),
+    options (options_),
+    session_name (session_name_),
+    inout (NULL),
+    out_buffer (NULL),
+    out_buffer_size (0),
+    write_size (0),
+    write_pos (0), 
+    first_message_offset (-1)
+{
+
+}
+
+int zmq::pgm_sender_t::init (const char *network_)
+{
+    return pgm_socket.init (network_);
+}
+
+void zmq::pgm_sender_t::plug (i_inout *inout_)
+{
+    
+    //  Alocate 2 fds for PGM socket.
+    int downlink_socket_fd;
+    int uplink_socket_fd;
+
+    encoder.set_inout (inout_);
+
+    //  Fill fds from PGM transport.
+    pgm_socket.get_sender_fds (&downlink_socket_fd, &uplink_socket_fd);
+
+    //  Add downlink_socket_fd into poller.
+    handle = add_fd (downlink_socket_fd);
+
+    //  Add uplink_socket_fd into the poller.
+    uplink_handle = add_fd (uplink_socket_fd);
+
+    //  Set POLLIN. We wont never want to stop polling for uplink = we never
+    //  want to stop porocess NAKs.
+    set_pollin (uplink_handle);
+
+    //  Set POLLOUT for downlink_socket_handle.
+    set_pollout (handle);
+
+    inout = inout_;
+
+    zmq_log (1, "plug: downlink_socket_fd %i, uplink_socket_fd %i, %s(%i)",
+        downlink_socket_fd, uplink_socket_fd, __FILE__, __LINE__);
+
+    std::cout << std::flush;
+}
+
+void zmq::pgm_sender_t::unplug ()
+{
+    rm_fd (handle);
+    rm_fd (uplink_handle);
+    encoder.set_inout (NULL);
+    inout = NULL;
+}
+
+void zmq::pgm_sender_t::revive ()
+{
+    set_pollout (handle);
+}
+
+zmq::pgm_sender_t::~pgm_sender_t ()
+{
+    if (out_buffer) {
+        pgm_socket.free_buffer (out_buffer);
+    }
+}
+
+//  In event on sender side means NAK or SPMR receiving from some peer.
+void zmq::pgm_sender_t::in_event ()
+{
+    pgm_socket.process_upstream ();
+}
+
+void zmq::pgm_sender_t::out_event ()
+{
+
+    //  POLLOUT event from send socket. If write buffer is empty, 
+    //  try to read new data from the encoder.
+    if (write_pos == write_size) {
+
+        //  Get buffer if we do not have already one.
+        if (!out_buffer) {
+            out_buffer = (unsigned char*) 
+                pgm_socket.get_buffer (&out_buffer_size);
+        }
+
+        assert (out_buffer_size > 0);
+
+        //  First two bytes /sizeof (uint16_t)/ are used to store message 
+        //  offset in following steps.
+        write_size = encoder.read (out_buffer + sizeof (uint16_t), 
+            out_buffer_size - sizeof (uint16_t), &first_message_offset);
+        write_pos = 0;
+
+        //  If there are no data to write stop polling for output.
+        if (!write_size) {
+            reset_pollout (handle);
+        } else {
+            // Addning uint16_t for offset in a case when encoder returned > 0B.
+            write_size += sizeof (uint16_t);
+        }
+    }
+
+    //  If there are any data to write, write them into the socket.
+    //  Note that all data has to written in one write_one_pkt_with_offset call.
+    if (write_pos < write_size) {
+        size_t nbytes = write_one_pkt_with_offset (out_buffer + write_pos, 
+            write_size - write_pos, (uint16_t) first_message_offset);
+
+        //  We can write all data or 0 which means rate limit reached.
+        if (write_size - write_pos != nbytes && nbytes != 0) {
+            zmq_log (1, "write_size - write_pos %i, nbytes %i, %s(%i)",
+                (int)(write_size - write_pos), (int)nbytes, __FILE__, __LINE__);
+            assert (false);
+        }
+
+        //  PGM rate limit reached nbytes is 0.
+        if (!nbytes) {
+            zmq_log (1, "pgm rate limit reached, %s(%i)\n", __FILE__, __LINE__);
+        }
+
+        //  After sending data slice is owned by tx window.
+        if (nbytes) {
+            out_buffer = NULL;
+        }
+
+        write_pos += nbytes;
+    }
+
+}
+
+/*
+void zmq::bp_pgm_sender_t::revive (pipe_t *pipe_)
+{
+    //  We have some messages in encoder.
+    if (!shutting_down) {
+                
+        //  Forward the revive command to the pipe. 
+        engine_base_t <false, true>::revive (pipe_);
+
+        //  There is at least one engine (that one which sent revive) that 
+        //  has messages ready. Try to write data to the socket, thus 
+        //  eliminating one polling for POLLOUT event.
+        //  Note that if write_size is zero it means that buffer is empty and
+        //  we can read data from encoder.
+        if (!write_size) {
+            poller->set_pollout (handle);
+            out_event (handle);
+        }
+    }
+}
+
+*/
+size_t zmq::pgm_sender_t::write_one_pkt_with_offset (unsigned char *data_, 
+    size_t size_, uint16_t offset_)
+{
+    zmq_log (1, "data_size %i, first message offset %i, %s(%i)",
+        (int) size_, offset_, __FILE__, __LINE__);
+
+    std::cout << std::flush;
+
+    //  Put offset information in the buffer.
+    put_uint16 (data_, offset_);
+   
+    //  Send data.
+    size_t nbytes = pgm_socket.send (data_, size_);
+
+    return nbytes;
+}
+#endif
+
+#endif

src/pgm_sender.hpp

+/*
+    Copyright (c) 2007-2009 FastMQ Inc.
+
+    This file is part of 0MQ.
+
+    0MQ is free software; you can redistribute it and/or modify it under
+    the terms of the Lesser GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    0MQ is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    Lesser GNU General Public License for more details.
+
+    You should have received a copy of the Lesser GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __ZMQ_BP_PGM_SENDER_HPP_INCLUDED__
+#define __ZMQ_BP_PGM_SENDER_HPP_INCLUDED__
+
+#include "platform.hpp"
+
+#if defined ZMQ_HAVE_OPENPGM
+
+#include <vector>
+
+#include "stdint.hpp"
+#include "io_object.hpp"
+#include "i_engine.hpp"
+#include "options.hpp"
+#include "pgm_socket.hpp"
+#include "zmq_encoder.hpp"
+
+namespace zmq
+{
+
+    class pgm_sender_t : public io_object_t, public i_engine
+    {
+
+    public:
+        pgm_sender_t (class io_thread_t *parent_, const options_t &options_, 
+            const char *session_name_);
+        ~pgm_sender_t ();
+
+        int init (const char *network_);
+
+        //  i_engine interface implementation.
+        void plug (struct i_inout *inout_);
+        void unplug ();
+        void revive ();
+
+        //  i_poll_events interface implementation.
+        void in_event ();
+        void out_event ();
+
+    private:
+
+        //  Send one APDU with first message offset information. 
+        //  Note that first 2 bytes in data_ are used to store the offset_
+        //  and thus user data has to start at data_ + sizeof (uint16_t).
+        size_t write_one_pkt_with_offset (unsigned char *data_, size_t size_,
+            uint16_t offset_);
+
+        //  Message encoder.
+        zmq_encoder_t encoder;
+
+        //  PGM socket.
+        pgm_socket_t pgm_socket;
+
+        //  Socket options.
+        options_t options;
+
+        //  Name of the session associated with the connecter.
+        std::string session_name;
+
+        //  Poll handle associated with PGM socket.
+        handle_t handle;
+        handle_t uplink_handle;
+
+        //  ?
+        i_inout *inout;
+
+        //  Output buffer from pgm_socket.
+#ifdef ZMQ_HAVE_WINDOWS
+        unsigned char out_buffer [pgm_win_max_apdu];
+#else
+        unsigned char *out_buffer;
+        
+        //  Output buffer size.
+        size_t out_buffer_size;
+#endif
+
+        size_t write_size;
+        size_t write_pos;
+
+        //  Offset of the first mesage in data chunk taken from encoder.
+        int first_message_offset;
+
+        pgm_sender_t (const pgm_sender_t&);
+        void operator = (const pgm_sender_t&);
+    };
+
+}
+
+#endif
+
+#endif

src/pgm_socket.hpp

+/*
+    Copyright (c) 2007-2009 FastMQ Inc.
+
+    This file is part of 0MQ.
+
+    0MQ is free software; you can redistribute it and/or modify it under
+    the terms of the Lesser GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    0MQ is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    Lesser GNU General Public License for more details.
+
+    You should have received a copy of the Lesser GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __PGM_SOCKET_HPP_INCLUDED__
+#define __PGM_SOCKET_HPP_INCLUDED__
+
+#include "platform.hpp"
+
+#if defined ZMQ_HAVE_OPENPGM
+
+#ifdef ZMQ_HAVE_LINUX
+#include <glib.h>
+#include <pgm/pgm.h>
+#else
+#include <Winsock2.h>
+#endif
+
+#include "stdint.hpp"
+#include "options.hpp"
+
+namespace zmq
+{
+    //  Encapsulates PGM socket.
+    class pgm_socket_t
+    {
+
+#ifdef ZMQ_HAVE_LINUX
+
+    public:
+        //  If receiver_ is true PGM transport is not generating SPM packets.
+        //  interface format: iface;mcast_group:port for raw PGM socket
+        //                    udp:iface;mcast_goup:port for UDP encapsulacion
+        pgm_socket_t (bool receiver_, const options_t &options_);
+
+        //  Closes the transport.
+        ~pgm_socket_t ();
+
+        //  Initialize PGM network structures (GSI, GSRs).
+        int init (const char *network_);
+
+        //  Open PGM transport. Parameters are the same as in constructor.
+        int open_transport (void);
+
+        //  Close transport.
+        void close_transport (void);
+        
+        //   Get receiver fds and store them into user allocated memory.
+        int get_receiver_fds (int *recv_fd_, int *waiting_pipe_fd_);
+
+        //   Get sender and receiver fds and store it to user allocated 
+        //   memory. Receive fd is used to process NAKs from peers.
+        int get_sender_fds (int *send_fd_, int *receive_fd_);
+
+        //  Send data as one APDU, transmit window owned memory.
+        size_t send (unsigned char *data_, size_t data_len_);
+
+        //  Allocates one slice for packet in tx window.
+        void *get_buffer (size_t *size_);
+
+        //  Fees memory allocated by get_buffer.
+        void free_buffer (void *data_);
+
+        //  Receive data from pgm socket.
+        ssize_t receive (void **data_);
+
+        //  POLLIN on sender side should mean NAK or SPMR receiving. 
+        //  process_upstream function is used to handle such a situation.
+        void process_upstream (void);
+
+    protected:
+    
+        //  OpenPGM transport
+        pgm_transport_t* g_transport;
+
+    private:
+        
+        //  Associated socket options.
+        options_t options;
+
+        //  Returns max tsdu size without fragmentation.
+        size_t get_max_tsdu_size (void);
+
+        //  Returns maximum count of apdus which fills readbuf_size_
+        size_t get_max_apdu_at_once (size_t readbuf_size_);
+
+        //  Return true if TSI has empty GSI ('\0') and sport 0.
+        bool tsi_empty (const pgm_tsi_t *tsi_);
+
+        //  Compare TSIs, return true if equal.
+        bool tsi_equal (const pgm_tsi_t *tsi_a_, const pgm_tsi_t *tsi_b_);
+        
+        //  true when pgm_socket should create receiving side.
+        bool receiver;
+
+        //  TIBCO Rendezvous format network info.
+        char network [256];
+
+        //  PGM transport port number.
+        uint16_t port_number;
+
+        //  If we are using UDP encapsulation.
+        bool udp_encapsulation;
+
+        //  Array of pgm_msgv_t structures to store received data 
+        //  from the socket (pgm_transport_recvmsgv).
+        pgm_msgv_t *pgm_msgv;
+
+        // How many bytes were read from pgm socket.
+        ssize_t nbytes_rec;
+
+        //  How many bytes were processed from last pgm socket read.
+        ssize_t nbytes_processed;
+        
+        //  How many messages from pgm_msgv were already sent up.
+        ssize_t pgm_msgv_processed;
+
+        //  Size of pgm_msgv array.
+        ssize_t pgm_msgv_len;
+
+        //  Sender transport uses 2 fd.
+        enum {pgm_sender_fd_count = 2};
+    
+        //  Receiver transport uses 2 fd.
+        enum {pgm_receiver_fd_count = 2};
+
+        //  TSI of the actual peer.
+        pgm_tsi_t tsi;
+
+        //  Previous peer TSI.
+        pgm_tsi_t retired_tsi;
+
+#endif
+    };
+}
+#endif
+
+#endif

src/socket_base.cpp

@@ -17,6 +17,8 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <iostream>
+
 #include <string>
 #include <algorithm>
 
@@ -35,9 +37,12 @@
 #include "uuid.hpp"
 #include "pipe.hpp"
 #include "err.hpp"
+#include "platform.hpp"
+#include "pgm_sender.hpp"
 
-zmq::socket_base_t::socket_base_t (app_thread_t *parent_) :
+zmq::socket_base_t::socket_base_t (app_thread_t *parent_, int type_) :
     object_t (parent_),
+    type (type_),
     current (0),
     active (0),
     pending_term_acks (0),
@@ -145,6 +150,22 @@ int zmq::socket_base_t::setsockopt (int option_, const void *optval_,
         errno = EFAULT;
         return -1;
 
+    case ZMQ_RATE:
+        if (optvallen_ != sizeof (uint32_t)) {
+            errno = EINVAL;
+            return -1;
+        }
+        options.rate = *((int32_t*) optval_);
+        return 0;
+        
+    case ZMQ_RECOVERY_IVL:
+        if (optvallen_ != sizeof (uint32_t)) {
+            errno = EINVAL;
+            return -1;
+        }
+        options.recovery_ivl = *((int32_t*) optval_);
+        return 0;
+
     default:
         errno = EINVAL;
         return -1;
@@ -170,6 +191,21 @@ int zmq::socket_base_t::connect (const char *addr_)
     std::string session_name ("#");
     session_name += uuid_t ().to_string ();
 
+    //  Parse addr_ string.
+    std::string addr_type;
+    std::string addr_args;
+
+    std::string addr (addr_);
+    std::string::size_type pos = addr.find ("://");
+
+    if (pos == std::string::npos) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    addr_type = addr.substr (0, pos);
+    addr_args = addr.substr (pos + 3);
+
     //  Create the session.
     io_thread_t *io_thread = choose_io_thread (options.affinity);
     session_t *session = new session_t (io_thread, this, session_name.c_str (),
@@ -198,20 +234,63 @@ int zmq::socket_base_t::connect (const char *addr_)
     send_plug (session);
     send_own (this, session);
 
-    //  Create the connecter object. Supply it with the session name so that
-    //  it can bind the new connection to the session once it is established.
-    zmq_connecter_t *connecter = new zmq_connecter_t (
-        choose_io_thread (options.affinity), this, options,
-        session_name.c_str ());
-    int rc = connecter->set_address (addr_);
-    if (rc != 0) {
-        delete connecter;
-        return -1;
+    if (addr_type == "tcp") {
+
+        //  Create the connecter object. Supply it with the session name so that
+        //  it can bind the new connection to the session once it is established.
+        zmq_connecter_t *connecter = new zmq_connecter_t (
+            choose_io_thread (options.affinity), this, options,
+            session_name.c_str ());
+        int rc = connecter->set_address (addr_args.c_str ());
+        if (rc != 0) {
+            delete connecter;
+            return -1;
+        }
+        send_plug (connecter);
+        send_own (this, connecter);
+
+        return 0;
     }
-    send_plug (connecter);
-    send_own (this, connecter);
 
-    return 0;
+#if defined ZMQ_HAVE_OPENPGM
+    if (addr_type == "pgm") {
+        
+        switch (type) {
+        case ZMQ_PUB:
+        {
+            pgm_sender_t *pgm_sender = 
+                new pgm_sender_t (choose_io_thread (options.affinity), options, 
+                session_name.c_str ());
+
+            int rc = pgm_sender->init (addr_args.c_str ());
+            if (rc != 0) {
+                delete pgm_sender;
+                return -1;
+            }
+    
+            //  Reserve a sequence number for following 'attach' command.
+            session->inc_seqnum ();
+            send_attach (session, pgm_sender);
+
+            pgm_sender = NULL;
+
+            break;
+        }
+        case ZMQ_SUB:
+            zmq_assert (false);
+            break;
+        default:
+            errno = EINVAL;
+            return -1;
+        }
+
+        return 0;
+    }
+#endif
+
+    //  Unknown address type.
+    errno = ENOTSUP;
+    return -1;
 }
 
 int zmq::socket_base_t::send (::zmq_msg_t *msg_, int flags_)

src/socket_base.hpp

@@ -38,7 +38,7 @@ namespace zmq
     {
     public:
 
-        socket_base_t (class app_thread_t *parent_);
+        socket_base_t (class app_thread_t *parent_, int type_);
         virtual ~socket_base_t ();
 
         //  Interface for communication with the API layer.
@@ -87,6 +87,9 @@ namespace zmq
         //  fair queueing.
         bool fetch (struct zmq_msg_t *msg_);
 
+        //  Type of the socket.
+        int type;
+
         //  List of all I/O objects owned by this socket. The socket is
         //  responsible for deallocating them before it quits.
         typedef std::set <class owned_t*> io_objects_t;

src/sub.cpp

@@ -23,7 +23,7 @@
 #include "err.hpp"
 
 zmq::sub_t::sub_t (class app_thread_t *parent_) :
-    socket_base_t (parent_),
+    socket_base_t (parent_, ZMQ_SUB),
     all_count (0)
 {
 }