This discussion is archived
2 Replies Latest reply: Oct 12, 2011 4:11 AM by jfranzoy RSS

Question: when sendmsg on a non-blocking sctp socket returns EWOULDBLOCK?

jfranzoy Newbie
Currently Being Moderated
First of all, thanks for reading my question. I sincerely apologizes if this question is out of topic.

We made a simple experiment, with disconcerting outcome:
- We establishe a one-to-one style SCTP association, with output and input buffer set at 20MB (it is just a test).
- Then we send a burst of 400KB: 100 messages of 4096 bytes each.

Our surprise was that some of these send reported that the socket was not writable.

What can be happening?
What are we misunderstanding?

Thank in advance.

Edited by: jfranzoy on 09/10/2011 01:19
  • 1. Re: Question: when sendmsg on a non-blocking sctp socket returns EWOULDBLOCK?
    jfranzoy Newbie
    Currently Being Moderated
    I am sorry to say but, the previouts description of the problem has a "bug". A more appropriate one would be:

    1 - An association is established with send buffers set in 20MB -setsockopt(SO_SNDBUF)
    2 - for( int i = 0; i < 100; ++i ) {
    3 - ....test is socket is writable with select() using timeout 0 (zero)
    4 - ........if socket is writtable, then send a 4096 message
    5 - }
    6 - wait for a while ( 30 seconds )
    7 - exit

    What happends is that sometimes the test in step ( 3 ) timeouts. We translate that as: the socket is NOT writable. Actually there is no EWOULDBLOCK.

    Any light in my darkness will be appreciated ...
    Thanks.

    Edited by: jfranzoy on 09/10/2011 00:20
  • 2. Re: Question: when sendmsg on a non-blocking sctp socket returns EWOULDBLOCK?
    jfranzoy Newbie
    Currently Being Moderated
    Well, ejem...

    A new experiment we've done, may help.

    If we do not test the socket with select() to know if it is writable previous to call sendmsg(), the behaviour is as expected: EWOULDBLOCK is returned only after sending sufficient messages to fill the send buffer.

    The experiment consists in:
    - create an sctp socket to listen for associations, let's call it acceptorSocket
    - create another sctp socket and connect to acceptorSocket, let's call it socket1
    - accept the new association, let's call the new one-to-one socket socket2
    - set socket1 SO_SNDBUF in 1000 * 1024
    - set socket2 SO_RCVBUF in 3000 * 1024
    - for( int i = 0; i < 4000; ++i ) {
    -      test if socket1 is writable with select(). If it's not increment selectToutCounter
    -      sndmsg(socket1...) if EWOULDBLOCK is returned increment wouldBlockCounter
    - }

    The result in my solaris box is :
    Sending 4000 1K messages
         send buf len = 1024000
         recv buf len = 3072000
    wouldBlockCounter=5
    selectToutCounter=997

    The question is, why select() says to me that socket1 is NOT writable and sndmsg doesn't fail? Remember that nobody is reading on the other side.

    Thanks


    DETAILS : this is the experimentation program
    -------------


    // platform dependent includes
    #include <unistd.h>
    #include <alloca.h>
    #include <sys/time.h>
    #include <sys/socket.h>
    #include <netinet/in.h>
    #include <netinet/sctp.h>
    #include <arpa/inet.h>
    #include <errno.h>
    #include <fcntl.h>
    #include <sys/types.h>

    // c++ includes
    #include <algorithm>
    #include <cstdlib>
    #include <cstring>
    #include <exception>
    #include <iostream>
    #include <iterator>



    namespace ats { namespace base {
    namespace test_sctp {

    int do_main( const int argc, const char **argv ) {
    int socketAcceptor = ::socket(AF_INET6, SOCK_STREAM, IPPROTO_SCTP);
    if ( socketAcceptor < 0 ) {
    std::cerr << "Error socket(acceptor): " << strerror(errno) << std::endl;
    return 1;
    }

    {
    int flags = ::fcntl( socketAcceptor, F_GETFL, 0 );
    if ( flags == -1 ) {
    std::cerr << "Error fcntl(F_GETFL): " << strerror(errno) << std::endl;
    return 1;
    }
    flags |= O_NONBLOCK;
    if ( ::fcntl( socketAcceptor, F_SETFL, flags ) == -1 ) {
    std::cerr << "Error fcntl(F_SETFL, O_NONBLOCK): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    const int len = 3000 * 1024;
    if ( ::setsockopt( socketAcceptor, SOL_SOCKET, SO_RCVBUF, &len, sizeof(len) ) != 0 ) {
    std::cerr << "Error setsockopt(SOL_SOCKET, SO_RCVBUF): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    const int flags = 0;
    if ( ::setsockopt( socketAcceptor, IPPROTO_SCTP, SCTP_NODELAY, &flags, sizeof(flags) ) != 0 ) {
    std::cerr << "Error setsockopt(IPPROTO_SCTP, SCTP_NODELAY): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    struct sockaddr_in6 addr0;
    std::memset( &addr0, 0, sizeof(addr0) );
    addr0.sin6_family = AF_INET6;
    addr0.sin6_port = htons( 10329 );
    if ( inet_pton( AF_INET6, "::1", &addr0.sin6_addr ) != 1 ) {
    std::cerr << "Error inet_pton(::1): " << strerror(errno) << std::endl;
    return 1;
    }
    errno = 0;
    if ( ::bind( socketAcceptor, (struct sockaddr*)&addr0, sizeof(addr0) ) != 0 ) {
    std::cerr << "Error bind(127.0.0.1): " << strerror(errno) << std::endl;
    return 1;
    }
    }
    if ( ::listen( socketAcceptor, 16 ) != 0 ) {
    std::cerr << "Error listen() : " << strerror(errno) << std::endl;
    return 1;
    }

    int socket1 = ::socket(AF_INET6, SOCK_STREAM, IPPROTO_SCTP);
    if ( socket1 < 0 ) {
    std::cerr << "Error socket(socket1): " << strerror(errno) << std::endl;
    return 1;
    }

    {
    int flags = ::fcntl( socket1, F_GETFL, 0 );
    if ( flags == -1 ) {
    std::cerr << "Error fcntl(socket1,F_GETFL): " << strerror(errno) << std::endl;
    return 1;
    }
    flags |= O_NONBLOCK;
    if ( ::fcntl( socket1, F_SETFL, flags ) == -1 ) {
    std::cerr << "Error fcntl(socket1, F_SETFL, O_NONBLOCK): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    const int len = 1000 * 1024;
    if ( ::setsockopt( socket1, SOL_SOCKET, SO_SNDBUF, &len, sizeof(len) ) != 0 ) {
    std::cerr << "Error setsockopt(socket1, SOL_SOCKET, SO_SNDBUF): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    const int flags = 0;
    if ( ::setsockopt( socket1, IPPROTO_SCTP, SCTP_NODELAY, &flags, sizeof(flags) ) != 0 ) {
    std::cerr << "Error setsockopt(socket1, IPPROTO_SCTP, SCTP_NODELAY): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    struct sockaddr_in6 addr0;
    std::memset( &addr0, 0, sizeof(addr0) );
    addr0.sin6_family = AF_INET6;
    addr0.sin6_port = htons( 10329 );
    if ( inet_pton( AF_INET6, "::1", &addr0.sin6_addr ) != 1 ) {
    std::cerr << "Error inet_pton(::1): " << strerror(errno) << std::endl;
    return 1;
    }
    if ( ::connect( socket1, (struct sockaddr*)&addr0, sizeof( addr0 ) ) == 0 || errno == EINPROGRESS ) {
    fd_set writefds;
    FD_ZERO(&writefds);
    FD_SET( socket1, &writefds );
    int rc = ::select( socket1+1, 0, &writefds, 0, 0 );
    if ( rc == -1 ) {
    std::cerr << "Error select(waitReadyForWrite socket1): " << strerror(errno) << std::endl;
    return 1;
    }
    }
    int pendingError = 0;
    socklen_t pendingErrorLen = sizeof(pendingError);
    if ( ::getsockopt( socket1, SOL_SOCKET, SO_ERROR, &pendingError, &pendingErrorLen ) ) {
    std::cerr << "Error getsockopt(socket1, SO_ERROR): " << strerror(errno) << std::endl;
    return 1;
    }
    if ( pendingError != 0 ) {
    std::cerr << "Error pendingError (socket1) != 0" << std::endl;
    return 1;
    }
    }

    int socket2 = ::accept( socketAcceptor, 0, 0 );
    if ( socket2 == -1 ) {
    std::cerr << "Error accept(): " << strerror(errno) << std::endl;
    return 1;
    }

    {
    int flags = ::fcntl( socket2, F_GETFL, 0 );
    if ( flags == -1 ) {
    std::cerr << "Error fcntl(socket2,F_GETFL): " << strerror(errno) << std::endl;
    return 1;
    }
    flags |= O_NONBLOCK;
    if ( ::fcntl( socket2, F_SETFL, flags ) == -1 ) {
    std::cerr << "Error fcntl(socket2, F_SETFL, O_NONBLOCK): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    const int len = 3000 * 1024;
    if ( ::setsockopt( socket2, SOL_SOCKET, SO_RCVBUF, &len, sizeof(len) ) != 0 ) {
    std::cerr << "Error setsockopt(socket2, SOL_SOCKET, SO_RCVBUF): " << strerror(errno) << std::endl;
    return 1;
    }
    }

    {
    const int flags = 0;
    if ( ::setsockopt( socket2, IPPROTO_SCTP, SCTP_NODELAY, &flags, sizeof(flags) ) != 0 ) {
    std::cerr << "Error setsockopt(socket2, IPPROTO_SCTP, SCTP_NODELAY): " << strerror(errno) << std::endl;
    return 1;
    }
    }


    int end = 100;
    if ( argc > 1 ) {
    end = std::atoi( argv[1] );
    }
    bool selectTouts = (bool)alloca( sizeof(bool) * end );
    std::fill( selectTouts, selectTouts + end, false );
    bool wouldBlocks = (bool )alloca( sizeof(bool) * end );
    std::fill( wouldBlocks, wouldBlocks + end, false );
    std::cerr << "Sending " << end << " 1K messages" << std::endl;
    int socket2ReceiveBufferLen = 0;
    {
    socklen_t bufferLenLen = sizeof( socket2ReceiveBufferLen );
    if ( ::getsockopt( socket2, SOL_SOCKET, SO_RCVBUF, &socket2ReceiveBufferLen,
    &bufferLenLen ) != 0 )
    {
    std::cerr << "Error getsockopt(SOL_SOCKET, SO_RCVBUF): " << strerror(errno) << std::endl;
    return 1;
    }
    }
    int socket1SendBufferLen = 0;
    {
    socklen_t bufferLenLen = sizeof(socket1SendBufferLen);
    if ( ::getsockopt( socket1, SOL_SOCKET, SO_SNDBUF, &socket1SendBufferLen, &bufferLenLen ) != 0 )
    {
    std::cerr << "Error getsockopt(socket1, SOL_SOCKET, SO_SNDBUF): " << strerror(errno) << std::endl;
    return 1;
    }
    }
    std::cerr << " send buf len = " << socket1SendBufferLen << std::endl;
    std::cerr << " recv buf len = " << socket2ReceiveBufferLen << std::endl;
    int wouldBlockCounter = 0;
    int selectToutCounter = 0;
    int nfds = socketAcceptor;
    if ( nfds < socket1 ) {
    nfds = socket1;
    }
    if ( nfds < socket2 ) {
    nfds = socket2;
    }
    ++ nfds;
    for( int i = 0; i < end; ++i ) {
    {
    fd_set writefds;
    FD_ZERO( &writefds );
    FD_SET( socket1, &writefds );
    struct timeval tv = { 0, 0 };
    int rc = ::select( socket1+1, 0, &writefds, 0, &tv );
    if ( rc == -1 ) {
    std::cerr << "Error select(): " << strerror(errno) << std::endl;
    return 1;
    }
    if ( rc == 0 || ( ! FD_ISSET(socket1, &writefds) ) ) {
    ++ selectToutCounter;
    selectTouts[i] = true;
    }
    }
    {
    char buffer[ 1024 ];
    struct msghdr message = { 0 };
    unsigned char info[ CMSG_SPACE( sizeof(struct ::sctp_sndrcvinfo) ) ];
    ::cmsghdr* ancillaryData = reinterpret_cast< ::cmsghdr* >( info );
    ancillaryData->cmsg_len = CMSG_LEN( sizeof(struct ::sctp_sndrcvinfo) );
    ancillaryData->cmsg_level = IPPROTO_SCTP;
    ancillaryData->cmsg_type = SCTP_SNDRCV;
    ::sctp_sndrcvinfo* sndrcvinfo = reinterpret_cast< ::sctp_sndrcvinfo* >(
    CMSG_DATA( reinterpret_cast<struct cmsghdr*>(info) ) );
    sndrcvinfo->sinfo_flags = MSG_UNORDERED;
    iovec buffersVec[1];
    buffersVec[0].iov_base = buffer;
    buffersVec[0].iov_len = 1024;
    message.msg_iov = buffersVec;
    message.msg_iovlen = 1;
    message.msg_flags = MSG_XPG4_2; // for ancylliary data
    message.msg_control = info;
    message.msg_controllen = sizeof( info );
    if ( ::sendmsg( socket1, &message, 0 ) < 0 ) {
    if ( errno == EWOULDBLOCK ) {
    ++ wouldBlockCounter;
    wouldBlocks[i] = true;
    } else {
    std::cerr << "Error sendmsg(): " << strerror(errno) << std::endl;
    return 1;
    }
    }
    }
    }
    std::cerr << "wouldBlockCounter=" << wouldBlockCounter << std::endl;
    std::cerr << "selectToutCounter=" << selectToutCounter << std::endl;
    std::cerr << "wouldBlocks=";
    std::copy( wouldBlocks, wouldBlocks + end, std::ostream_iterator<bool>(std::cerr,"") );
    std::cerr << std::endl;
    std::cerr << "selectTouts=";
    std::copy( selectTouts, selectTouts + end, std::ostream_iterator<bool>(std::cerr,"") );
    std::cerr << std::endl;
    return 0;
    }

    }}} // ats::base::test_sctp namespace


    int main( const int argc, const char **argv )
    try {
    return ats::base::test_sctp::do_main( argc, argv );
    } catch ( std::exception& e ) {
    std::cerr << e.what() << std::endl;
    return 1;
    } catch ( ... ) {
    std::cerr << "Unknown exception thrown" << std::endl;
    return 1;
    }

    Edited by: jfranzoy on Oct 12, 2011 3:51 AM
    Added the example

Legend

  • Correct Answers - 10 points
  • Helpful Answers - 5 points