From: Ralph Lange <ralph.lange@gmx.de>
Date: Fri, 30 Oct 2015 16:59:47 +0100
Subject: [PATCH] rsrv: merge Michael's patch for binding to a specific
 interface

rsrv: when binding to a specific interface also bind to iface bcast
    An oddness of BSD sockets (not winsock) is that binding to
    INADDR_ANY will receive unicast and broadcast, but binding to
    a specific interface address receives only unicast.  The trick
    is to bind a second socket to the interface broadcast address,
    which will then receive only broadcasts.
rsrv: unnecessary use of global
    remove global prsrv_cast_client
    rename prsrv_cast_client in cast_server()
rsrv: placeholder for cast_server thread shutdown
rsrv: more renaming
rsrv: casr show multiple UDP clients
---
 src/ioc/rsrv/camessage.c     |  32 +++---------
 src/ioc/rsrv/caserverio.c    |   2 +-
 src/ioc/rsrv/caservertask.c  |  33 ++++++++++--
 src/ioc/rsrv/cast_server.c   | 122 +++++++++++++++++++++++++------------------
 src/ioc/rsrv/online_notify.c |  58 +++++++++++++++++---
 src/ioc/rsrv/server.h        |  11 ++--
 6 files changed, 165 insertions(+), 93 deletions(-)

diff --git src/ioc/rsrv/camessage.c src/ioc/rsrv/camessage.c
index 469c29b..51ab8f3 100644
--- src/ioc/rsrv/camessage.c
+++ src/ioc/rsrv/camessage.c
@@ -1126,7 +1126,7 @@ static void casAccessRightsCB(ASCLIENTPVT ascpvt, asClientStatus type)
     pclient = pciu->client;
     assert(pclient);
 
-    if(pclient == prsrv_cast_client){
+    if(pclient->proto==IPPROTO_UDP){
         return;
     }
 
@@ -1193,7 +1193,7 @@ static void access_rights_reply ( struct channel_in_use * pciu )
     int             v41;
     int             status;
 
-    assert ( pciu->client != prsrv_cast_client );
+    assert ( pciu->client->proto!=IPPROTO_UDP );
 
     /*
      * noop if this is an old client
@@ -1321,7 +1321,7 @@ static int claim_ciu_action ( caHdrLargeArray *mp,
         }
     }
     else {
-        epicsMutexMustLock(prsrv_cast_client->chanListLock);
+        epicsMutexMustLock(client->chanListLock);
         /*
          * clients which dont claim their
          * channel in use block prior to
@@ -1331,7 +1331,7 @@ static int claim_ciu_action ( caHdrLargeArray *mp,
         if(!pciu){
             errlogPrintf("CAS: client timeout disconnect id=%d\n",
                 mp->m_cid);
-            epicsMutexUnlock(prsrv_cast_client->chanListLock);
+            epicsMutexUnlock(client->chanListLock);
             SEND_LOCK(client);
             send_err(
                 mp,
@@ -1343,33 +1343,15 @@ static int claim_ciu_action ( caHdrLargeArray *mp,
         }
 
         /*
-         * duplicate claim message are unacceptable
-         * (so we disconnect the client)
-         */
-        if (pciu->client!=prsrv_cast_client) {
-            errlogPrintf("CAS: duplicate claim disconnect id=%d\n",
-                mp->m_cid);
-            epicsMutexUnlock(prsrv_cast_client->chanListLock);
-            SEND_LOCK(client);
-            send_err(
-                mp,
-                ECA_INTERNAL,
-                client,
-                "duplicate claim in old connect protocol");
-            SEND_UNLOCK(client);
-            return RSRV_ERROR;
-        }
-
-        /*
          * remove channel in use block from
          * the UDP client where it could time
          * out and place it on the client
          * who is claiming it
          */
         ellDelete(
-            &prsrv_cast_client->chanList,
+            &client->chanList,
             &pciu->node);
-        epicsMutexUnlock(prsrv_cast_client->chanListLock);
+        epicsMutexUnlock(client->chanListLock);
 
         epicsMutexMustLock(client->chanListLock);
         pciu->state = rsrvCS_pendConnectResp;
@@ -2623,7 +2605,7 @@ int camessage ( struct client *client )
         if ( CASDEBUG > 2 )
             log_header (NULL, client, &msg, pBody, nmsg);
 
-        if ( client == prsrv_cast_client ) {
+        if ( client->proto==IPPROTO_UDP ) {
             if ( msg.m_cmmd < NELEMENTS ( udpJumpTable ) ) {
                 status = ( *udpJumpTable[msg.m_cmmd] )( &msg, pBody, client );
                 if (status!=RSRV_OK) {
diff --git src/ioc/rsrv/caserverio.c src/ioc/rsrv/caserverio.c
index c5865d9..15a2637 100644
--- src/ioc/rsrv/caserverio.c
+++ src/ioc/rsrv/caserverio.c
@@ -222,7 +222,7 @@ void cas_send_dg_msg ( struct client * pclient )
     /*
      * add placeholder for the first version message should it be needed
      */
-    rsrv_version_reply ( prsrv_cast_client );
+    rsrv_version_reply ( pclient );
 
     SEND_UNLOCK(pclient);
 
diff --git src/ioc/rsrv/caservertask.c src/ioc/rsrv/caservertask.c
index 132fe46..f2dfd91 100644
--- src/ioc/rsrv/caservertask.c
+++ src/ioc/rsrv/caservertask.c
@@ -266,6 +266,7 @@ int rsrv_init (void)
     clientQlock = epicsMutexMustCreate();
 
     ellInit ( &clientQ );
+    ellInit ( &clientQudp );
     freeListInitPvt ( &rsrvClientFreeList, sizeof(struct client), 8 );
     freeListInitPvt ( &rsrvChanFreeList, sizeof(struct channel_in_use), 512 );
     freeListInitPvt ( &rsrvEventFreeList, sizeof(struct event_ext), 512 );
@@ -300,7 +301,6 @@ int rsrv_init (void)
     freeListInitPvt ( &rsrvLargeBufFreeListTCP, rsrvSizeofLargeBufTCP, 1 );
     ellInit ( &casIntfAddrList );
     ellInit ( &beaconAddrList );
-    prsrv_cast_client = NULL;
     pCaBucket = NULL;
 
     castcp_startStopEvent = epicsEventMustCreate(epicsEventEmpty);
@@ -499,12 +499,35 @@ void casr (unsigned level)
         log_one_client(client, level);
         client = (struct client *) ellNext(&client->node);
     }
-    UNLOCK_CLIENTQ
 
-    if (level>=2 && prsrv_cast_client) {
-        printf( "UDP Server:\n" );
-        log_one_client(prsrv_cast_client, level);
+    if (level>=2) {
+        client = (struct client *) ellNext ( &clientQudp.node );
+        while (client) {
+            struct sockaddr_in addr;
+            osiSocklen_t alen = sizeof(addr);
+            char    buf[40];
+
+            if (!getsockname(client->udpRecv, (struct sockaddr*)&addr, &alen)) {
+                ipAddrToDottedIP (&addr, buf, sizeof(buf));
+            } else {
+                strcpy(buf, "<unknown>");
+            }
+
+            printf( "UDP Name Server: recvfrom %s", buf );
+
+            alen = sizeof(addr);
+            if (!getsockname(client->sock, (struct sockaddr*)&addr, &alen)) {
+                ipAddrToDottedIP (&addr, buf, sizeof(buf));
+            } else {
+                strcpy(buf, "<unknown>");
+            }
+
+            printf( " sendto %s\n", buf );
+
+            client = (struct client *) ellNext(&client->node);
+        }
     }
+    UNLOCK_CLIENTQ
 
     if (level>=2u) {
         bytes_reserved = 0u;
diff --git src/ioc/rsrv/cast_server.c src/ioc/rsrv/cast_server.c
index 2932f2b..5467fa7 100644
--- src/ioc/rsrv/cast_server.c
+++ src/ioc/rsrv/cast_server.c
@@ -54,7 +54,7 @@
 /*
  * clean_addrq
  */
-static void clean_addrq(void)
+static void clean_addrq(struct client *client)
 {
     struct channel_in_use * pciu;
     struct channel_in_use * pnextciu;
@@ -67,9 +67,9 @@ static void clean_addrq(void)
 
     epicsTimeGetCurrent ( &current );
 
-    epicsMutexMustLock ( prsrv_cast_client->chanListLock );
+    epicsMutexMustLock ( client->chanListLock );
     pnextciu = (struct channel_in_use *) 
-            prsrv_cast_client->chanList.node.next;
+            client->chanList.node.next;
 
     while( (pciu = pnextciu) ) {
         pnextciu = (struct channel_in_use *)pciu->node.next;
@@ -77,7 +77,7 @@ static void clean_addrq(void)
         delay = epicsTimeDiffInSeconds(&current,&pciu->time_at_creation);
         if (delay > timeout) {
 
-            ellDelete(&prsrv_cast_client->chanList, &pciu->node);
+            ellDelete(&client->chanList, &pciu->node);
             LOCK_CLIENTQ;
             s = bucketRemoveItemUnsignedId (
                 pCaBucket,
@@ -96,7 +96,7 @@ static void clean_addrq(void)
             if(delay>maxdelay) maxdelay = delay;
         }
     }
-    epicsMutexUnlock ( prsrv_cast_client->chanListLock );
+    epicsMutexUnlock ( client->chanListLock );
 
 #   ifdef DEBUG
     if(ndelete){
@@ -115,30 +115,35 @@ static void clean_addrq(void)
  */
 void cast_server(void *pParm)
 {
-    osiSockAddrNode     *paddrNode;
+    cast_config *conf = pParm;
+    osiSockAddr     *paddrNode = &conf->pAddr;
     struct sockaddr_in  sin;
     int                 status;
     int                 count=0;
+    int                 mysocket=0;
     struct sockaddr_in  new_recv_addr;
     osiSocklen_t        recv_addr_size;
     osiSockIoctl_t      nchars;
+    SOCKET              recv_sock;
+    struct client      *client;
 
     recv_addr_size = sizeof(new_recv_addr);
 
-    if( IOC_cast_sock!=0 && IOC_cast_sock!=INVALID_SOCKET ) {
-        epicsSocketDestroy ( IOC_cast_sock );
-    }
-
     /* 
      *  Open the socket.
      *  Use ARPA Internet address format and datagram socket.
      */
 
-    if ( ( IOC_cast_sock = epicsSocketCreate (AF_INET, SOCK_DGRAM, 0) ) == INVALID_SOCKET ) {
+    if ( ( recv_sock = epicsSocketCreate (AF_INET, SOCK_DGRAM, 0) ) == INVALID_SOCKET ) {
         epicsPrintf ("CAS: cast socket creation error\n");
         epicsThreadSuspendSelf ();
     }
 
+    if(conf->reply_sock==INVALID_SOCKET) {
+        conf->reply_sock = recv_sock; /* assume that the socket capable of unicast send is created first */
+        mysocket = 1;
+    }
+
     /*
      * some concern that vxWorks will run out of mBuf's
      * if this change is made
@@ -164,19 +169,17 @@ void cast_server(void *pParm)
     }
 #endif
 
-    epicsSocketEnableAddressUseForDatagramFanout ( IOC_cast_sock );
-
-    paddrNode = (osiSockAddrNode *) ellFirst ( &casIntfAddrList );
+    epicsSocketEnableAddressUseForDatagramFanout ( recv_sock );
 
-    memcpy(&sin, &paddrNode->addr.ia, sizeof (sin));
+    memcpy(&sin, &paddrNode->ia, sizeof (sin));
 
     /* get server's Internet address */
-    if( bind(IOC_cast_sock, (struct sockaddr *)&sin, sizeof (sin)) < 0){
+    if( bind(recv_sock, (struct sockaddr *)&sin, sizeof (sin)) < 0){
         char sockErrBuf[64];
         epicsSocketConvertErrnoToString ( 
             sockErrBuf, sizeof ( sockErrBuf ) );
         epicsPrintf ("CAS: UDP server port bind error was \"%s\"\n", sockErrBuf );
-        epicsSocketDestroy ( IOC_cast_sock );
+        epicsSocketDestroy ( recv_sock );
         epicsThreadSuspendSelf ();
     }
 
@@ -186,27 +189,37 @@ void cast_server(void *pParm)
      *
      */
     while ( TRUE ) {
-        prsrv_cast_client = create_client ( IOC_cast_sock, IPPROTO_UDP );
-        if ( prsrv_cast_client ) {
+        client = create_client ( conf->reply_sock, IPPROTO_UDP );
+        if ( client ) {
             break;
         }
         epicsThreadSleep(300.0);
     }
+    client->udpRecv = recv_sock;
 
-    casAttachThreadToClient ( prsrv_cast_client );
+    assert(client->node.next==NULL && client->node.previous==NULL);
+    LOCK_CLIENTQ;
+    ellAdd ( &clientQudp, &client->node );
+    UNLOCK_CLIENTQ;
+
+    casAttachThreadToClient ( client );
 
     /*
      * add placeholder for the first version message should it be needed
      */
-    rsrv_version_reply ( prsrv_cast_client );
+    rsrv_version_reply ( client );
+
+    /* these pointers become invalid after signaling casudp_startStopEvent */
+    conf = NULL;
+    paddrNode = NULL;
 
     epicsEventSignal(casudp_startStopEvent);
 
     while (TRUE) {
         status = recvfrom (
-            IOC_cast_sock,
-            prsrv_cast_client->recv.buf,
-            prsrv_cast_client->recv.maxstk,
+            recv_sock,
+            client->recv.buf,
+            client->recv.maxstk,
             0,
             (struct sockaddr *)&new_recv_addr, 
             &recv_addr_size);
@@ -221,69 +234,69 @@ void cast_server(void *pParm)
             }
         }
         else if (casudp_ctl == ctlRun) {
-            prsrv_cast_client->recv.cnt = (unsigned) status;
-            prsrv_cast_client->recv.stk = 0ul;
-            epicsTimeGetCurrent(&prsrv_cast_client->time_at_last_recv);
+            client->recv.cnt = (unsigned) status;
+            client->recv.stk = 0ul;
+            epicsTimeGetCurrent(&client->time_at_last_recv);
 
-            prsrv_cast_client->minor_version_number = 0;
-            prsrv_cast_client->seqNoOfReq = 0;
+            client->minor_version_number = 0;
+            client->seqNoOfReq = 0;
 
             /*
              * If we are talking to a new client flush to the old one 
              * in case we are holding UDP messages waiting to 
              * see if the next message is for this same client.
              */
-            if (prsrv_cast_client->send.stk>sizeof(caHdr)) {
-                status = memcmp(&prsrv_cast_client->addr, 
+            if (client->send.stk>sizeof(caHdr)) {
+                status = memcmp(&client->addr,
                     &new_recv_addr, recv_addr_size);
                 if(status){     
                     /* 
                      * if the address is different 
                      */
-                    cas_send_dg_msg(prsrv_cast_client);
-                    prsrv_cast_client->addr = new_recv_addr;
+                    cas_send_dg_msg(client);
+                    client->addr = new_recv_addr;
                 }
             }
             else {
-                prsrv_cast_client->addr = new_recv_addr;
+                client->addr = new_recv_addr;
             }
 
             if (CASDEBUG>1) {
                 char    buf[40];
     
-                ipAddrToDottedIP (&prsrv_cast_client->addr, buf, sizeof(buf));
+                ipAddrToDottedIP (&client->addr, buf, sizeof(buf));
                 errlogPrintf ("CAS: cast server msg of %d bytes from addr %s\n", 
-                    prsrv_cast_client->recv.cnt, buf);
+                    client->recv.cnt, buf);
             }
 
             if (CASDEBUG>2)
-                count = ellCount (&prsrv_cast_client->chanList);
+                count = ellCount (&client->chanList);
 
-            status = camessage ( prsrv_cast_client );
+            status = camessage ( client );
             if(status == RSRV_OK){
-                if(prsrv_cast_client->recv.cnt != 
-                    prsrv_cast_client->recv.stk){
+                if(client->recv.cnt !=
+                    client->recv.stk){
                     char buf[40];
         
-                    ipAddrToDottedIP (&prsrv_cast_client->addr, buf, sizeof(buf));
+                    ipAddrToDottedIP (&client->addr, buf, sizeof(buf));
 
                     epicsPrintf ("CAS: partial (damaged?) UDP msg of %d bytes from %s ?\n",
-                        prsrv_cast_client->recv.cnt-prsrv_cast_client->recv.stk, buf);
+                        client->recv.cnt-client->recv.stk, buf);
                 }
             }
             else {
                 char buf[40];
     
-                ipAddrToDottedIP (&prsrv_cast_client->addr, buf, sizeof(buf));
+                ipAddrToDottedIP (&client->addr, buf, sizeof(buf));
 
                 epicsPrintf ("CAS: invalid (damaged?) UDP request from %s ?\n", buf);
             }
 
             if (CASDEBUG>2) {
-                if ( ellCount (&prsrv_cast_client->chanList) ) {
+                if ( ellCount (&client->chanList) ) {
                     errlogPrintf ("CAS: Fnd %d name matches (%d tot)\n",
-                        ellCount(&prsrv_cast_client->chanList)-count,
-                        ellCount(&prsrv_cast_client->chanList));
+                        ellCount(&client->chanList)-count,
+                        ellCount(&client->chanList));
                 }
             }
         }
@@ -292,15 +305,22 @@ void cast_server(void *pParm)
          * allow messages to batch up if more are comming
          */
         nchars = 0; /* supress purify warning */
-        status = socket_ioctl(IOC_cast_sock, FIONREAD, &nchars);
+        status = socket_ioctl(recv_sock, FIONREAD, &nchars);
         if (status<0) {
             errlogPrintf ("CA cast server: Unable to fetch N characters pending\n");
-            cas_send_dg_msg (prsrv_cast_client);
-            clean_addrq ();
+            cas_send_dg_msg (client);
+            clean_addrq (client);
         }
         else if (nchars == 0) {
-            cas_send_dg_msg (prsrv_cast_client);
-            clean_addrq ();
+            cas_send_dg_msg (client);
+            clean_addrq (client);
         }
     }
+
+    /* ATM never reached, just a placeholder */
+
+    if(!mysocket)
+        client->sock = INVALID_SOCKET; /* only one cast_server should destroy the reply socket */
+    destroy_client(client);
+    epicsSocketDestroy(recv_sock);
 }
diff --git src/ioc/rsrv/online_notify.c src/ioc/rsrv/online_notify.c
index 6ccc5e8..ea60d9b 100644
--- src/ioc/rsrv/online_notify.c
+++ src/ioc/rsrv/online_notify.c
@@ -74,7 +74,6 @@ void rsrv_online_notify_task(void *pParm)
     char                        buf[16];
     unsigned                    priorityOfUDP;
     epicsThreadBooleanStatus    tbs;
-    epicsThreadId               tid;
     
     taskwdInsert (epicsThreadGetIdSelf(),NULL,NULL);
     
@@ -223,14 +222,59 @@ void rsrv_online_notify_task(void *pParm)
     casudp_startStopEvent = epicsEventMustCreate(epicsEventEmpty);
     casudp_ctl = ctlPause;
 
-    tid = epicsThreadCreate ( "CAS-UDP", priorityOfUDP,
-        epicsThreadGetStackSize (epicsThreadStackMedium),
-        cast_server, 0 );
-    if ( tid == 0 ) {
-        epicsPrintf ( "CAS: unable to start UDP daemon thread\n" );
+    {
+        /* casudp_startStopEvent ensures that this struct
+         * lives until the cast_server thread(s) are done with it.
+         */
+        cast_config config;
+
+        config.pAddr = ((osiSockAddrNode *) ellFirst ( &casIntfAddrList ))->addr;
+        config.reply_sock = INVALID_SOCKET;
+
+        epicsThreadMustCreate ( "CAS-UDP", priorityOfUDP,
+                                epicsThreadGetStackSize (epicsThreadStackMedium),
+                                cast_server, &config );
+
+        epicsEventMustWait(casudp_startStopEvent);
+
+#if !defined(_WIN32)
+        /* An oddness of BSD sockets (not winsock) is that binding to
+         * INADDR_ANY will receive unicast and broadcast, but binding to
+         * a specific interface address receives only unicast.  The trick
+         * is to bind a second socket to the interface broadcast address,
+         * which will then receive only broadcasts.
+         */
+        if (config.pAddr.ia.sin_addr.s_addr != htonl(INADDR_ANY)) {
+            ELLLIST bcastList = ELLLIST_INIT;
+
+            osiSockDiscoverBroadcastAddresses (&bcastList,
+                                               sock, &config.pAddr); // match addr
+
+            if(ellCount(&bcastList)==0) {
+                errlogPrintf("CAS UDP: failed to find interface broadcast address\n");
+
+            } else {
+                osiSockAddrNode *pNode = (osiSockAddrNode*)ellFirst(&bcastList);
+                assert(config.pAddr.sa.sa_family==pNode->addr.sa.sa_family);
+
+                if (config.pAddr.ia.sin_addr.s_addr != pNode->addr.ia.sin_addr.s_addr) {
+
+                    /* copy the address, keep the port */
+                    config.pAddr.ia.sin_addr.s_addr = pNode->addr.ia.sin_addr.s_addr;
+
+                    epicsThreadMustCreate ( "CAS-UDPB", priorityOfUDP,
+                                            epicsThreadGetStackSize (epicsThreadStackMedium),
+                                            cast_server, &config );
+
+                    epicsEventMustWait(casudp_startStopEvent);
+                }
+            }
+
+            ellFree(&bcastList);
+        }
+#endif
     }
 
-    epicsEventMustWait(casudp_startStopEvent);
     epicsEventSignal(beacon_startStopEvent);
 
     while (TRUE) {
diff --git src/ioc/rsrv/server.h src/ioc/rsrv/server.h
index c457285..c328195 100644
--- src/ioc/rsrv/server.h
+++ src/ioc/rsrv/server.h
@@ -88,7 +88,7 @@ typedef struct client {
   char                  *pUserName;
   char                  *pHostName;
   epicsEventId          blockSem; /* used whenever the client blocks */
-  SOCKET                sock;
+  SOCKET                sock, udpRecv;
   int                   proto;
   epicsThreadId         tid;
   unsigned              minor_version_number;
@@ -137,6 +137,10 @@ struct event_ext {
     char                    modified;   /* mod & ev flw ctrl enbl */
 };
 
+typedef struct {
+    osiSockAddr pAddr;
+    SOCKET reply_sock;
+} cast_config;
 
 enum ctl {ctlInit, ctlRun, ctlPause, ctlExit};
 
@@ -161,13 +165,12 @@ enum ctl {ctlInit, ctlRun, ctlPause, ctlExit};
 
 GLBLTYPE int                CASDEBUG;
 GLBLTYPE SOCKET             IOC_sock;
-GLBLTYPE SOCKET             IOC_cast_sock;
 GLBLTYPE unsigned short     ca_server_port;
-GLBLTYPE ELLLIST            clientQ; /* locked by clientQlock */
+GLBLTYPE ELLLIST            clientQ; /* (TCP clients) locked by clientQlock */
+GLBLTYPE ELLLIST            clientQudp; /* locked by clientQlock */
 GLBLTYPE ELLLIST            beaconAddrList;
 GLBLTYPE ELLLIST            casIntfAddrList;
 GLBLTYPE epicsMutexId       clientQlock;
-GLBLTYPE struct client      *prsrv_cast_client;
 GLBLTYPE BUCKET             *pCaBucket;
 GLBLTYPE void               *rsrvClientFreeList;
 GLBLTYPE void               *rsrvChanFreeList;
-- 
1.9.4.msysgit.2

