[mpich-discuss] Problems with LD_PRELOAD on TCP functions
Alex Margolin
alex.margolin at mail.huji.ac.il
Tue Feb 7 12:19:02 CST 2012
Hi,
While working on the LD_PRELOAD adapter I've encountered a basic problem, not related to my implementation (I hope...).
I've removed all the logic and left only the wrapper functions and it worked. I wanted to make sure I'm covering all the bases, so I've XORed all the data I send() and recv() - and then a got this failure message:
[singularity:26013] mca_oob_tcp_peer_recv_connect_ack: invalid header type: -1263225674
[singularity:26013] [[54490,1],1] routed:binomial: Connection to lifeline [[54490,0],0] lost
[singularity:26012] mca_oob_tcp_peer_recv_connect_ack: invalid header type: -1263225674
[singularity:26012] [[54490,1],0] routed:binomial: Connection to lifeline [[54490,0],0] lost
[singularity:26014] mca_oob_tcp_peer_recv_connect_ack: invalid header type: -1263225674
[singularity:26014] [[54490,1],2] routed:binomial: Connection to lifeline [[54490,0],0] lost
[singularity:26015] mca_oob_tcp_peer_recv_connect_ack: invalid header type: -1263225674
[singularity:26015] [[54490,1],3] routed:binomial: Connection to lifeline [[54490,0],0] lost
The code I ran should have no effect on the run (sender XORs, and receiver UnXORs with the same content), unless I'm missing a send/recv function...
My code I ran is attached at the bottom. Does anyone have an Idea as to why does it fail or how to make it work?
I'll appreciate any ideas,
Alex
P.S. This code works (w/ and w/o XORing) on other (older) MPI implementations, but I was hoping to make it work with the latest stable OpenMPI.
On Jan 21, 2012, at 2:33 PM CST, Alex Margolin wrote:
>> On 01/21/2012 10:25 PM, Alex Margolin wrote:
>>> Hi,
>>>
>>> I've written a general-purpose adapter for TCP-based sockets in linux apps (libdicom.so).
>>> The adapter does some optimizations, but mostly forwards the calls to the original functions.
>>> When I tried to run it with MPICH, the application got stuck (used Ctrl-C to terminate).
>>> To debug this, I've also written a dummy lib which forwards calls without any changes (only printouts).
>>> For some reason, the dummy fails too, but with different output...
>>>
>>> I used LD_PRELOAD to catch the following (mostly) Berkeley socket API functions:
>>> - Socket creation socket, socketpair
>>> - Socket establishment: bind, listen, connect, accept
>>> - Socket data transfer: send, recv, read, write, readv, writev, recvfrom, sendto
>>> - Socket events: select, poll
>>> - File descriptor actions: close, dup, dup2, fcntl
>>>
>>> My adapter detects TCP sockets using the following condition, and should "handle" only these sockets (other calls are automatically forwarded):
>>> #define SHOULDNT_BE_DICOM ((domain != AF_INET) || ((protocol != IPPROTO_TCP)&& (protocol != IPPROTO_IP)) || (0 == (type& SOCK_STREAM)))
>>> ...
/*
* Naive implementation of the socket migration concept -
* save last known address and use UDP to send there.
*/
#define PRINTF //printf
#define XORER 0xb4
#define SHOULDNT_BE_DICOM ((domain != AF_INET) || ((protocol !=
IPPROTO_TCP)&& (protocol != IPPROTO_IP)) || (0 == (type& SOCK_STREAM)))
#define _GNU_SOURCE
#include<sys/types.h>
#include<sys/socket.h>
#include<sys/ioctl.h>
#include<netinet/in.h>
#include<arpa/inet.h>
#include<errno.h>
#include<time.h> //for nanosleep
#include<stdlib.h>
#include<sys/stat.h>
#include<sys/time.h>
#include<string.h>
#include<stdio.h>
#include<stddef.h>
#include<sys/uio.h>
#include<poll.h>
#include<signal.h>
#include<dlfcn.h> // For dynamic (re-)linking
#include<stdarg.h>
#include<sched.h> // For sched_get/setaffinity()
/* Socket control */
typedef int(*socket_orig)(int domain, int type, int protocol);
typedef int(*socketpair_orig)(int domain, int type, int protocol, int
sv[2]);
typedef int(*bind_orig)(int sockfd, const struct sockaddr *addr,
socklen_t addrlen);
typedef int(*connect_orig)(int sockfd, const struct sockaddr *addr,
socklen_t addrlen);
typedef int(*accept_orig)(int sockfd, struct sockaddr *addr, socklen_t
*addrlen);
typedef int(*close_orig)(int fd);
typedef int(*poll_orig)(struct pollfd *fds, nfds_t nfds, int timeout);
typedef int(*select_orig)(int nfds, fd_set *readfds, fd_set *writefds,
fd_set *exceptfds, struct timeval *timeout);
typedef int(*fcntl_orig)(int fd, int cmd, ... );
/* Socket reading */
typedef ssize_t(*read_orig)(int sockfd, void *buf, size_t len);
typedef ssize_t(*recv_orig)(int sockfd, void *buf, size_t len, int flags);
typedef ssize_t(*readv_orig)(int fd, const struct iovec *iov, int iovcnt);
typedef ssize_t(*recvfrom_orig)(int sockfd, void *buf, size_t len, int
flags,
struct sockaddr *src_addr, socklen_t *addrlen);
/* Socket writing */
typedef ssize_t(*send_orig)(int sockfd, const void *buf, size_t len, int
flags);
typedef ssize_t(*write_orig)(int sockfd, const void *buf, size_t len);
typedef ssize_t(*writev_orig)(int fd, const struct iovec *iov, int iovcnt);
typedef ssize_t(*sendto_orig)(int sockfd, const void *buf, size_t len,
int flags,
const struct sockaddr *dest_addr, socklen_t addrlen);
int sock_counter[10000] = {0};
int socket(int domain, int type, int protocol)
{
int sock = (*(socket_orig)dlsym(RTLD_NEXT, "socket"))(domain, type,
protocol);
if (!(SHOULDNT_BE_DICOM)) { sock_counter[sock]++; PRINTF("Opened
%i\n", sock); }
PRINTF("#%i socket() == %i (%i)\n", getpid(), sock, sock_counter[sock]);
return sock;
}
int socketpair(int domain, int type, int protocol, int sv[2])
{
int sock = (*(socketpair_orig)dlsym(RTLD_NEXT, "socketpair"))(domain,
type, protocol, sv);
PRINTF("#%i socketpair() == %i [%i, %i] (%i)\n", getpid(), sock,
sv[0], sv[1], sock_counter[sock]);
return sock;
}
int close(int fd)
{
if (sock_counter[fd]> 0)
{
PRINTF("close(%i)\n", fd);
sock_counter[fd]--;
}
return (*(close_orig)dlsym(RTLD_NEXT, "close"))(fd);
}
int bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen)
{
return (*(bind_orig)dlsym(RTLD_NEXT, "bind"))(sockfd, addr, addrlen);
}
int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen)
{
return (*(connect_orig)dlsym(RTLD_NEXT, "connect"))(sockfd, addr,
addrlen);
}
int accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen)
{
return (*(accept_orig)dlsym(RTLD_NEXT, "accept"))(sockfd, addr, addrlen);
}
ssize_t recv(int sockfd, void *buf, size_t len, int flags)
{
int a = (*(recv_orig)dlsym(RTLD_NEXT, "recv"))(sockfd, buf, len, flags);
#ifdef XORER
int i; if (sock_counter[sockfd]> 0) for (i = 0; i< len; i++)
((char*)buf)[i] ^= XORER;
#endif
if (sock_counter[sockfd])
PRINTF("#%i recv(%i,%i,%i) = %i\n", getpid(), sockfd, len, flags, a);
return a;
}
ssize_t read(int sockfd, void *buf, size_t len)
{
int a = (*(read_orig)dlsym(RTLD_NEXT, "read"))(sockfd, buf, len);
#ifdef XORER
int i; if (sock_counter[sockfd]> 0) for (i = 0; i< len; i++)
((char*)buf)[i] ^= XORER;
#endif
if (sock_counter[sockfd])
PRINTF("#%i read(%i,%i,%i) = %i\n", getpid(), sockfd, len, a);
return a;
}
ssize_t readv(int fd, const struct iovec *iov, int iovcnt)
{
int a = (*(readv_orig)dlsym(RTLD_NEXT, "readv"))(fd, iov, iovcnt);
#ifdef XORER
int i,j; if (sock_counter[fd]> 0) for (j = 0; j< iovcnt; j++) for (i
= 0; i< iov[j].iov_len; i++) ((char*)(iov[j].iov_base))[i] ^= XORER;
#endif
if (sock_counter[fd])
PRINTF("#%i readv(%i,%p,%i) = %i\n", getpid(), fd, iov, iovcnt, a);
return a;
}
ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags,
struct sockaddr *src_addr, socklen_t *addrlen)
{
int a = (*(recvfrom_orig)dlsym(RTLD_NEXT, "recvfrom"))(sockfd, buf,
len, flags, src_addr, addrlen);
#ifdef XORER
int i; if (sock_counter[sockfd]> 0) for (i = 0; i< len; i++)
((char*)buf)[i] ^= XORER;
#endif
if (sock_counter[sockfd])
PRINTF("#%i recvfrom(%i,%i,%i,%p) = %i\n", getpid(), sockfd, len,
flags, src_addr, a);
return a;
}
ssize_t send(int sockfd, const void *buf, size_t len, int flags)
{
int a = (*(send_orig)dlsym(RTLD_NEXT, "send"))(sockfd, buf, len, flags);
#ifdef XORER
int i; if (sock_counter[sockfd]> 0) for (i = 0; i< len; i++)
((char*)buf)[i] ^= XORER;
#endif
if (sock_counter[sockfd])
PRINTF("#%i send(%i,%i,%i) = %i\n", getpid(), sockfd, len, flags, a);
return a;
}
ssize_t write(int sockfd, const void *buf, size_t len)
{
#ifdef XORER
int i; if (sock_counter[sockfd]> 0) for (i = 0; i< len; i++)
((char*)buf)[i] ^= XORER;
#endif
int a = (*(write_orig)dlsym(RTLD_NEXT, "write"))(sockfd, buf, len);
if (sock_counter[sockfd])
PRINTF("#%i write(%i,%i) = %i\n", getpid(), sockfd, len, a);
return a;
}
ssize_t writev(int fd, const struct iovec *iov, int iovcnt)
{
#ifdef XORER
int i,j; if (sock_counter[fd]> 0) for (j = 0; j< iovcnt; j++) for (i
= 0; i< iov[j].iov_len; i++) ((char*)(iov[j].iov_base))[i] ^= XORER;
#endif
int a = (*(writev_orig)dlsym(RTLD_NEXT, "writev"))(fd, iov, iovcnt);
if (sock_counter[fd])
PRINTF("#%i writev(%i,%p,%i) = %i\n", getpid(), fd, iov, iovcnt, a);
return a;
}
ssize_t sendto(int sockfd, const void *buf, size_t len, int flags,
const struct sockaddr *dest_addr, socklen_t addrlen)
{
#ifdef XORER
int i; if (sock_counter[sockfd]> 0) for (i = 0; i< len; i++)
((char*)buf)[i] ^= XORER;
#endif
int a = (*(sendto_orig)dlsym(RTLD_NEXT, "sendto"))(sockfd, buf, len,
flags, dest_addr, addrlen);
if (sock_counter[sockfd])
PRINTF("#%i sendto(%i,%i,%i,%p,%i) = %i\n", getpid(), sockfd, len,
flags, dest_addr, addrlen, a);
return a;
}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/mpich-discuss/attachments/20120207/00f90687/attachment.htm>
More information about the mpich-discuss
mailing list