Thread: EPOLL echo server not concurrent

Hi all,

I am working on an epoll version of an echoserver that I am porting from a multithreaded version I wrote.

What it should do:
The server should get a connection from a client > say x client connected > print x message from said client.

What it is doing:
The server looks like it is only accepting one connection at a time, and any other clients are queued. I thought maybe at first that it was because it was running on the same host, but I tried from another box and it looks like it is still blocking.


Any suggestions?

Code:

#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>
#include <netdb.h>
#include <strings.h>
#include <string.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <signal.h>

#define TRUE 		1
#define FALSE 		0
#define EPOLL_QUEUE_LEN 10
#define BUFLEN		1024
#define SERVER_PORT	7000

//Globals
int fd_server;

struct clientStuff
{
  int sockfd;
  struct sockaddr_in client;
};

// Function prototypes
static void SystemFatal (const char* message);
static void* ClearSocket (void* arg);
void close_fd (int);

int main (int argc, char* argv[]) 
{  
  int i, arg, visits; 
  int num_fds, fd_new, epoll_fd;
  static struct epoll_event events[EPOLL_QUEUE_LEN], event;
  int port = SERVER_PORT;
  struct sockaddr_in addr, remote_addr;
  socklen_t addr_size = sizeof(struct sockaddr_in);
  struct sigaction act;
  FILE *fp;
  visits =  0;

  // set up the signal handler to close the server socket when CTRL-c is received
  act.sa_handler = close_fd;
  act.sa_flags = 0;
  if ((sigemptyset (&act.sa_mask) == -1 || sigaction (SIGINT, &act, NULL) == -1))
  {
    perror ("Failed to set SIGINT handler");
    exit (EXIT_FAILURE);
  }
  
  // Create the listening socket
  fd_server = socket (AF_INET, SOCK_STREAM, 0);
  if (fd_server == -1) 
    SystemFatal("socket");
  
  // set SO_REUSEADDR so port can be resused imemediately after exit, i.e., after CTRL-c
  arg = 1;
  if (setsockopt (fd_server, SOL_SOCKET, SO_REUSEADDR, &arg, sizeof(arg)) == -1) 
    SystemFatal("setsockopt");
  
  // Make the server listening socket non-blocking
  if (fcntl (fd_server, F_SETFL, O_NONBLOCK | fcntl (fd_server, F_GETFL, 0)) == -1) 
    SystemFatal("fcntl");
  
  // Bind to the specified listening port
  memset (&addr, 0, sizeof (struct sockaddr_in));
  addr.sin_family = AF_INET;
  addr.sin_addr.s_addr = htonl(INADDR_ANY);
  addr.sin_port = htons(port);
  if (bind (fd_server, (struct sockaddr*) &addr, sizeof(addr)) == -1) 
    SystemFatal("bind");
  
  // Listen for fd_news; SOMAXCONN is 128 by default
  if (listen (fd_server, SOMAXCONN) == -1) 
    SystemFatal("listen");
  
  // Create the epoll file descriptor
  epoll_fd = epoll_create(EPOLL_QUEUE_LEN);
  if (epoll_fd == -1) 
    SystemFatal("epoll_create");
  
  // Add the server socket to the epoll event loop
  event.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLET;
  event.data.fd = fd_server;
  if (epoll_ctl (epoll_fd, EPOLL_CTL_ADD, fd_server, &event) == -1) 
    SystemFatal("epoll_ctl");
  
  // Execute the epoll event loop
  while (TRUE) 
  {
     // signal( SIGABRT, function);
    //struct epoll_event events[256];
    num_fds = epoll_wait (epoll_fd, events, EPOLL_QUEUE_LEN, -1);
    if (num_fds < 0) 
      SystemFatal ("Error in epoll_wait!");
    
    for (i = 0; i < num_fds; i++) 
    {
      // Case 1: Error condition
      if (events[i].events & (EPOLLHUP | EPOLLERR)) 
      {
	fputs("epoll: EPOLLERR", stderr);
	close(events[i].data.fd);
	continue;
      }
      assert (events[i].events & EPOLLIN);
      
      // Case 2: Server is receiving a connection request
      if (events[i].data.fd == fd_server) 
      {
	struct clientStuff c1;

	fd_new = accept (fd_server, (struct sockaddr*) &remote_addr, &addr_size);
        
	  printf(">> Initializing remote address: %s\n", inet_ntoa(remote_addr.sin_addr));
	  
	  visits++;
	  
	  fp=fopen("server-output.txt", "a"); /* Open file */
	  fprintf(fp, "Connections-to-date,%u\n",visits);
	  fprintf(fp, "%s,connect\n",inet_ntoa(remote_addr.sin_addr));
	  fclose(fp);
	  
	  printf("Connections to date: %u \n",visits);	  

	c1.sockfd = fd_new;
	printf("sockfd: %u\n",c1.sockfd);
	c1.client = remote_addr;
	
	if (fd_new == -1) 
	{
	  if (errno != EAGAIN && errno != EWOULDBLOCK) 
	  {
	    perror("accept");
	  }
	  continue;
	}
	
	// Make the fd_new non-blocking
	if (fcntl (fd_new, F_SETFL, O_NONBLOCK | fcntl(fd_new, F_GETFL, 0)) == -1) 
	  SystemFatal("fcntl");
	
	// Add the new socket descriptor to the epoll loop
	  event.data.fd = fd_new;
	  if (epoll_ctl (epoll_fd, EPOLL_CTL_ADD, fd_new, &event) == -1) 
	    SystemFatal ("epoll_ctl");
	  
	  continue;
      }
      
      // Case 3: One of the sockets has read data
      if (!ClearSocket(events[i].data.fd)) 
      {
	// epoll will remove the fd from its set
	// automatically when the fd is closed
	close (events[i].data.fd);
      }
    }
  }
  close(fd_server);
  exit (EXIT_SUCCESS);
}



static void* ClearSocket (void* arg) 
{
 struct clientStuff *ptr;
  struct sockaddr_in client_addr;
  int bytes_to_read, sockfd;
  char *bp, buf[BUFLEN];
  ssize_t n;
  FILE *fp;
  
  //ptr = (struct clientStuff *)arg;

  sockfd = (int) arg;
    //printf("sockfd: %u",ptr->sockfd);
 // client_addr = ptr->client;
  bp = buf;
  bytes_to_read = BUFLEN -1;
  
  while(1) {
    n = read(sockfd, bp, bytes_to_read);
    
      bp += n;
      bytes_to_read -= n;
      
      if (bytes_to_read <= 0)
      {
	/* File I/O Preparations */
	fp=fopen("server-output.txt", "a");
	fprintf(fp,"%s,Recv,%s\n",inet_ntoa(client_addr.sin_addr), buf);	
	printf("%s,Recv,%s\n", inet_ntoa(client_addr.sin_addr), buf);
      }
      if((write(sockfd, "DATA-ACK", strlen("DATA-ACK"))) == -1) {
	  fprintf(fp, "%s,Disconnect\n",inet_ntoa(client_addr.sin_addr));
	  fclose(fp);
	  printf("Error on read\n");
	  close(sockfd);
	  return 1;
      }
      
      fprintf(fp, "%s,Sent,DATA-ACK\n",inet_ntoa(client_addr.sin_addr));
      fclose(fp);
      sleep(9);
    
  } 
  return 1;
}

// Prints the error stored in errno and aborts the program.
static void SystemFatal(const char* message) 
{
  perror (message);
  exit (EXIT_FAILURE);
}

// close fd
void close_fd (int signo)
{
  close(fd_server);
  exit (EXIT_SUCCESS);
}

I know this is a old thread and probably may answer will not help because is to late but I considered a interesting challenge to find the issue.

At first everything seem to be done by the book... but the issue is actually simple.

The problem is the while(1) in the ClearSocket function which will keep looping until the client close the connection (write fails). And while doing this no other connections are handled.

The correct approach is to use separate buffers and states for each connection and as soon as we manage to read any data from the socket break the loop and allow other connection to be handled.