From e18a033b921d0d79fa8278f853548e6125b93e0c Mon Sep 17 00:00:00 2001 From: Konstantin Ananyev Date: Tue, 31 Oct 2017 12:40:17 +0000 Subject: Integrate TLDK with NGINX Created a clone of nginx (from https://github.com/nginx/nginx) to demonstrate and benchmark TLDK library integrated with real world application. A new nginx module is created and and BSD socket-like API is implemented on top of native TLDK API. Note, that right now only minimalistic subset of socket-like API is provided: - accept - close - readv - recv - writev so only limited nginx functionality is available for a moment. Change-Id: Ie1efe9349a0538da4348a48fb8306cbf636b5a92 Signed-off-by: Mohammad Abdul Awal Signed-off-by: Reshma Pattan Signed-off-by: Remy Horton Signed-off-by: Konstantin Ananyev --- app/nginx/src/event/modules/ngx_epoll_module.c | 1052 ++++++++++++++++++++++++ 1 file changed, 1052 insertions(+) create mode 100644 app/nginx/src/event/modules/ngx_epoll_module.c (limited to 'app/nginx/src/event/modules/ngx_epoll_module.c') diff --git a/app/nginx/src/event/modules/ngx_epoll_module.c b/app/nginx/src/event/modules/ngx_epoll_module.c new file mode 100644 index 0000000..76aee08 --- /dev/null +++ b/app/nginx/src/event/modules/ngx_epoll_module.c @@ -0,0 +1,1052 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Nginx, Inc. + */ + + +#include +#include +#include + + +#if (NGX_TEST_BUILD_EPOLL) + +/* epoll declarations */ + +#define EPOLLIN 0x001 +#define EPOLLPRI 0x002 +#define EPOLLOUT 0x004 +#define EPOLLERR 0x008 +#define EPOLLHUP 0x010 +#define EPOLLRDNORM 0x040 +#define EPOLLRDBAND 0x080 +#define EPOLLWRNORM 0x100 +#define EPOLLWRBAND 0x200 +#define EPOLLMSG 0x400 + +#define EPOLLRDHUP 0x2000 + +#define EPOLLEXCLUSIVE 0x10000000 +#define EPOLLONESHOT 0x40000000 +#define EPOLLET 0x80000000 + +#define EPOLL_CTL_ADD 1 +#define EPOLL_CTL_DEL 2 +#define EPOLL_CTL_MOD 3 + +typedef union epoll_data { + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; +} epoll_data_t; + +struct epoll_event { + uint32_t events; + epoll_data_t data; +}; + + +int epoll_create(int size); + +int epoll_create(int size) +{ + return -1; +} + + +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); + +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) +{ + return -1; +} + + +int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout); + +int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout) +{ + return -1; +} + +#if (NGX_HAVE_EVENTFD) +#define SYS_eventfd 323 +#endif + +#if (NGX_HAVE_FILE_AIO) + +#define SYS_io_setup 245 +#define SYS_io_destroy 246 +#define SYS_io_getevents 247 + +typedef u_int aio_context_t; + +struct io_event { + uint64_t data; /* the data field from the iocb */ + uint64_t obj; /* what iocb this event came from */ + int64_t res; /* result code for this event */ + int64_t res2; /* secondary result */ +}; + + +#endif +#endif /* NGX_TEST_BUILD_EPOLL */ + + +typedef struct { + ngx_uint_t events; + ngx_uint_t aio_requests; +} ngx_epoll_conf_t; + + +static ngx_int_t ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer); +#if (NGX_HAVE_EVENTFD) +static ngx_int_t ngx_epoll_notify_init(ngx_log_t *log); +static void ngx_epoll_notify_handler(ngx_event_t *ev); +#endif +#if (NGX_HAVE_EPOLLRDHUP) +static void ngx_epoll_test_rdhup(ngx_cycle_t *cycle); +#endif +static void ngx_epoll_done(ngx_cycle_t *cycle); +static ngx_int_t ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, + ngx_uint_t flags); +static ngx_int_t ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event, + ngx_uint_t flags); +static ngx_int_t ngx_epoll_add_connection(ngx_connection_t *c); +static ngx_int_t ngx_epoll_del_connection(ngx_connection_t *c, + ngx_uint_t flags); +#if (NGX_HAVE_EVENTFD) +static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler); +#endif +static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, + ngx_uint_t flags); + +#if (NGX_HAVE_FILE_AIO) +static void ngx_epoll_eventfd_handler(ngx_event_t *ev); +#endif + +static void *ngx_epoll_create_conf(ngx_cycle_t *cycle); +static char *ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf); + +static int ep = -1; +static struct epoll_event *event_list; +static ngx_uint_t nevents; + +#if (NGX_HAVE_EVENTFD) +static int notify_fd = -1; +static ngx_event_t notify_event; +static ngx_connection_t notify_conn; +#endif + +#if (NGX_HAVE_FILE_AIO) + +int ngx_eventfd = -1; +aio_context_t ngx_aio_ctx = 0; + +static ngx_event_t ngx_eventfd_event; +static ngx_connection_t ngx_eventfd_conn; + +#endif + +#if (NGX_HAVE_EPOLLRDHUP) +ngx_uint_t ngx_use_epoll_rdhup; +#endif + +static ngx_str_t epoll_name = ngx_string("epoll"); + +static ngx_command_t ngx_epoll_commands[] = { + + { ngx_string("epoll_events"), + NGX_EVENT_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + 0, + offsetof(ngx_epoll_conf_t, events), + NULL }, + + { ngx_string("worker_aio_requests"), + NGX_EVENT_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + 0, + offsetof(ngx_epoll_conf_t, aio_requests), + NULL }, + + ngx_null_command +}; + + +static ngx_event_module_t ngx_epoll_module_ctx = { + &epoll_name, + ngx_epoll_create_conf, /* create configuration */ + ngx_epoll_init_conf, /* init configuration */ + + { + ngx_epoll_add_event, /* add an event */ + ngx_epoll_del_event, /* delete an event */ + ngx_epoll_add_event, /* enable an event */ + ngx_epoll_del_event, /* disable an event */ + ngx_epoll_add_connection, /* add an connection */ + ngx_epoll_del_connection, /* delete an connection */ +#if (NGX_HAVE_EVENTFD) + ngx_epoll_notify, /* trigger a notify */ +#else + NULL, /* trigger a notify */ +#endif + ngx_epoll_process_events, /* process the events */ + ngx_epoll_init, /* init the events */ + ngx_epoll_done, /* done the events */ + } +}; + +ngx_module_t ngx_epoll_module = { + NGX_MODULE_V1, + &ngx_epoll_module_ctx, /* module context */ + ngx_epoll_commands, /* module directives */ + NGX_EVENT_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +#if (NGX_HAVE_FILE_AIO) + +/* + * We call io_setup(), io_destroy() io_submit(), and io_getevents() directly + * as syscalls instead of libaio usage, because the library header file + * supports eventfd() since 0.3.107 version only. + */ + +static int +io_setup(u_int nr_reqs, aio_context_t *ctx) +{ + return syscall(SYS_io_setup, nr_reqs, ctx); +} + + +static int +io_destroy(aio_context_t ctx) +{ + return syscall(SYS_io_destroy, ctx); +} + + +static int +io_getevents(aio_context_t ctx, long min_nr, long nr, struct io_event *events, + struct timespec *tmo) +{ + return syscall(SYS_io_getevents, ctx, min_nr, nr, events, tmo); +} + + +static void +ngx_epoll_aio_init(ngx_cycle_t *cycle, ngx_epoll_conf_t *epcf) +{ + int n; + struct epoll_event ee; + +#if (NGX_HAVE_SYS_EVENTFD_H) + ngx_eventfd = eventfd(0, 0); +#else + ngx_eventfd = syscall(SYS_eventfd, 0); +#endif + + if (ngx_eventfd == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "eventfd() failed"); + ngx_file_aio = 0; + return; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "eventfd: %d", ngx_eventfd); + + n = 1; + + if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "ioctl(eventfd, FIONBIO) failed"); + goto failed; + } + + if (io_setup(epcf->aio_requests, &ngx_aio_ctx) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "io_setup() failed"); + goto failed; + } + + ngx_eventfd_event.data = &ngx_eventfd_conn; + ngx_eventfd_event.handler = ngx_epoll_eventfd_handler; + ngx_eventfd_event.log = cycle->log; + ngx_eventfd_event.active = 1; + ngx_eventfd_conn.fd = ngx_eventfd; + ngx_eventfd_conn.read = &ngx_eventfd_event; + ngx_eventfd_conn.log = cycle->log; + + ee.events = EPOLLIN|EPOLLET; + ee.data.ptr = &ngx_eventfd_conn; + + if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) != -1) { + return; + } + + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed"); + + if (io_destroy(ngx_aio_ctx) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "io_destroy() failed"); + } + +failed: + + if (close(ngx_eventfd) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "eventfd close() failed"); + } + + ngx_eventfd = -1; + ngx_aio_ctx = 0; + ngx_file_aio = 0; +} + +#endif + + +static ngx_int_t +ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer) +{ + ngx_epoll_conf_t *epcf; + + epcf = ngx_event_get_conf(cycle->conf_ctx, ngx_epoll_module); + + if (ep == -1) { + ep = epoll_create(cycle->connection_n / 2); + + if (ep == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "epoll_create() failed"); + return NGX_ERROR; + } + +#if (NGX_HAVE_EVENTFD) + if (ngx_epoll_notify_init(cycle->log) != NGX_OK) { + ngx_epoll_module_ctx.actions.notify = NULL; + } +#endif + +#if (NGX_HAVE_FILE_AIO) + ngx_epoll_aio_init(cycle, epcf); +#endif + +#if (NGX_HAVE_EPOLLRDHUP) + ngx_epoll_test_rdhup(cycle); +#endif + } + + if (nevents < epcf->events) { + if (event_list) { + ngx_free(event_list); + } + + event_list = ngx_alloc(sizeof(struct epoll_event) * epcf->events, + cycle->log); + if (event_list == NULL) { + return NGX_ERROR; + } + } + + nevents = epcf->events; + + ngx_io = ngx_os_io; + + ngx_event_actions = ngx_epoll_module_ctx.actions; + +#if (NGX_HAVE_CLEAR_EVENT) + ngx_event_flags = NGX_USE_CLEAR_EVENT +#else + ngx_event_flags = NGX_USE_LEVEL_EVENT +#endif + |NGX_USE_GREEDY_EVENT + |NGX_USE_EPOLL_EVENT; + + return NGX_OK; +} + + +#if (NGX_HAVE_EVENTFD) + +static ngx_int_t +ngx_epoll_notify_init(ngx_log_t *log) +{ + struct epoll_event ee; + +#if (NGX_HAVE_SYS_EVENTFD_H) + notify_fd = eventfd(0, 0); +#else + notify_fd = syscall(SYS_eventfd, 0); +#endif + + if (notify_fd == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "eventfd() failed"); + return NGX_ERROR; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, log, 0, + "notify eventfd: %d", notify_fd); + + notify_event.handler = ngx_epoll_notify_handler; + notify_event.log = log; + notify_event.active = 1; + + notify_conn.fd = notify_fd; + notify_conn.read = ¬ify_event; + notify_conn.log = log; + + ee.events = EPOLLIN|EPOLLET; + ee.data.ptr = ¬ify_conn; + + if (epoll_ctl(ep, EPOLL_CTL_ADD, notify_fd, &ee) == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed"); + + if (close(notify_fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + "eventfd close() failed"); + } + + return NGX_ERROR; + } + + return NGX_OK; +} + + +static void +ngx_epoll_notify_handler(ngx_event_t *ev) +{ + ssize_t n; + uint64_t count; + ngx_err_t err; + ngx_event_handler_pt handler; + + if (++ev->index == NGX_MAX_UINT32_VALUE) { + ev->index = 0; + + n = read(notify_fd, &count, sizeof(uint64_t)); + + err = ngx_errno; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "read() eventfd %d: %z count:%uL", notify_fd, n, count); + + if ((size_t) n != sizeof(uint64_t)) { + ngx_log_error(NGX_LOG_ALERT, ev->log, err, + "read() eventfd %d failed", notify_fd); + } + } + + handler = ev->data; + handler(ev); +} + +#endif + + +#if (NGX_HAVE_EPOLLRDHUP) + +static void +ngx_epoll_test_rdhup(ngx_cycle_t *cycle) +{ + int s[2], events; + struct epoll_event ee; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, s) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "socketpair() failed"); + return; + } + + ee.events = EPOLLET|EPOLLIN|EPOLLRDHUP; + + if (epoll_ctl(ep, EPOLL_CTL_ADD, s[0], &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "epoll_ctl() failed"); + goto failed; + } + + if (close(s[1]) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "close() failed"); + s[1] = -1; + goto failed; + } + + s[1] = -1; + + events = epoll_wait(ep, &ee, 1, 5000); + + if (events == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "epoll_wait() failed"); + goto failed; + } + + if (events) { + ngx_use_epoll_rdhup = ee.events & EPOLLRDHUP; + + } else { + ngx_log_error(NGX_LOG_ALERT, cycle->log, NGX_ETIMEDOUT, + "epoll_wait() timed out"); + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "testing the EPOLLRDHUP flag: %s", + ngx_use_epoll_rdhup ? "success" : "fail"); + +failed: + + if (s[1] != -1 && close(s[1]) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "close() failed"); + } + + if (close(s[0]) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "close() failed"); + } +} + +#endif + + +static void +ngx_epoll_done(ngx_cycle_t *cycle) +{ + if (close(ep) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "epoll close() failed"); + } + + ep = -1; + +#if (NGX_HAVE_EVENTFD) + + if (close(notify_fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "eventfd close() failed"); + } + + notify_fd = -1; + +#endif + +#if (NGX_HAVE_FILE_AIO) + + if (ngx_eventfd != -1) { + + if (io_destroy(ngx_aio_ctx) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "io_destroy() failed"); + } + + if (close(ngx_eventfd) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "eventfd close() failed"); + } + + ngx_eventfd = -1; + } + + ngx_aio_ctx = 0; + +#endif + + ngx_free(event_list); + + event_list = NULL; + nevents = 0; +} + + +static ngx_int_t +ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags) +{ + int op; + uint32_t events, prev; + ngx_event_t *e; + ngx_connection_t *c; + struct epoll_event ee; + + c = ev->data; + + events = (uint32_t) event; + + if (event == NGX_READ_EVENT) { + e = c->write; + prev = EPOLLOUT; +#if (NGX_READ_EVENT != EPOLLIN|EPOLLRDHUP) + events = EPOLLIN|EPOLLRDHUP; +#endif + + } else { + e = c->read; + prev = EPOLLIN|EPOLLRDHUP; +#if (NGX_WRITE_EVENT != EPOLLOUT) + events = EPOLLOUT; +#endif + } + + if (e->active) { + op = EPOLL_CTL_MOD; + events |= prev; + + } else { + op = EPOLL_CTL_ADD; + } + +#if (NGX_HAVE_EPOLLEXCLUSIVE && NGX_HAVE_EPOLLRDHUP) + if (flags & NGX_EXCLUSIVE_EVENT) { + events &= ~EPOLLRDHUP; + } +#endif + + ee.events = events | (uint32_t) flags; + ee.data.ptr = (void *) ((uintptr_t) c | ev->instance); + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "epoll add event: fd:%d op:%d ev:%08XD", + c->fd, op, ee.events); + + if (epoll_ctl(ep, op, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, + "epoll_ctl(%d, %d) failed", op, c->fd); + return NGX_ERROR; + } + + ev->active = 1; +#if 0 + ev->oneshot = (flags & NGX_ONESHOT_EVENT) ? 1 : 0; +#endif + + return NGX_OK; +} + + +static ngx_int_t +ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags) +{ + int op; + uint32_t prev; + ngx_event_t *e; + ngx_connection_t *c; + struct epoll_event ee; + + /* + * when the file descriptor is closed, the epoll automatically deletes + * it from its queue, so we do not need to delete explicitly the event + * before the closing the file descriptor + */ + + if (flags & NGX_CLOSE_EVENT) { + ev->active = 0; + return NGX_OK; + } + + c = ev->data; + + if (event == NGX_READ_EVENT) { + e = c->write; + prev = EPOLLOUT; + + } else { + e = c->read; + prev = EPOLLIN|EPOLLRDHUP; + } + + if (e->active) { + op = EPOLL_CTL_MOD; + ee.events = prev | (uint32_t) flags; + ee.data.ptr = (void *) ((uintptr_t) c | ev->instance); + + } else { + op = EPOLL_CTL_DEL; + ee.events = 0; + ee.data.ptr = NULL; + } + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "epoll del event: fd:%d op:%d ev:%08XD", + c->fd, op, ee.events); + + if (epoll_ctl(ep, op, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, + "epoll_ctl(%d, %d) failed", op, c->fd); + return NGX_ERROR; + } + + ev->active = 0; + + return NGX_OK; +} + + +static ngx_int_t +ngx_epoll_add_connection(ngx_connection_t *c) +{ + struct epoll_event ee; + + ee.events = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP; + ee.data.ptr = (void *) ((uintptr_t) c | c->read->instance); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "epoll add connection: fd:%d ev:%08XD", c->fd, ee.events); + + if (epoll_ctl(ep, EPOLL_CTL_ADD, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, + "epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->fd); + return NGX_ERROR; + } + + c->read->active = 1; + c->write->active = 1; + + return NGX_OK; +} + + +static ngx_int_t +ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags) +{ + int op; + struct epoll_event ee; + + /* + * when the file descriptor is closed the epoll automatically deletes + * it from its queue so we do not need to delete explicitly the event + * before the closing the file descriptor + */ + + if (flags & NGX_CLOSE_EVENT) { + c->read->active = 0; + c->write->active = 0; + return NGX_OK; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "epoll del connection: fd:%d", c->fd); + + op = EPOLL_CTL_DEL; + ee.events = 0; + ee.data.ptr = NULL; + + if (epoll_ctl(ep, op, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, + "epoll_ctl(%d, %d) failed", op, c->fd); + return NGX_ERROR; + } + + c->read->active = 0; + c->write->active = 0; + + return NGX_OK; +} + + +#if (NGX_HAVE_EVENTFD) + +static ngx_int_t +ngx_epoll_notify(ngx_event_handler_pt handler) +{ + static uint64_t inc = 1; + + notify_event.data = handler; + + if ((size_t) write(notify_fd, &inc, sizeof(uint64_t)) != sizeof(uint64_t)) { + ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno, + "write() to eventfd %d failed", notify_fd); + return NGX_ERROR; + } + + return NGX_OK; +} + +#endif + + +static ngx_int_t +ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) +{ + int events; + uint32_t revents; + ngx_int_t instance, i; + ngx_uint_t level; + ngx_err_t err; + ngx_event_t *rev, *wev; + ngx_queue_t *queue; + ngx_connection_t *c; + + /* NGX_TIMER_INFINITE == INFTIM */ + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "epoll timer: %M", timer); + + events = epoll_wait(ep, event_list, (int) nevents, timer); + + err = (events == -1) ? ngx_errno : 0; + + if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) { + ngx_time_update(); + } + + if (err) { + if (err == NGX_EINTR) { + + if (ngx_event_timer_alarm) { + ngx_event_timer_alarm = 0; + return NGX_OK; + } + + level = NGX_LOG_INFO; + + } else { + level = NGX_LOG_ALERT; + } + + ngx_log_error(level, cycle->log, err, "epoll_wait() failed"); + return NGX_ERROR; + } + + if (events == 0) { + if (timer != NGX_TIMER_INFINITE) { + return NGX_OK; + } + + ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, + "epoll_wait() returned no events without timeout"); + return NGX_ERROR; + } + + for (i = 0; i < events; i++) { + c = event_list[i].data.ptr; + + instance = (uintptr_t) c & 1; + c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1); + + rev = c->read; + + if (c->fd == -1 || rev->instance != instance) { + + /* + * the stale event from a file descriptor + * that was just closed in this iteration + */ + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "epoll: stale event %p", c); + continue; + } + + revents = event_list[i].events; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "epoll: fd:%d ev:%04XD d:%p", + c->fd, revents, event_list[i].data.ptr); + + if (revents & (EPOLLERR|EPOLLHUP)) { + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "epoll_wait() error on fd:%d ev:%04XD", + c->fd, revents); + + /* + * if the error events were returned, add EPOLLIN and EPOLLOUT + * to handle the events at least in one active handler + */ + + revents |= EPOLLIN|EPOLLOUT; + } + +#if 0 + if (revents & ~(EPOLLIN|EPOLLOUT|EPOLLERR|EPOLLHUP)) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, + "strange epoll_wait() events fd:%d ev:%04XD", + c->fd, revents); + } +#endif + + if ((revents & EPOLLIN) && rev->active) { + +#if (NGX_HAVE_EPOLLRDHUP) + if (revents & EPOLLRDHUP) { + rev->pending_eof = 1; + } + + rev->available = 1; +#endif + + rev->ready = 1; + + if (flags & NGX_POST_EVENTS) { + queue = rev->accept ? &ngx_posted_accept_events + : &ngx_posted_events; + + ngx_post_event(rev, queue); + + } else { + rev->handler(rev); + } + } + + wev = c->write; + + if ((revents & EPOLLOUT) && wev->active) { + + if (c->fd == -1 || wev->instance != instance) { + + /* + * the stale event from a file descriptor + * that was just closed in this iteration + */ + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "epoll: stale event %p", c); + continue; + } + + wev->ready = 1; +#if (NGX_THREADS) + wev->complete = 1; +#endif + + if (flags & NGX_POST_EVENTS) { + ngx_post_event(wev, &ngx_posted_events); + + } else { + wev->handler(wev); + } + } + } + + return NGX_OK; +} + + +#if (NGX_HAVE_FILE_AIO) + +static void +ngx_epoll_eventfd_handler(ngx_event_t *ev) +{ + int n, events; + long i; + uint64_t ready; + ngx_err_t err; + ngx_event_t *e; + ngx_event_aio_t *aio; + struct io_event event[64]; + struct timespec ts; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler"); + + n = read(ngx_eventfd, &ready, 8); + + err = ngx_errno; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n); + + if (n != 8) { + if (n == -1) { + if (err == NGX_EAGAIN) { + return; + } + + ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed"); + return; + } + + ngx_log_error(NGX_LOG_ALERT, ev->log, 0, + "read(eventfd) returned only %d bytes", n); + return; + } + + ts.tv_sec = 0; + ts.tv_nsec = 0; + + while (ready) { + + events = io_getevents(ngx_aio_ctx, 1, 64, event, &ts); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "io_getevents: %d", events); + + if (events > 0) { + ready -= events; + + for (i = 0; i < events; i++) { + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "io_event: %XL %XL %L %L", + event[i].data, event[i].obj, + event[i].res, event[i].res2); + + e = (ngx_event_t *) (uintptr_t) event[i].data; + + e->complete = 1; + e->active = 0; + e->ready = 1; + + aio = e->data; + aio->res = event[i].res; + + ngx_post_event(e, &ngx_posted_events); + } + + continue; + } + + if (events == 0) { + return; + } + + /* events == -1 */ + ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, + "io_getevents() failed"); + return; + } +} + +#endif + + +static void * +ngx_epoll_create_conf(ngx_cycle_t *cycle) +{ + ngx_epoll_conf_t *epcf; + + epcf = ngx_palloc(cycle->pool, sizeof(ngx_epoll_conf_t)); + if (epcf == NULL) { + return NULL; + } + + epcf->events = NGX_CONF_UNSET; + epcf->aio_requests = NGX_CONF_UNSET; + + return epcf; +} + + +static char * +ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf) +{ + ngx_epoll_conf_t *epcf = conf; + + ngx_conf_init_uint_value(epcf->events, 512); + ngx_conf_init_uint_value(epcf->aio_requests, 32); + + return NGX_CONF_OK; +} -- cgit 1.2.3-korg