From e18a033b921d0d79fa8278f853548e6125b93e0c Mon Sep 17 00:00:00 2001 From: Konstantin Ananyev Date: Tue, 31 Oct 2017 12:40:17 +0000 Subject: Integrate TLDK with NGINX Created a clone of nginx (from https://github.com/nginx/nginx) to demonstrate and benchmark TLDK library integrated with real world application. A new nginx module is created and and BSD socket-like API is implemented on top of native TLDK API. Note, that right now only minimalistic subset of socket-like API is provided: - accept - close - readv - recv - writev so only limited nginx functionality is available for a moment. Change-Id: Ie1efe9349a0538da4348a48fb8306cbf636b5a92 Signed-off-by: Mohammad Abdul Awal Signed-off-by: Reshma Pattan Signed-off-by: Remy Horton Signed-off-by: Konstantin Ananyev --- .../modules/ngx_http_upstream_least_conn_module.c | 314 +++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 app/nginx/src/http/modules/ngx_http_upstream_least_conn_module.c (limited to 'app/nginx/src/http/modules/ngx_http_upstream_least_conn_module.c') diff --git a/app/nginx/src/http/modules/ngx_http_upstream_least_conn_module.c b/app/nginx/src/http/modules/ngx_http_upstream_least_conn_module.c new file mode 100644 index 0000000..ebe0627 --- /dev/null +++ b/app/nginx/src/http/modules/ngx_http_upstream_least_conn_module.c @@ -0,0 +1,314 @@ + +/* + * Copyright (C) Maxim Dounin + * Copyright (C) Nginx, Inc. + */ + + +#include +#include +#include + + +static ngx_int_t ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_get_least_conn_peer( + ngx_peer_connection_t *pc, void *data); +static char *ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + + +static ngx_command_t ngx_http_upstream_least_conn_commands[] = { + + { ngx_string("least_conn"), + NGX_HTTP_UPS_CONF|NGX_CONF_NOARGS, + ngx_http_upstream_least_conn, + 0, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_upstream_least_conn_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_upstream_least_conn_module = { + NGX_MODULE_V1, + &ngx_http_upstream_least_conn_module_ctx, /* module context */ + ngx_http_upstream_least_conn_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_int_t +ngx_http_upstream_init_least_conn(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, + "init least conn"); + + if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { + return NGX_ERROR; + } + + us->peer.init = ngx_http_upstream_init_least_conn_peer; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "init least conn peer"); + + if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) { + return NGX_ERROR; + } + + r->upstream->peer.get = ngx_http_upstream_get_least_conn_peer; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_rr_peer_data_t *rrp = data; + + time_t now; + uintptr_t m; + ngx_int_t rc, total; + ngx_uint_t i, n, p, many; + ngx_http_upstream_rr_peer_t *peer, *best; + ngx_http_upstream_rr_peers_t *peers; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, try: %ui", pc->tries); + + if (rrp->peers->single) { + return ngx_http_upstream_get_round_robin_peer(pc, rrp); + } + + pc->cached = 0; + pc->connection = NULL; + + now = ngx_time(); + + peers = rrp->peers; + + ngx_http_upstream_rr_peers_wlock(peers); + + best = NULL; + total = 0; + +#if (NGX_SUPPRESS_WARN) + many = 0; + p = 0; +#endif + + for (peer = peers->peer, i = 0; + peer; + peer = peer->next, i++) + { + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + if (rrp->tried[n] & m) { + continue; + } + + if (peer->down) { + continue; + } + + if (peer->max_fails + && peer->fails >= peer->max_fails + && now - peer->checked <= peer->fail_timeout) + { + continue; + } + + if (peer->max_conns && peer->conns >= peer->max_conns) { + continue; + } + + /* + * select peer with least number of connections; if there are + * multiple peers with the same number of connections, select + * based on round-robin + */ + + if (best == NULL + || peer->conns * best->weight < best->conns * peer->weight) + { + best = peer; + many = 0; + p = i; + + } else if (peer->conns * best->weight == best->conns * peer->weight) { + many = 1; + } + } + + if (best == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, no peer found"); + + goto failed; + } + + if (many) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, many"); + + for (peer = best, i = p; + peer; + peer = peer->next, i++) + { + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + if (rrp->tried[n] & m) { + continue; + } + + if (peer->down) { + continue; + } + + if (peer->conns * best->weight != best->conns * peer->weight) { + continue; + } + + if (peer->max_fails + && peer->fails >= peer->max_fails + && now - peer->checked <= peer->fail_timeout) + { + continue; + } + + if (peer->max_conns && peer->conns >= peer->max_conns) { + continue; + } + + peer->current_weight += peer->effective_weight; + total += peer->effective_weight; + + if (peer->effective_weight < peer->weight) { + peer->effective_weight++; + } + + if (peer->current_weight > best->current_weight) { + best = peer; + p = i; + } + } + } + + best->current_weight -= total; + + if (now - best->checked > best->fail_timeout) { + best->checked = now; + } + + pc->sockaddr = best->sockaddr; + pc->socklen = best->socklen; + pc->name = &best->name; + + best->conns++; + + rrp->current = best; + + n = p / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); + + rrp->tried[n] |= m; + + ngx_http_upstream_rr_peers_unlock(peers); + + return NGX_OK; + +failed: + + if (peers->next) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get least conn peer, backup servers"); + + rrp->peers = peers->next; + + n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1)) + / (8 * sizeof(uintptr_t)); + + for (i = 0; i < n; i++) { + rrp->tried[i] = 0; + } + + ngx_http_upstream_rr_peers_unlock(peers); + + rc = ngx_http_upstream_get_least_conn_peer(pc, rrp); + + if (rc != NGX_BUSY) { + return rc; + } + + ngx_http_upstream_rr_peers_wlock(peers); + } + + ngx_http_upstream_rr_peers_unlock(peers); + + pc->name = peers->name; + + return NGX_BUSY; +} + + +static char * +ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf; + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + if (uscf->peer.init_upstream) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "load balancing method redefined"); + } + + uscf->peer.init_upstream = ngx_http_upstream_init_least_conn; + + uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_WEIGHT + |NGX_HTTP_UPSTREAM_MAX_CONNS + |NGX_HTTP_UPSTREAM_MAX_FAILS + |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT + |NGX_HTTP_UPSTREAM_DOWN + |NGX_HTTP_UPSTREAM_BACKUP; + + return NGX_CONF_OK; +} -- cgit 1.2.3-korg