diff options
Diffstat (limited to 'app/nginx/src/core/ngx_output_chain.c')
-rw-r--r-- | app/nginx/src/core/ngx_output_chain.c | 767 |
1 files changed, 767 insertions, 0 deletions
diff --git a/app/nginx/src/core/ngx_output_chain.c b/app/nginx/src/core/ngx_output_chain.c new file mode 100644 index 0000000..7f5dc78 --- /dev/null +++ b/app/nginx/src/core/ngx_output_chain.c @@ -0,0 +1,767 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_event.h> + + +#if 0 +#define NGX_SENDFILE_LIMIT 4096 +#endif + +/* + * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly + * to an application memory from a device if parameters are aligned + * to device sector boundary (512 bytes). They fallback to usual read + * operation if the parameters are not aligned. + * Linux allows DIRECTIO only if the parameters are aligned to a filesystem + * sector boundary, otherwise it returns EINVAL. The sector size is + * usually 512 bytes, however, on XFS it may be 4096 bytes. + */ + +#define NGX_NONE 1 + + +static ngx_inline ngx_int_t + ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf); +#if (NGX_HAVE_AIO_SENDFILE) +static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, + ngx_file_t *file); +#endif +static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool, + ngx_chain_t **chain, ngx_chain_t *in); +static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, + off_t bsize); +static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, + off_t bsize); +static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx); + + +ngx_int_t +ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in) +{ + off_t bsize; + ngx_int_t rc, last; + ngx_chain_t *cl, *out, **last_out; + + if (ctx->in == NULL && ctx->busy == NULL +#if (NGX_HAVE_FILE_AIO || NGX_THREADS) + && !ctx->aio +#endif + ) + { + /* + * the short path for the case when the ctx->in and ctx->busy chains + * are empty, the incoming chain is empty too or has the single buf + * that does not require the copy + */ + + if (in == NULL) { + return ctx->output_filter(ctx->filter_ctx, in); + } + + if (in->next == NULL +#if (NGX_SENDFILE_LIMIT) + && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT) +#endif + && ngx_output_chain_as_is(ctx, in->buf)) + { + return ctx->output_filter(ctx->filter_ctx, in); + } + } + + /* add the incoming buf to the chain ctx->in */ + + if (in) { + if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) { + return NGX_ERROR; + } + } + + out = NULL; + last_out = &out; + last = NGX_NONE; + + for ( ;; ) { + +#if (NGX_HAVE_FILE_AIO || NGX_THREADS) + if (ctx->aio) { + return NGX_AGAIN; + } +#endif + + while (ctx->in) { + + /* + * cycle while there are the ctx->in bufs + * and there are the free output bufs to copy in + */ + + bsize = ngx_buf_size(ctx->in->buf); + + if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) { + + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, + "zero size buf in output " + "t:%d r:%d f:%d %p %p-%p %p %O-%O", + ctx->in->buf->temporary, + ctx->in->buf->recycled, + ctx->in->buf->in_file, + ctx->in->buf->start, + ctx->in->buf->pos, + ctx->in->buf->last, + ctx->in->buf->file, + ctx->in->buf->file_pos, + ctx->in->buf->file_last); + + ngx_debug_point(); + + ctx->in = ctx->in->next; + + continue; + } + + if (ngx_output_chain_as_is(ctx, ctx->in->buf)) { + + /* move the chain link to the output chain */ + + cl = ctx->in; + ctx->in = cl->next; + + *last_out = cl; + last_out = &cl->next; + cl->next = NULL; + + continue; + } + + if (ctx->buf == NULL) { + + rc = ngx_output_chain_align_file_buf(ctx, bsize); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc != NGX_OK) { + + if (ctx->free) { + + /* get the free buf */ + + cl = ctx->free; + ctx->buf = cl->buf; + ctx->free = cl->next; + + ngx_free_chain(ctx->pool, cl); + + } else if (out || ctx->allocated == ctx->bufs.num) { + + break; + + } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) { + return NGX_ERROR; + } + } + } + + rc = ngx_output_chain_copy_buf(ctx); + + if (rc == NGX_ERROR) { + return rc; + } + + if (rc == NGX_AGAIN) { + if (out) { + break; + } + + return rc; + } + + /* delete the completed buf from the ctx->in chain */ + + if (ngx_buf_size(ctx->in->buf) == 0) { + ctx->in = ctx->in->next; + } + + cl = ngx_alloc_chain_link(ctx->pool); + if (cl == NULL) { + return NGX_ERROR; + } + + cl->buf = ctx->buf; + cl->next = NULL; + *last_out = cl; + last_out = &cl->next; + ctx->buf = NULL; + } + + if (out == NULL && last != NGX_NONE) { + + if (ctx->in) { + return NGX_AGAIN; + } + + return last; + } + + last = ctx->output_filter(ctx->filter_ctx, out); + + if (last == NGX_ERROR || last == NGX_DONE) { + return last; + } + + ngx_chain_update_chains(ctx->pool, &ctx->free, &ctx->busy, &out, + ctx->tag); + last_out = &out; + } +} + + +static ngx_inline ngx_int_t +ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf) +{ + ngx_uint_t sendfile; + + if (ngx_buf_special(buf)) { + return 1; + } + +#if (NGX_THREADS) + if (buf->in_file) { + buf->file->thread_handler = ctx->thread_handler; + buf->file->thread_ctx = ctx->filter_ctx; + } +#endif + + if (buf->in_file && buf->file->directio) { + return 0; + } + + sendfile = ctx->sendfile; + +#if (NGX_SENDFILE_LIMIT) + + if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) { + sendfile = 0; + } + +#endif + + if (!sendfile) { + + if (!ngx_buf_in_memory(buf)) { + return 0; + } + + buf->in_file = 0; + } + +#if (NGX_HAVE_AIO_SENDFILE) + if (ctx->aio_preload && buf->in_file) { + (void) ngx_output_chain_aio_setup(ctx, buf->file); + } +#endif + + if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) { + return 0; + } + + if (ctx->need_in_temp && (buf->memory || buf->mmap)) { + return 0; + } + + return 1; +} + + +#if (NGX_HAVE_AIO_SENDFILE) + +static ngx_int_t +ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file) +{ + ngx_event_aio_t *aio; + + if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) { + return NGX_ERROR; + } + + aio = file->aio; + + aio->data = ctx->filter_ctx; + aio->preload_handler = ctx->aio_preload; + + return NGX_OK; +} + +#endif + + +static ngx_int_t +ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain, + ngx_chain_t *in) +{ + ngx_chain_t *cl, **ll; +#if (NGX_SENDFILE_LIMIT) + ngx_buf_t *b, *buf; +#endif + + ll = chain; + + for (cl = *chain; cl; cl = cl->next) { + ll = &cl->next; + } + + while (in) { + + cl = ngx_alloc_chain_link(pool); + if (cl == NULL) { + return NGX_ERROR; + } + +#if (NGX_SENDFILE_LIMIT) + + buf = in->buf; + + if (buf->in_file + && buf->file_pos < NGX_SENDFILE_LIMIT + && buf->file_last > NGX_SENDFILE_LIMIT) + { + /* split a file buf on two bufs by the sendfile limit */ + + b = ngx_calloc_buf(pool); + if (b == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(b, buf, sizeof(ngx_buf_t)); + + if (ngx_buf_in_memory(buf)) { + buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos); + b->last = buf->pos; + } + + buf->file_pos = NGX_SENDFILE_LIMIT; + b->file_last = NGX_SENDFILE_LIMIT; + + cl->buf = b; + + } else { + cl->buf = buf; + in = in->next; + } + +#else + cl->buf = in->buf; + in = in->next; + +#endif + + cl->next = NULL; + *ll = cl; + ll = &cl->next; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize) +{ + size_t size; + ngx_buf_t *in; + + in = ctx->in->buf; + + if (in->file == NULL || !in->file->directio) { + return NGX_DECLINED; + } + + ctx->directio = 1; + + size = (size_t) (in->file_pos - (in->file_pos & ~(ctx->alignment - 1))); + + if (size == 0) { + + if (bsize >= (off_t) ctx->bufs.size) { + return NGX_DECLINED; + } + + size = (size_t) bsize; + + } else { + size = (size_t) ctx->alignment - size; + + if ((off_t) size > bsize) { + size = (size_t) bsize; + } + } + + ctx->buf = ngx_create_temp_buf(ctx->pool, size); + if (ctx->buf == NULL) { + return NGX_ERROR; + } + + /* + * we do not set ctx->buf->tag, because we do not want + * to reuse the buf via ctx->free list + */ + +#if (NGX_HAVE_ALIGNED_DIRECTIO) + ctx->unaligned = 1; +#endif + + return NGX_OK; +} + + +static ngx_int_t +ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize) +{ + size_t size; + ngx_buf_t *b, *in; + ngx_uint_t recycled; + + in = ctx->in->buf; + size = ctx->bufs.size; + recycled = 1; + + if (in->last_in_chain) { + + if (bsize < (off_t) size) { + + /* + * allocate a small temp buf for a small last buf + * or its small last part + */ + + size = (size_t) bsize; + recycled = 0; + + } else if (!ctx->directio + && ctx->bufs.num == 1 + && (bsize < (off_t) (size + size / 4))) + { + /* + * allocate a temp buf that equals to a last buf, + * if there is no directio, the last buf size is lesser + * than 1.25 of bufs.size and the temp buf is single + */ + + size = (size_t) bsize; + recycled = 0; + } + } + + b = ngx_calloc_buf(ctx->pool); + if (b == NULL) { + return NGX_ERROR; + } + + if (ctx->directio) { + + /* + * allocate block aligned to a disk sector size to enable + * userland buffer direct usage conjunctly with directio + */ + + b->start = ngx_pmemalign(ctx->pool, size, (size_t) ctx->alignment); + if (b->start == NULL) { + return NGX_ERROR; + } + + } else { + b->start = ngx_palloc(ctx->pool, size); + if (b->start == NULL) { + return NGX_ERROR; + } + } + + b->pos = b->start; + b->last = b->start; + b->end = b->last + size; + b->temporary = 1; + b->tag = ctx->tag; + b->recycled = recycled; + + ctx->buf = b; + ctx->allocated++; + + return NGX_OK; +} + + +static ngx_int_t +ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx) +{ + off_t size; + ssize_t n; + ngx_buf_t *src, *dst; + ngx_uint_t sendfile; + + src = ctx->in->buf; + dst = ctx->buf; + + size = ngx_buf_size(src); + size = ngx_min(size, dst->end - dst->pos); + + sendfile = ctx->sendfile && !ctx->directio; + +#if (NGX_SENDFILE_LIMIT) + + if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) { + sendfile = 0; + } + +#endif + + if (ngx_buf_in_memory(src)) { + ngx_memcpy(dst->pos, src->pos, (size_t) size); + src->pos += (size_t) size; + dst->last += (size_t) size; + + if (src->in_file) { + + if (sendfile) { + dst->in_file = 1; + dst->file = src->file; + dst->file_pos = src->file_pos; + dst->file_last = src->file_pos + size; + + } else { + dst->in_file = 0; + } + + src->file_pos += size; + + } else { + dst->in_file = 0; + } + + if (src->pos == src->last) { + dst->flush = src->flush; + dst->last_buf = src->last_buf; + dst->last_in_chain = src->last_in_chain; + } + + } else { + +#if (NGX_HAVE_ALIGNED_DIRECTIO) + + if (ctx->unaligned) { + if (ngx_directio_off(src->file->fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno, + ngx_directio_off_n " \"%s\" failed", + src->file->name.data); + } + } + +#endif + +#if (NGX_HAVE_FILE_AIO) + if (ctx->aio_handler) { + n = ngx_file_aio_read(src->file, dst->pos, (size_t) size, + src->file_pos, ctx->pool); + if (n == NGX_AGAIN) { + ctx->aio_handler(ctx, src->file); + return NGX_AGAIN; + } + + } else +#endif +#if (NGX_THREADS) + if (ctx->thread_handler) { + src->file->thread_task = ctx->thread_task; + src->file->thread_handler = ctx->thread_handler; + src->file->thread_ctx = ctx->filter_ctx; + + n = ngx_thread_read(src->file, dst->pos, (size_t) size, + src->file_pos, ctx->pool); + if (n == NGX_AGAIN) { + ctx->thread_task = src->file->thread_task; + return NGX_AGAIN; + } + + } else +#endif + { + n = ngx_read_file(src->file, dst->pos, (size_t) size, + src->file_pos); + } + +#if (NGX_HAVE_ALIGNED_DIRECTIO) + + if (ctx->unaligned) { + ngx_err_t err; + + err = ngx_errno; + + if (ngx_directio_on(src->file->fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno, + ngx_directio_on_n " \"%s\" failed", + src->file->name.data); + } + + ngx_set_errno(err); + + ctx->unaligned = 0; + } + +#endif + + if (n == NGX_ERROR) { + return (ngx_int_t) n; + } + + if (n != size) { + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, + ngx_read_file_n " read only %z of %O from \"%s\"", + n, size, src->file->name.data); + return NGX_ERROR; + } + + dst->last += n; + + if (sendfile) { + dst->in_file = 1; + dst->file = src->file; + dst->file_pos = src->file_pos; + dst->file_last = src->file_pos + n; + + } else { + dst->in_file = 0; + } + + src->file_pos += n; + + if (src->file_pos == src->file_last) { + dst->flush = src->flush; + dst->last_buf = src->last_buf; + dst->last_in_chain = src->last_in_chain; + } + } + + return NGX_OK; +} + + +ngx_int_t +ngx_chain_writer(void *data, ngx_chain_t *in) +{ + ngx_chain_writer_ctx_t *ctx = data; + + off_t size; + ngx_chain_t *cl, *ln, *chain; + ngx_connection_t *c; + + c = ctx->connection; + + for (size = 0; in; in = in->next) { + +#if 1 + if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) { + + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, + "zero size buf in chain writer " + "t:%d r:%d f:%d %p %p-%p %p %O-%O", + in->buf->temporary, + in->buf->recycled, + in->buf->in_file, + in->buf->start, + in->buf->pos, + in->buf->last, + in->buf->file, + in->buf->file_pos, + in->buf->file_last); + + ngx_debug_point(); + + continue; + } +#endif + + size += ngx_buf_size(in->buf); + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0, + "chain writer buf fl:%d s:%uO", + in->buf->flush, ngx_buf_size(in->buf)); + + cl = ngx_alloc_chain_link(ctx->pool); + if (cl == NULL) { + return NGX_ERROR; + } + + cl->buf = in->buf; + cl->next = NULL; + *ctx->last = cl; + ctx->last = &cl->next; + } + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, + "chain writer in: %p", ctx->out); + + for (cl = ctx->out; cl; cl = cl->next) { + +#if 1 + if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) { + + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, + "zero size buf in chain writer " + "t:%d r:%d f:%d %p %p-%p %p %O-%O", + cl->buf->temporary, + cl->buf->recycled, + cl->buf->in_file, + cl->buf->start, + cl->buf->pos, + cl->buf->last, + cl->buf->file, + cl->buf->file_pos, + cl->buf->file_last); + + ngx_debug_point(); + + continue; + } +#endif + + size += ngx_buf_size(cl->buf); + } + + if (size == 0 && !c->buffered) { + return NGX_OK; + } + + chain = c->send_chain(c, ctx->out, ctx->limit); + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, + "chain writer out: %p", chain); + + if (chain == NGX_CHAIN_ERROR) { + return NGX_ERROR; + } + + for (cl = ctx->out; cl && cl != chain; /* void */) { + ln = cl; + cl = cl->next; + ngx_free_chain(ctx->pool, ln); + } + + ctx->out = chain; + + if (ctx->out == NULL) { + ctx->last = &ctx->out; + + if (!c->buffered) { + return NGX_OK; + } + } + + return NGX_AGAIN; +} |