1
2 /*
3 * Copyright (C) Igor Sysoev
4 */
5
6
7 #include <ngx_config.h>
8 #include <ngx_core.h>
9 #include <ngx_event.h>
10
11
12 #if 0
13 #define NGX_SENDFILE_LIMIT 4096
14 #endif
15
16 /*
17 * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly
18 * to an application memory from a device if parameters are aligned
19 * to device sector boundary (512 bytes). They fallback to usual read
20 * operation if the parameters are not aligned.
21 * Linux allows DIRECTIO only if the parameters are aligned to a filesystem
22 * sector boundary, otherwise it returns EINVAL. The sector size is
23 * usually 512 bytes, however, on XFS it may be 4096 bytes.
24 */
25
26 #define NGX_NONE 1
27
28
29 static ngx_inline ngx_int_t
30 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
31 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
32 ngx_chain_t **chain, ngx_chain_t *in);
33 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
34 off_t bsize);
35 static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
36 off_t bsize);
37 static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
38
39
40 ngx_int_t
41 ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
42 {
43 off_t bsize;
44 ngx_int_t rc, last;
45 ngx_chain_t *cl, *out, **last_out;
46
47 if (ctx->in == NULL && ctx->busy == NULL) {
48
49 /*
50 * the short path for the case when the ctx->in and ctx->busy chains
51 * are empty, the incoming chain is empty too or has the single buf
52 * that does not require the copy
53 */
54
55 if (in == NULL) {
56 return ctx->output_filter(ctx->filter_ctx, in);
57 }
58
59 if (in->next == NULL
60 #if (NGX_SENDFILE_LIMIT)
61 && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
62 #endif
63 && ngx_output_chain_as_is(ctx, in->buf))
64 {
65 return ctx->output_filter(ctx->filter_ctx, in);
66 }
67 }
68
69 /* add the incoming buf to the chain ctx->in */
70
71 if (in) {
72 if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
73 return NGX_ERROR;
74 }
75 }
76
77 #if (NGX_HAVE_FILE_AIO)
78 if (ctx->aio) {
79 return NGX_AGAIN;
80 }
81 #endif
82
83 out = NULL;
84 last_out = &out;
85 last = NGX_NONE;
86
87 for ( ;; ) {
88
89 while (ctx->in) {
90
91 /*
92 * cycle while there are the ctx->in bufs
93 * and there are the free output bufs to copy in
94 */
95
96 bsize = ngx_buf_size(ctx->in->buf);
97
98 if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
99
100 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
101 "zero size buf in output "
102 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
103 ctx->in->buf->temporary,
104 ctx->in->buf->recycled,
105 ctx->in->buf->in_file,
106 ctx->in->buf->start,
107 ctx->in->buf->pos,
108 ctx->in->buf->last,
109 ctx->in->buf->file,
110 ctx->in->buf->file_pos,
111 ctx->in->buf->file_last);
112
113 ngx_debug_point();
114
115 ctx->in = ctx->in->next;
116
117 continue;
118 }
119
120 if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
121
122 /* move the chain link to the output chain */
123
124 cl = ctx->in;
125 ctx->in = cl->next;
126
127 *last_out = cl;
128 last_out = &cl->next;
129 cl->next = NULL;
130
131 continue;
132 }
133
134 if (ctx->buf == NULL) {
135
136 rc = ngx_output_chain_align_file_buf(ctx, bsize);
137
138 if (rc == NGX_ERROR) {
139 return NGX_ERROR;
140 }
141
142 if (rc != NGX_OK) {
143
144 if (ctx->free) {
145
146 /* get the free buf */
147
148 cl = ctx->free;
149 ctx->buf = cl->buf;
150 ctx->free = cl->next;
151
152 ngx_free_chain(ctx->pool, cl);
153
154 } else if (out || ctx->allocated == ctx->bufs.num) {
155
156 break;
157
158 } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
159 return NGX_ERROR;
160 }
161 }
162 }
163
164 rc = ngx_output_chain_copy_buf(ctx);
165
166 if (rc == NGX_ERROR) {
167 return rc;
168 }
169
170 if (rc == NGX_AGAIN) {
171 if (out) {
172 break;
173 }
174
175 return rc;
176 }
177
178 /* delete the completed buf from the ctx->in chain */
179
180 if (ngx_buf_size(ctx->in->buf) == 0) {
181 ctx->in = ctx->in->next;
182 }
183
184 cl = ngx_alloc_chain_link(ctx->pool);
185 if (cl == NULL) {
186 return NGX_ERROR;
187 }
188
189 cl->buf = ctx->buf;
190 cl->next = NULL;
191 *last_out = cl;
192 last_out = &cl->next;
193 ctx->buf = NULL;
194 }
195
196 if (out == NULL && last != NGX_NONE) {
197
198 if (ctx->in) {
199 return NGX_AGAIN;
200 }
201
202 return last;
203 }
204
205 last = ctx->output_filter(ctx->filter_ctx, out);
206
207 if (last == NGX_ERROR || last == NGX_DONE) {
208 return last;
209 }
210
211 ngx_chain_update_chains(&ctx->free, &ctx->busy, &out, ctx->tag);
212 last_out = &out;
213 }
214 }
215
216
217 static ngx_inline ngx_int_t
218 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
219 {
220 ngx_uint_t sendfile;
221
222 if (ngx_buf_special(buf)) {
223 return 1;
224 }
225
226 if (buf->in_file && buf->file->directio) {
227 return 0;
228 }
229
230 sendfile = ctx->sendfile;
231
232 #if (NGX_SENDFILE_LIMIT)
233
234 if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
235 sendfile = 0;
236 }
237
238 #endif
239
240 if (!sendfile) {
241
242 if (!ngx_buf_in_memory(buf)) {
243 return 0;
244 }
245
246 buf->in_file = 0;
247 }
248
249 if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
250 return 0;
251 }
252
253 if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
254 return 0;
255 }
256
257 return 1;
258 }
259
260
261 static ngx_int_t
262 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
263 ngx_chain_t *in)
264 {
265 ngx_chain_t *cl, **ll;
266 #if (NGX_SENDFILE_LIMIT)
267 ngx_buf_t *b, *buf;
268 #endif
269
270 ll = chain;
271
272 for (cl = *chain; cl; cl = cl->next) {
273 ll = &cl->next;
274 }
275
276 while (in) {
277
278 cl = ngx_alloc_chain_link(pool);
279 if (cl == NULL) {
280 return NGX_ERROR;
281 }
282
283 #if (NGX_SENDFILE_LIMIT)
284
285 buf = in->buf;
286
287 if (buf->in_file
288 && buf->file_pos < NGX_SENDFILE_LIMIT
289 && buf->file_last > NGX_SENDFILE_LIMIT)
290 {
291 /* split a file buf on two bufs by the sendfile limit */
292
293 b = ngx_calloc_buf(pool);
294 if (b == NULL) {
295 return NGX_ERROR;
296 }
297
298 ngx_memcpy(b, buf, sizeof(ngx_buf_t));
299
300 if (ngx_buf_in_memory(buf)) {
301 buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
302 b->last = buf->pos;
303 }
304
305 buf->file_pos = NGX_SENDFILE_LIMIT;
306 b->file_last = NGX_SENDFILE_LIMIT;
307
308 cl->buf = b;
309
310 } else {
311 cl->buf = buf;
312 in = in->next;
313 }
314
315 #else
316 cl->buf = in->buf;
317 in = in->next;
318
319 #endif
320
321 cl->next = NULL;
322 *ll = cl;
323 ll = &cl->next;
324 }
325
326 return NGX_OK;
327 }
328
329
330 static ngx_int_t
331 ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
332 {
333 size_t size;
334 ngx_buf_t *in;
335
336 in = ctx->in->buf;
337
338 if (in->file == NULL || !in->file->directio) {
339 return NGX_DECLINED;
340 }
341
342 ctx->directio = 1;
343
344 size = (size_t) (in->file_pos - (in->file_pos & ~(ctx->alignment - 1)));
345
346 if (size == 0) {
347
348 if (bsize >= (off_t) ctx->bufs.size) {
349 return NGX_DECLINED;
350 }
351
352 size = (size_t) bsize;
353
354 } else {
355 size = (size_t) ctx->alignment - size;
356
357 if ((off_t) size > bsize) {
358 size = (size_t) bsize;
359 }
360 }
361
362 ctx->buf = ngx_create_temp_buf(ctx->pool, size);
363 if (ctx->buf == NULL) {
364 return NGX_ERROR;
365 }
366
367 /*
368 * we do not set ctx->buf->tag, because we do not want
369 * to reuse the buf via ctx->free list
370 */
371
372 #if (NGX_HAVE_ALIGNED_DIRECTIO)
373 ctx->unaligned = 1;
374 #endif
375
376 return NGX_OK;
377 }
378
379
380 static ngx_int_t
381 ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
382 {
383 size_t size;
384 ngx_buf_t *b, *in;
385 ngx_uint_t recycled;
386
387 in = ctx->in->buf;
388 size = ctx->bufs.size;
389 recycled = 1;
390
391 if (in->last_in_chain) {
392
393 if (bsize < (off_t) size) {
394
395 /*
396 * allocate a small temp buf for a small last buf
397 * or its small last part
398 */
399
400 size = (size_t) bsize;
401 recycled = 0;
402
403 } else if (!ctx->directio
404 && ctx->bufs.num == 1
405 && (bsize < (off_t) (size + size / 4)))
406 {
407 /*
408 * allocate a temp buf that equals to a last buf,
409 * if there is no directio, the last buf size is lesser
410 * than 1.25 of bufs.size and the temp buf is single
411 */
412
413 size = (size_t) bsize;
414 recycled = 0;
415 }
416 }
417
418 b = ngx_calloc_buf(ctx->pool);
419 if (b == NULL) {
420 return NGX_ERROR;
421 }
422
423 if (ctx->directio) {
424
425 /*
426 * allocate block aligned to a disk sector size to enable
427 * userland buffer direct usage conjunctly with directio
428 */
429
430 b->start = ngx_pmemalign(ctx->pool, size, (size_t) ctx->alignment);
431 if (b->start == NULL) {
432 return NGX_ERROR;
433 }
434
435 } else {
436 b->start = ngx_palloc(ctx->pool, size);
437 if (b->start == NULL) {
438 return NGX_ERROR;
439 }
440 }
441
442 b->pos = b->start;
443 b->last = b->start;
444 b->end = b->last + size;
445 b->temporary = 1;
446 b->tag = ctx->tag;
447 b->recycled = recycled;
448
449 ctx->buf = b;
450 ctx->allocated++;
451
452 return NGX_OK;
453 }
454
455
456 static ngx_int_t
457 ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
458 {
459 off_t size;
460 ssize_t n;
461 ngx_buf_t *src, *dst;
462 ngx_uint_t sendfile;
463
464 src = ctx->in->buf;
465 dst = ctx->buf;
466
467 size = ngx_buf_size(src);
468
469 if (size > dst->end - dst->pos) {
470 size = dst->end - dst->pos;
471 }
472
473 sendfile = ctx->sendfile & !ctx->directio;
474
475 #if (NGX_SENDFILE_LIMIT)
476
477 if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
478 sendfile = 0;
479 }
480
481 #endif
482
483 if (ngx_buf_in_memory(src)) {
484 ngx_memcpy(dst->pos, src->pos, (size_t) size);
485 src->pos += (size_t) size;
486 dst->last += (size_t) size;
487
488 if (src->in_file) {
489
490 if (sendfile) {
491 dst->in_file = 1;
492 dst->file = src->file;
493 dst->file_pos = src->file_pos;
494 dst->file_last = src->file_pos + size;
495
496 } else {
497 dst->in_file = 0;
498 }
499
500 src->file_pos += size;
501
502 } else {
503 dst->in_file = 0;
504 }
505
506 if (src->pos == src->last) {
507 dst->flush = src->flush;
508 dst->last_buf = src->last_buf;
509 dst->last_in_chain = src->last_in_chain;
510 }
511
512 } else {
513
514 #if (NGX_HAVE_ALIGNED_DIRECTIO)
515
516 if (ctx->unaligned) {
517 if (ngx_directio_off(src->file->fd) == NGX_FILE_ERROR) {
518 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
519 ngx_directio_off_n " \"%s\" failed",
520 src->file->name.data);
521 }
522 }
523
524 #endif
525
526 #if (NGX_HAVE_FILE_AIO)
527
528 if (ctx->aio_handler) {
529 n = ngx_file_aio_read(src->file, dst->pos, (size_t) size,
530 src->file_pos, ctx->pool);
531 if (n == NGX_AGAIN) {
532 ctx->aio_handler(ctx, src->file);
533 return NGX_AGAIN;
534 }
535
536 } else {
537 n = ngx_read_file(src->file, dst->pos, (size_t) size,
538 src->file_pos);
539 }
540 #else
541
542 n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos);
543
544 #endif
545
546 #if (NGX_HAVE_ALIGNED_DIRECTIO)
547
548 if (ctx->unaligned) {
549 ngx_err_t err;
550
551 err = ngx_errno;
552
553 if (ngx_directio_on(src->file->fd) == NGX_FILE_ERROR) {
554 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
555 ngx_directio_on_n " \"%s\" failed",
556 src->file->name.data);
557 }
558
559 ngx_set_errno(err);
560
561 ctx->unaligned = 0;
562 }
563
564 #endif
565
566 if (n == NGX_ERROR) {
567 return (ngx_int_t) n;
568 }
569
570 if (n != size) {
571 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
572 ngx_read_file_n " read only %z of %O from \"%s\"",
573 n, size, src->file->name.data);
574 return NGX_ERROR;
575 }
576
577 dst->last += n;
578
579 if (sendfile) {
580 dst->in_file = 1;
581 dst->file = src->file;
582 dst->file_pos = src->file_pos;
583 dst->file_last = src->file_pos + n;
584
585 } else {
586 dst->in_file = 0;
587 }
588
589 src->file_pos += n;
590
591 if (src->file_pos == src->file_last) {
592 dst->flush = src->flush;
593 dst->last_buf = src->last_buf;
594 dst->last_in_chain = src->last_in_chain;
595 }
596 }
597
598 return NGX_OK;
599 }
600
601
602 ngx_int_t
603 ngx_chain_writer(void *data, ngx_chain_t *in)
604 {
605 ngx_chain_writer_ctx_t *ctx = data;
606
607 off_t size;
608 ngx_chain_t *cl;
609 ngx_connection_t *c;
610
611 c = ctx->connection;
612
613 for (size = 0; in; in = in->next) {
614
615 #if 1
616 if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
617 ngx_debug_point();
618 }
619 #endif
620
621 size += ngx_buf_size(in->buf);
622
623 ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0,
624 "chain writer buf fl:%d s:%uO",
625 in->buf->flush, ngx_buf_size(in->buf));
626
627 cl = ngx_alloc_chain_link(ctx->pool);
628 if (cl == NULL) {
629 return NGX_ERROR;
630 }
631
632 cl->buf = in->buf;
633 cl->next = NULL;
634 *ctx->last = cl;
635 ctx->last = &cl->next;
636 }
637
638 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
639 "chain writer in: %p", ctx->out);
640
641 for (cl = ctx->out; cl; cl = cl->next) {
642
643 #if 1
644 if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
645 ngx_debug_point();
646 }
647
648 #endif
649
650 size += ngx_buf_size(cl->buf);
651 }
652
653 if (size == 0 && !c->buffered) {
654 return NGX_OK;
655 }
656
657 ctx->out = c->send_chain(c, ctx->out, ctx->limit);
658
659 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
660 "chain writer out: %p", ctx->out);
661
662 if (ctx->out == NGX_CHAIN_ERROR) {
663 return NGX_ERROR;
664 }
665
666 if (ctx->out == NULL) {
667 ctx->last = &ctx->out;
668
669 if (!c->buffered) {
670 return NGX_OK;
671 }
672 }
673
674 return NGX_AGAIN;
675 }
676
This page was automatically generated by the
LXR engine.
Visit the LXR main site for more
information.