1
2 /*
3 * Copyright (C) Igor Sysoev
4 */
5
6
7 #include <ngx_config.h>
8 #include <ngx_core.h>
9 #include <ngx_event.h>
10
11
12 #if 0
13 #define NGX_SENDFILE_LIMIT 4096
14 #endif
15
16 /*
17 * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly
18 * to an application memory from a device if parameters are aligned
19 * to device sector boundary(512 bytes). They fallback to usual read
20 * operation if the parameters are not aligned.
21 * Linux allows DIRECTIO only if the parameters are aligned to a filesystem
22 * sector boundary, otherwise it returns EINVAL. The sector size is
23 * usually 512 bytes, however, on XFS it may be 4096 bytes.
24 */
25 #define NGX_DIRECTIO_BLOCK 4096
26
27
28 #define NGX_NONE 1
29
30
31 static ngx_inline ngx_int_t
32 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
33 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
34 ngx_chain_t **chain, ngx_chain_t *in);
35 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
36 off_t bsize);
37 static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
38 off_t bsize);
39 static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
40
41
42 ngx_int_t
43 ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
44 {
45 off_t bsize;
46 ngx_int_t rc, last;
47 ngx_chain_t *cl, *out, **last_out;
48
49 if (ctx->in == NULL && ctx->busy == NULL) {
50
51 /*
52 * the short path for the case when the ctx->in and ctx->busy chains
53 * are empty, the incoming chain is empty too or has the single buf
54 * that does not require the copy
55 */
56
57 if (in == NULL) {
58 return ctx->output_filter(ctx->filter_ctx, in);
59 }
60
61 if (in->next == NULL
62 #if (NGX_SENDFILE_LIMIT)
63 && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
64 #endif
65 && ngx_output_chain_as_is(ctx, in->buf))
66 {
67 return ctx->output_filter(ctx->filter_ctx, in);
68 }
69 }
70
71 /* add the incoming buf to the chain ctx->in */
72
73 if (in) {
74 if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
75 return NGX_ERROR;
76 }
77 }
78
79 out = NULL;
80 last_out = &out;
81 last = NGX_NONE;
82
83 for ( ;; ) {
84
85 while (ctx->in) {
86
87 /*
88 * cycle while there are the ctx->in bufs
89 * and there are the free output bufs to copy in
90 */
91
92 bsize = ngx_buf_size(ctx->in->buf);
93
94 if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
95
96 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
97 "zero size buf in output "
98 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
99 ctx->in->buf->temporary,
100 ctx->in->buf->recycled,
101 ctx->in->buf->in_file,
102 ctx->in->buf->start,
103 ctx->in->buf->pos,
104 ctx->in->buf->last,
105 ctx->in->buf->file,
106 ctx->in->buf->file_pos,
107 ctx->in->buf->file_last);
108
109 ngx_debug_point();
110
111 ctx->in = ctx->in->next;
112
113 continue;
114 }
115
116 if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
117
118 /* move the chain link to the output chain */
119
120 cl = ctx->in;
121 ctx->in = cl->next;
122
123 *last_out = cl;
124 last_out = &cl->next;
125 cl->next = NULL;
126
127 continue;
128 }
129
130 if (ctx->buf == NULL) {
131
132 rc = ngx_output_chain_align_file_buf(ctx, bsize);
133
134 if (rc == NGX_ERROR) {
135 return NGX_ERROR;
136 }
137
138 if (rc != NGX_OK) {
139
140 if (ctx->free) {
141
142 /* get the free buf */
143
144 cl = ctx->free;
145 ctx->buf = cl->buf;
146 ctx->free = cl->next;
147
148 ngx_free_chain(ctx->pool, cl);
149
150 } else if (out || ctx->allocated == ctx->bufs.num) {
151
152 break;
153
154 } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
155 return NGX_ERROR;
156 }
157 }
158 }
159
160 rc = ngx_output_chain_copy_buf(ctx);
161
162 if (rc == NGX_ERROR) {
163 return rc;
164 }
165
166 if (rc == NGX_AGAIN) {
167 if (out) {
168 break;
169 }
170
171 return rc;
172 }
173
174 /* delete the completed buf from the ctx->in chain */
175
176 if (ngx_buf_size(ctx->in->buf) == 0) {
177 ctx->in = ctx->in->next;
178 }
179
180 cl = ngx_alloc_chain_link(ctx->pool);
181 if (cl == NULL) {
182 return NGX_ERROR;
183 }
184
185 cl->buf = ctx->buf;
186 cl->next = NULL;
187 *last_out = cl;
188 last_out = &cl->next;
189 ctx->buf = NULL;
190 }
191
192 if (out == NULL && last != NGX_NONE) {
193
194 if (ctx->in) {
195 return NGX_AGAIN;
196 }
197
198 return last;
199 }
200
201 last = ctx->output_filter(ctx->filter_ctx, out);
202
203 if (last == NGX_ERROR || last == NGX_DONE) {
204 return last;
205 }
206
207 ngx_chain_update_chains(&ctx->free, &ctx->busy, &out, ctx->tag);
208 last_out = &out;
209 }
210 }
211
212
213 static ngx_inline ngx_int_t
214 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
215 {
216 ngx_uint_t sendfile;
217
218 if (ngx_buf_special(buf)) {
219 return 1;
220 }
221
222 if (buf->in_file && buf->file->directio) {
223 return 0;
224 }
225
226 sendfile = ctx->sendfile;
227
228 #if (NGX_SENDFILE_LIMIT)
229
230 if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
231 sendfile = 0;
232 }
233
234 #endif
235
236 if (!sendfile) {
237
238 if (!ngx_buf_in_memory(buf)) {
239 return 0;
240 }
241
242 buf->in_file = 0;
243 }
244
245 if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
246 return 0;
247 }
248
249 if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
250 return 0;
251 }
252
253 return 1;
254 }
255
256
257 static ngx_int_t
258 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
259 ngx_chain_t *in)
260 {
261 ngx_chain_t *cl, **ll;
262 #if (NGX_SENDFILE_LIMIT)
263 ngx_buf_t *b, *buf;
264 #endif
265
266 ll = chain;
267
268 for (cl = *chain; cl; cl = cl->next) {
269 ll = &cl->next;
270 }
271
272 while (in) {
273
274 cl = ngx_alloc_chain_link(pool);
275 if (cl == NULL) {
276 return NGX_ERROR;
277 }
278
279 #if (NGX_SENDFILE_LIMIT)
280
281 buf = in->buf;
282
283 if (buf->in_file
284 && buf->file_pos < NGX_SENDFILE_LIMIT
285 && buf->file_last > NGX_SENDFILE_LIMIT)
286 {
287 /* split a file buf on two bufs by the sendfile limit */
288
289 b = ngx_calloc_buf(pool);
290 if (b == NULL) {
291 return NGX_ERROR;
292 }
293
294 ngx_memcpy(b, buf, sizeof(ngx_buf_t));
295
296 if (ngx_buf_in_memory(buf)) {
297 buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
298 b->last = buf->pos;
299 }
300
301 buf->file_pos = NGX_SENDFILE_LIMIT;
302 b->file_last = NGX_SENDFILE_LIMIT;
303
304 cl->buf = b;
305
306 } else {
307 cl->buf = buf;
308 in = in->next;
309 }
310
311 #else
312 cl->buf = in->buf;
313 in = in->next;
314
315 #endif
316
317 *ll = cl;
318 ll = &cl->next;
319 }
320
321 *ll = NULL;
322
323 return NGX_OK;
324 }
325
326
327 static ngx_int_t
328 ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
329 {
330 size_t size;
331 ngx_buf_t *in;
332
333 in = ctx->in->buf;
334
335 if (in->file == NULL || !in->file->directio) {
336 return NGX_DECLINED;
337 }
338
339 ctx->directio = 1;
340
341 size = (size_t) (in->file_pos - (in->file_pos & ~(NGX_DIRECTIO_BLOCK - 1)));
342
343 if (size == 0) {
344
345 if (bsize >= (off_t) ctx->bufs.size) {
346 return NGX_DECLINED;
347 }
348
349 size = (size_t) bsize;
350
351 } else {
352 size = NGX_DIRECTIO_BLOCK - size;
353
354 if ((off_t) size > bsize) {
355 size = (size_t) bsize;
356 }
357 }
358
359 ctx->buf = ngx_create_temp_buf(ctx->pool, size);
360 if (ctx->buf == NULL) {
361 return NGX_ERROR;
362 }
363
364 /*
365 * we do not set ctx->buf->tag, because we do not want
366 * to reuse the buf via ctx->free list
367 */
368
369 #if (NGX_HAVE_ALIGNED_DIRECTIO)
370 ctx->unaligned = 1;
371 #endif
372
373 return NGX_OK;
374 }
375
376
377 static ngx_int_t
378 ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
379 {
380 size_t size;
381 ngx_buf_t *b, *in;
382 ngx_uint_t recycled;
383
384 in = ctx->in->buf;
385 size = ctx->bufs.size;
386 recycled = 1;
387
388 if (in->last_in_chain) {
389
390 if (bsize < (off_t) size) {
391
392 /*
393 * allocate a small temp buf for a small last buf
394 * or its small last part
395 */
396
397 size = (size_t) bsize;
398 recycled = 0;
399
400 } else if (!ctx->directio
401 && ctx->bufs.num == 1
402 && (bsize < (off_t) (size + size / 4)))
403 {
404 /*
405 * allocate a temp buf that equals to a last buf,
406 * if there is no directio, the last buf size is lesser
407 * than 1.25 of bufs.size and the temp buf is single
408 */
409
410 size = (size_t) bsize;
411 recycled = 0;
412 }
413 }
414
415 b = ngx_calloc_buf(ctx->pool);
416 if (b == NULL) {
417 return NGX_ERROR;
418 }
419
420 if (ctx->directio) {
421
422 /*
423 * allocate block aligned to a disk sector size to enable
424 * userland buffer direct usage conjunctly with directio
425 */
426
427 b->start = ngx_pmemalign(ctx->pool, size, NGX_DIRECTIO_BLOCK);
428 if (b->start == NULL) {
429 return NGX_ERROR;
430 }
431
432 } else {
433 b->start = ngx_palloc(ctx->pool, size);
434 if (b->start == NULL) {
435 return NGX_ERROR;
436 }
437 }
438
439 b->pos = b->start;
440 b->last = b->start;
441 b->end = b->last + size;
442 b->temporary = 1;
443 b->tag = ctx->tag;
444 b->recycled = recycled;
445
446 ctx->buf = b;
447 ctx->allocated++;
448
449 return NGX_OK;
450 }
451
452
453 static ngx_int_t
454 ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
455 {
456 off_t size;
457 ssize_t n;
458 ngx_buf_t *src, *dst;
459 ngx_uint_t sendfile;
460
461 src = ctx->in->buf;
462 dst = ctx->buf;
463
464 size = ngx_buf_size(src);
465
466 if (size > dst->end - dst->pos) {
467 size = dst->end - dst->pos;
468 }
469
470 sendfile = ctx->sendfile & !ctx->directio;
471
472 #if (NGX_SENDFILE_LIMIT)
473
474 if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
475 sendfile = 0;
476 }
477
478 #endif
479
480 if (ngx_buf_in_memory(src)) {
481 ngx_memcpy(dst->pos, src->pos, (size_t) size);
482 src->pos += (size_t) size;
483 dst->last += (size_t) size;
484
485 if (src->in_file) {
486
487 if (sendfile) {
488 dst->in_file = 1;
489 dst->file = src->file;
490 dst->file_pos = src->file_pos;
491 dst->file_last = src->file_pos + size;
492
493 } else {
494 dst->in_file = 0;
495 }
496
497 src->file_pos += size;
498
499 } else {
500 dst->in_file = 0;
501 }
502
503 if (src->pos == src->last) {
504 dst->flush = src->flush;
505 dst->last_buf = src->last_buf;
506 dst->last_in_chain = src->last_in_chain;
507 }
508
509 } else {
510
511 #if (NGX_HAVE_ALIGNED_DIRECTIO)
512
513 if (ctx->unaligned) {
514 if (ngx_directio_off(src->file->fd) == -1) {
515 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
516 ngx_directio_off_n " \"%s\" failed",
517 src->file->name.data);
518 }
519 }
520
521 #endif
522
523 n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos);
524
525 #if (NGX_HAVE_ALIGNED_DIRECTIO)
526
527 if (ctx->unaligned) {
528 ngx_err_t err;
529
530 err = ngx_errno;
531
532 if (ngx_directio_on(src->file->fd) == -1) {
533 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
534 ngx_directio_on_n " \"%s\" failed",
535 src->file->name.data);
536 }
537
538 ngx_set_errno(err);
539
540 ctx->unaligned = 0;
541 }
542
543 #endif
544
545 if (n == NGX_ERROR) {
546 return (ngx_int_t) n;
547 }
548
549 #if (NGX_FILE_AIO_READ)
550 if (n == NGX_AGAIN) {
551 return (ngx_int_t) n;
552 }
553 #endif
554
555 if (n != size) {
556 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
557 ngx_read_file_n " read only %z of %O from \"%s\"",
558 n, size, src->file->name.data);
559 if (n == 0) {
560 return NGX_ERROR;
561 }
562 }
563
564 dst->last += n;
565
566 if (sendfile) {
567 dst->in_file = 1;
568 dst->file = src->file;
569 dst->file_pos = src->file_pos;
570 dst->file_last = src->file_pos + n;
571
572 } else {
573 dst->in_file = 0;
574 }
575
576 src->file_pos += n;
577
578 if (src->file_pos == src->file_last) {
579 dst->flush = src->flush;
580 dst->last_buf = src->last_buf;
581 dst->last_in_chain = src->last_in_chain;
582 }
583 }
584
585 return NGX_OK;
586 }
587
588
589 ngx_int_t
590 ngx_chain_writer(void *data, ngx_chain_t *in)
591 {
592 ngx_chain_writer_ctx_t *ctx = data;
593
594 off_t size;
595 ngx_chain_t *cl;
596 ngx_connection_t *c;
597
598 c = ctx->connection;
599
600 for (size = 0; in; in = in->next) {
601
602 #if 1
603 if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
604 ngx_debug_point();
605 }
606 #endif
607
608 size += ngx_buf_size(in->buf);
609
610 ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0,
611 "chain writer buf fl:%d s:%uO",
612 in->buf->flush, ngx_buf_size(in->buf));
613
614 cl = ngx_alloc_chain_link(ctx->pool);
615 if (cl == NULL) {
616 return NGX_ERROR;
617 }
618
619 cl->buf = in->buf;
620 cl->next = NULL;
621 *ctx->last = cl;
622 ctx->last = &cl->next;
623 }
624
625 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
626 "chain writer in: %p", ctx->out);
627
628 for (cl = ctx->out; cl; cl = cl->next) {
629
630 #if 1
631 if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
632 ngx_debug_point();
633 }
634
635 #endif
636
637 size += ngx_buf_size(cl->buf);
638 }
639
640 if (size == 0 && !c->buffered) {
641 return NGX_OK;
642 }
643
644 ctx->out = c->send_chain(c, ctx->out, ctx->limit);
645
646 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
647 "chain writer out: %p", ctx->out);
648
649 if (ctx->out == NGX_CHAIN_ERROR) {
650 return NGX_ERROR;
651 }
652
653 if (ctx->out == NULL) {
654 ctx->last = &ctx->out;
655
656 if (!c->buffered) {
657 return NGX_OK;
658 }
659 }
660
661 return NGX_AGAIN;
662 }
663
This page was automatically generated by the
LXR engine.
Visit the LXR main site for more
information.