1
2 /*
3 * Copyright (C) Igor Sysoev
4 */
5
6
7 #include <ngx_config.h>
8 #include <ngx_core.h>
9 #include <ngx_http.h>
10
11
12 static ngx_int_t ngx_http_upstream_cmp_servers(const void *one,
13 const void *two);
14 static ngx_uint_t
15 ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers);
16
17
18 ngx_int_t
19 ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
20 ngx_http_upstream_srv_conf_t *us)
21 {
22 ngx_url_t u;
23 ngx_uint_t i, j, n;
24 ngx_http_upstream_server_t *server;
25 ngx_http_upstream_rr_peers_t *peers, *backup;
26
27 us->peer.init = ngx_http_upstream_init_round_robin_peer;
28
29 if (us->servers) {
30 server = us->servers->elts;
31
32 n = 0;
33
34 for (i = 0; i < us->servers->nelts; i++) {
35 if (server[i].backup) {
36 continue;
37 }
38
39 n += server[i].naddrs;
40 }
41
42 peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
43 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
44 if (peers == NULL) {
45 return NGX_ERROR;
46 }
47
48 peers->single = (n == 1);
49 peers->number = n;
50 peers->name = &us->host;
51
52 n = 0;
53
54 for (i = 0; i < us->servers->nelts; i++) {
55 for (j = 0; j < server[i].naddrs; j++) {
56 if (server[i].backup) {
57 continue;
58 }
59
60 peers->peer[n].sockaddr = server[i].addrs[j].sockaddr;
61 peers->peer[n].socklen = server[i].addrs[j].socklen;
62 peers->peer[n].name = server[i].addrs[j].name;
63 peers->peer[n].max_fails = server[i].max_fails;
64 peers->peer[n].fail_timeout = server[i].fail_timeout;
65 peers->peer[n].down = server[i].down;
66 peers->peer[n].weight = server[i].down ? 0 : server[i].weight;
67 peers->peer[n].current_weight = peers->peer[n].weight;
68 n++;
69 }
70 }
71
72 us->peer.data = peers;
73
74 ngx_sort(&peers->peer[0], (size_t) n,
75 sizeof(ngx_http_upstream_rr_peer_t),
76 ngx_http_upstream_cmp_servers);
77
78 /* backup servers */
79
80 n = 0;
81
82 for (i = 0; i < us->servers->nelts; i++) {
83 if (!server[i].backup) {
84 continue;
85 }
86
87 n += server[i].naddrs;
88 }
89
90 if (n == 0) {
91 return NGX_OK;
92 }
93
94 backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
95 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
96 if (backup == NULL) {
97 return NGX_ERROR;
98 }
99
100 peers->single = 0;
101 backup->single = 0;
102 backup->number = n;
103 backup->name = &us->host;
104
105 n = 0;
106
107 for (i = 0; i < us->servers->nelts; i++) {
108 for (j = 0; j < server[i].naddrs; j++) {
109 if (!server[i].backup) {
110 continue;
111 }
112
113 backup->peer[n].sockaddr = server[i].addrs[j].sockaddr;
114 backup->peer[n].socklen = server[i].addrs[j].socklen;
115 backup->peer[n].name = server[i].addrs[j].name;
116 backup->peer[n].weight = server[i].weight;
117 backup->peer[n].current_weight = server[i].weight;
118 backup->peer[n].max_fails = server[i].max_fails;
119 backup->peer[n].fail_timeout = server[i].fail_timeout;
120 backup->peer[n].down = server[i].down;
121 n++;
122 }
123 }
124
125 peers->next = backup;
126
127 ngx_sort(&backup->peer[0], (size_t) n,
128 sizeof(ngx_http_upstream_rr_peer_t),
129 ngx_http_upstream_cmp_servers);
130
131 return NGX_OK;
132 }
133
134
135 /* an upstream implicitly defined by proxy_pass, etc. */
136
137 if (us->port == 0 && us->default_port == 0) {
138 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
139 "no port in upstream \"%V\" in %s:%ui",
140 &us->host, us->file_name, us->line);
141 return NGX_ERROR;
142 }
143
144 ngx_memzero(&u, sizeof(ngx_url_t));
145
146 u.host = us->host;
147 u.port = (in_port_t) (us->port ? us->port : us->default_port);
148
149 if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
150 if (u.err) {
151 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
152 "%s in upstream \"%V\" in %s:%ui",
153 u.err, &us->host, us->file_name, us->line);
154 }
155
156 return NGX_ERROR;
157 }
158
159 n = u.naddrs;
160
161 peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
162 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
163 if (peers == NULL) {
164 return NGX_ERROR;
165 }
166
167 peers->single = (n == 1);
168 peers->number = n;
169 peers->name = &us->host;
170
171 for (i = 0; i < u.naddrs; i++) {
172 peers->peer[i].sockaddr = u.addrs[i].sockaddr;
173 peers->peer[i].socklen = u.addrs[i].socklen;
174 peers->peer[i].name = u.addrs[i].name;
175 peers->peer[i].weight = 1;
176 peers->peer[i].current_weight = 1;
177 peers->peer[i].max_fails = 1;
178 peers->peer[i].fail_timeout = 10;
179 }
180
181 us->peer.data = peers;
182
183 /* implicitly defined upstream has no backup servers */
184
185 return NGX_OK;
186 }
187
188
189 static ngx_int_t
190 ngx_http_upstream_cmp_servers(const void *one, const void *two)
191 {
192 ngx_http_upstream_rr_peer_t *first, *second;
193
194 first = (ngx_http_upstream_rr_peer_t *) one;
195 second = (ngx_http_upstream_rr_peer_t *) two;
196
197 return (first->weight < second->weight);
198 }
199
200
201 ngx_int_t
202 ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
203 ngx_http_upstream_srv_conf_t *us)
204 {
205 ngx_uint_t n;
206 ngx_http_upstream_rr_peer_data_t *rrp;
207
208 rrp = r->upstream->peer.data;
209
210 if (rrp == NULL) {
211 rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
212 if (rrp == NULL) {
213 return NGX_ERROR;
214 }
215
216 r->upstream->peer.data = rrp;
217 }
218
219 rrp->peers = us->peer.data;
220 rrp->current = 0;
221
222 if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
223 rrp->tried = &rrp->data;
224 rrp->data = 0;
225
226 } else {
227 n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
228 / (8 * sizeof(uintptr_t));
229
230 rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
231 if (rrp->tried == NULL) {
232 return NGX_ERROR;
233 }
234 }
235
236 r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
237 r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
238 r->upstream->peer.tries = rrp->peers->number;
239 #if (NGX_HTTP_SSL)
240 r->upstream->peer.set_session =
241 ngx_http_upstream_set_round_robin_peer_session;
242 r->upstream->peer.save_session =
243 ngx_http_upstream_save_round_robin_peer_session;
244 #endif
245
246 return NGX_OK;
247 }
248
249
250 ngx_int_t
251 ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
252 ngx_http_upstream_resolved_t *ur)
253 {
254 u_char *p;
255 size_t len;
256 ngx_uint_t i, n;
257 struct sockaddr_in *sin;
258 ngx_http_upstream_rr_peers_t *peers;
259 ngx_http_upstream_rr_peer_data_t *rrp;
260
261 rrp = r->upstream->peer.data;
262
263 if (rrp == NULL) {
264 rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
265 if (rrp == NULL) {
266 return NGX_ERROR;
267 }
268
269 r->upstream->peer.data = rrp;
270 }
271
272 peers = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_rr_peers_t)
273 + sizeof(ngx_http_upstream_rr_peer_t) * (ur->naddrs - 1));
274 if (peers == NULL) {
275 return NGX_ERROR;
276 }
277
278 peers->single = (ur->naddrs == 1);
279 peers->number = ur->naddrs;
280 peers->name = &ur->host;
281
282 if (ur->sockaddr) {
283 peers->peer[0].sockaddr = ur->sockaddr;
284 peers->peer[0].socklen = ur->socklen;
285 peers->peer[0].name = ur->host;
286 peers->peer[0].weight = 1;
287 peers->peer[0].current_weight = 1;
288 peers->peer[0].max_fails = 1;
289 peers->peer[0].fail_timeout = 10;
290
291 } else {
292
293 for (i = 0; i < ur->naddrs; i++) {
294
295 len = NGX_INET_ADDRSTRLEN + sizeof(":65536") - 1;
296
297 p = ngx_pnalloc(r->pool, len);
298 if (p == NULL) {
299 return NGX_ERROR;
300 }
301
302 len = ngx_inet_ntop(AF_INET, &ur->addrs[i], p, NGX_INET_ADDRSTRLEN);
303 len = ngx_sprintf(&p[len], ":%d", ur->port) - p;
304
305 sin = ngx_pcalloc(r->pool, sizeof(struct sockaddr_in));
306 if (sin == NULL) {
307 return NGX_ERROR;
308 }
309
310 sin->sin_family = AF_INET;
311 sin->sin_port = htons(ur->port);
312 sin->sin_addr.s_addr = ur->addrs[i];
313
314 peers->peer[i].sockaddr = (struct sockaddr *) sin;
315 peers->peer[i].socklen = sizeof(struct sockaddr_in);
316 peers->peer[i].name.len = len;
317 peers->peer[i].name.data = p;
318 peers->peer[i].weight = 1;
319 peers->peer[i].current_weight = 1;
320 peers->peer[i].max_fails = 1;
321 peers->peer[i].fail_timeout = 10;
322 }
323 }
324
325 rrp->peers = peers;
326 rrp->current = 0;
327
328 if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
329 rrp->tried = &rrp->data;
330 rrp->data = 0;
331
332 } else {
333 n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
334 / (8 * sizeof(uintptr_t));
335
336 rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
337 if (rrp->tried == NULL) {
338 return NGX_ERROR;
339 }
340 }
341
342 r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
343 r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
344 r->upstream->peer.tries = rrp->peers->number;
345 #if (NGX_HTTP_SSL)
346 r->upstream->peer.set_session =
347 ngx_http_upstream_set_round_robin_peer_session;
348 r->upstream->peer.save_session =
349 ngx_http_upstream_save_round_robin_peer_session;
350 #endif
351
352 return NGX_OK;
353 }
354
355
356 ngx_int_t
357 ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
358 {
359 ngx_http_upstream_rr_peer_data_t *rrp = data;
360
361 time_t now;
362 uintptr_t m;
363 ngx_int_t rc;
364 ngx_uint_t i, n;
365 ngx_connection_t *c;
366 ngx_http_upstream_rr_peer_t *peer;
367 ngx_http_upstream_rr_peers_t *peers;
368
369 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
370 "get rr peer, try: %ui", pc->tries);
371
372 now = ngx_time();
373
374 /* ngx_lock_mutex(rrp->peers->mutex); */
375
376 if (rrp->peers->last_cached) {
377
378 /* cached connection */
379
380 c = rrp->peers->cached[rrp->peers->last_cached];
381 rrp->peers->last_cached--;
382
383 /* ngx_unlock_mutex(ppr->peers->mutex); */
384
385 #if (NGX_THREADS)
386 c->read->lock = c->read->own_lock;
387 c->write->lock = c->write->own_lock;
388 #endif
389
390 pc->connection = c;
391 pc->cached = 1;
392
393 return NGX_OK;
394 }
395
396 pc->cached = 0;
397 pc->connection = NULL;
398
399 if (rrp->peers->single) {
400 peer = &rrp->peers->peer[0];
401
402 } else {
403
404 /* there are several peers */
405
406 if (pc->tries == rrp->peers->number) {
407
408 /* it's a first try - get a current peer */
409
410 i = pc->tries;
411
412 for ( ;; ) {
413 rrp->current = ngx_http_upstream_get_peer(rrp->peers);
414
415 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
416 "get rr peer, current: %ui %i",
417 rrp->current,
418 rrp->peers->peer[rrp->current].current_weight);
419
420 n = rrp->current / (8 * sizeof(uintptr_t));
421 m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
422
423 if (!(rrp->tried[n] & m)) {
424 peer = &rrp->peers->peer[rrp->current];
425
426 if (!peer->down) {
427
428 if (peer->max_fails == 0
429 || peer->fails < peer->max_fails)
430 {
431 break;
432 }
433
434 if (now - peer->accessed > peer->fail_timeout) {
435 peer->fails = 0;
436 break;
437 }
438
439 peer->current_weight = 0;
440
441 } else {
442 rrp->tried[n] |= m;
443 }
444
445 pc->tries--;
446 }
447
448 if (pc->tries == 0) {
449 goto failed;
450 }
451
452 if (--i == 0) {
453 ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
454 "round robin upstream stuck on %ui tries",
455 pc->tries);
456 goto failed;
457 }
458 }
459
460 peer->current_weight--;
461
462 } else {
463
464 i = pc->tries;
465
466 for ( ;; ) {
467 n = rrp->current / (8 * sizeof(uintptr_t));
468 m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
469
470 if (!(rrp->tried[n] & m)) {
471
472 peer = &rrp->peers->peer[rrp->current];
473
474 if (!peer->down) {
475
476 if (peer->max_fails == 0
477 || peer->fails < peer->max_fails)
478 {
479 break;
480 }
481
482 if (now - peer->accessed > peer->fail_timeout) {
483 peer->fails = 0;
484 break;
485 }
486
487 peer->current_weight = 0;
488
489 } else {
490 rrp->tried[n] |= m;
491 }
492
493 pc->tries--;
494 }
495
496 rrp->current++;
497
498 if (rrp->current >= rrp->peers->number) {
499 rrp->current = 0;
500 }
501
502 if (pc->tries == 0) {
503 goto failed;
504 }
505
506 if (--i == 0) {
507 ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
508 "round robin upstream stuck on %ui tries",
509 pc->tries);
510 goto failed;
511 }
512 }
513
514 peer->current_weight--;
515 }
516
517 rrp->tried[n] |= m;
518 }
519
520 pc->sockaddr = peer->sockaddr;
521 pc->socklen = peer->socklen;
522 pc->name = &peer->name;
523
524 /* ngx_unlock_mutex(rrp->peers->mutex); */
525
526 if (pc->tries == 1 && rrp->peers->next) {
527 pc->tries += rrp->peers->next->number;
528
529 n = rrp->peers->next->number / (8 * sizeof(uintptr_t)) + 1;
530 for (i = 0; i < n; i++) {
531 rrp->tried[i] = 0;
532 }
533 }
534
535 return NGX_OK;
536
537 failed:
538
539 peers = rrp->peers;
540
541 if (peers->next) {
542
543 /* ngx_unlock_mutex(peers->mutex); */
544
545 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");
546
547 rrp->peers = peers->next;
548 pc->tries = rrp->peers->number;
549
550 n = rrp->peers->number / (8 * sizeof(uintptr_t)) + 1;
551 for (i = 0; i < n; i++) {
552 rrp->tried[i] = 0;
553 }
554
555 rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);
556
557 if (rc != NGX_BUSY) {
558 return rc;
559 }
560
561 /* ngx_lock_mutex(peers->mutex); */
562 }
563
564 /* all peers failed, mark them as live for quick recovery */
565
566 for (i = 0; i < peers->number; i++) {
567 peers->peer[i].fails = 0;
568 }
569
570 /* ngx_unlock_mutex(peers->mutex); */
571
572 pc->name = peers->name;
573
574 return NGX_BUSY;
575 }
576
577
578 static ngx_uint_t
579 ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers)
580 {
581 ngx_uint_t i, n;
582 ngx_http_upstream_rr_peer_t *peer;
583
584 peer = &peers->peer[0];
585
586 for ( ;; ) {
587
588 for (i = 0; i < peers->number; i++) {
589
590 if (peer[i].current_weight <= 0) {
591 continue;
592 }
593
594 n = i;
595
596 while (i < peers->number - 1) {
597
598 i++;
599
600 if (peer[i].current_weight <= 0) {
601 continue;
602 }
603
604 if (peer[n].current_weight * 1000 / peer[i].current_weight
605 > peer[n].weight * 1000 / peer[i].weight)
606 {
607 return n;
608 }
609
610 n = i;
611 }
612
613 if (peer[i].current_weight > 0) {
614 n = i;
615 }
616
617 return n;
618 }
619
620 for (i = 0; i < peers->number; i++) {
621 peer[i].current_weight = peer[i].weight;
622 }
623 }
624 }
625
626
627 void
628 ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
629 ngx_uint_t state)
630 {
631 ngx_http_upstream_rr_peer_data_t *rrp = data;
632
633 time_t now;
634 ngx_http_upstream_rr_peer_t *peer;
635
636 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
637 "free rr peer %ui %ui", pc->tries, state);
638
639 if (state == 0 && pc->tries == 0) {
640 return;
641 }
642
643 /* TODO: NGX_PEER_KEEPALIVE */
644
645 if (rrp->peers->single) {
646 pc->tries = 0;
647 return;
648 }
649
650 if (state & NGX_PEER_FAILED) {
651 now = ngx_time();
652
653 peer = &rrp->peers->peer[rrp->current];
654
655 /* ngx_lock_mutex(rrp->peers->mutex); */
656
657 peer->fails++;
658 peer->accessed = now;
659
660 if (peer->max_fails) {
661 peer->current_weight -= peer->weight / peer->max_fails;
662 }
663
664 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
665 "free rr peer failed: %ui %i",
666 rrp->current, peer->current_weight);
667
668 if (peer->current_weight < 0) {
669 peer->current_weight = 0;
670 }
671
672 /* ngx_unlock_mutex(rrp->peers->mutex); */
673 }
674
675 rrp->current++;
676
677 if (rrp->current >= rrp->peers->number) {
678 rrp->current = 0;
679 }
680
681 if (pc->tries) {
682 pc->tries--;
683 }
684
685 /* ngx_unlock_mutex(rrp->peers->mutex); */
686 }
687
688
689 #if (NGX_HTTP_SSL)
690
691 ngx_int_t
692 ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
693 void *data)
694 {
695 ngx_http_upstream_rr_peer_data_t *rrp = data;
696
697 ngx_int_t rc;
698 ngx_ssl_session_t *ssl_session;
699 ngx_http_upstream_rr_peer_t *peer;
700
701 peer = &rrp->peers->peer[rrp->current];
702
703 /* TODO: threads only mutex */
704 /* ngx_lock_mutex(rrp->peers->mutex); */
705
706 ssl_session = peer->ssl_session;
707
708 rc = ngx_ssl_set_session(pc->connection, ssl_session);
709
710 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
711 "set session: %p:%d",
712 ssl_session, ssl_session ? ssl_session->references : 0);
713
714 /* ngx_unlock_mutex(rrp->peers->mutex); */
715
716 return rc;
717 }
718
719
720 void
721 ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
722 void *data)
723 {
724 ngx_http_upstream_rr_peer_data_t *rrp = data;
725
726 ngx_ssl_session_t *old_ssl_session, *ssl_session;
727 ngx_http_upstream_rr_peer_t *peer;
728
729 ssl_session = ngx_ssl_get_session(pc->connection);
730
731 if (ssl_session == NULL) {
732 return;
733 }
734
735 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
736 "save session: %p:%d", ssl_session, ssl_session->references);
737
738 peer = &rrp->peers->peer[rrp->current];
739
740 /* TODO: threads only mutex */
741 /* ngx_lock_mutex(rrp->peers->mutex); */
742
743 old_ssl_session = peer->ssl_session;
744 peer->ssl_session = ssl_session;
745
746 /* ngx_unlock_mutex(rrp->peers->mutex); */
747
748 if (old_ssl_session) {
749
750 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
751 "old session: %p:%d",
752 old_ssl_session, old_ssl_session->references);
753
754 /* TODO: may block */
755
756 ngx_ssl_free_session(old_ssl_session);
757 }
758 }
759
760 #endif
761
This page was automatically generated by the
LXR engine.
Visit the LXR main site for more
information.