1
2 /*
3 * Copyright (C) Igor Sysoev
4 */
5
6
7 #include <ngx_config.h>
8 #include <ngx_core.h>
9 #include <ngx_http.h>
10
11
12 static int ngx_http_upstream_cmp_servers(const void *one, const void *two);
13 static ngx_uint_t
14 ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers);
15
16
17 ngx_int_t
18 ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
19 ngx_http_upstream_srv_conf_t *us)
20 {
21 ngx_url_t u;
22 ngx_uint_t i, j, n;
23 ngx_http_upstream_server_t *server;
24 ngx_http_upstream_rr_peers_t *peers, *backup;
25
26 us->peer.init = ngx_http_upstream_init_round_robin_peer;
27
28 if (us->servers) {
29 server = us->servers->elts;
30
31 n = 0;
32
33 for (i = 0; i < us->servers->nelts; i++) {
34 if (server[i].backup) {
35 continue;
36 }
37
38 n += server[i].naddrs;
39 }
40
41 peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
42 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
43 if (peers == NULL) {
44 return NGX_ERROR;
45 }
46
47 peers->single = (n == 1);
48 peers->number = n;
49 peers->name = &us->host;
50
51 n = 0;
52
53 for (i = 0; i < us->servers->nelts; i++) {
54 for (j = 0; j < server[i].naddrs; j++) {
55 if (server[i].backup) {
56 continue;
57 }
58
59 peers->peer[n].sockaddr = server[i].addrs[j].sockaddr;
60 peers->peer[n].socklen = server[i].addrs[j].socklen;
61 peers->peer[n].name = server[i].addrs[j].name;
62 peers->peer[n].max_fails = server[i].max_fails;
63 peers->peer[n].fail_timeout = server[i].fail_timeout;
64 peers->peer[n].down = server[i].down;
65 peers->peer[n].weight = server[i].down ? 0 : server[i].weight;
66 peers->peer[n].current_weight = peers->peer[n].weight;
67 n++;
68 }
69 }
70
71 us->peer.data = peers;
72
73 ngx_sort(&peers->peer[0], (size_t) n,
74 sizeof(ngx_http_upstream_rr_peer_t),
75 ngx_http_upstream_cmp_servers);
76
77 /* backup servers */
78
79 n = 0;
80
81 for (i = 0; i < us->servers->nelts; i++) {
82 if (!server[i].backup) {
83 continue;
84 }
85
86 n += server[i].naddrs;
87 }
88
89 if (n == 0) {
90 return NGX_OK;
91 }
92
93 backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
94 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
95 if (backup == NULL) {
96 return NGX_ERROR;
97 }
98
99 peers->single = 0;
100 backup->single = 0;
101 backup->number = n;
102 backup->name = &us->host;
103
104 n = 0;
105
106 for (i = 0; i < us->servers->nelts; i++) {
107 for (j = 0; j < server[i].naddrs; j++) {
108 if (!server[i].backup) {
109 continue;
110 }
111
112 backup->peer[n].sockaddr = server[i].addrs[j].sockaddr;
113 backup->peer[n].socklen = server[i].addrs[j].socklen;
114 backup->peer[n].name = server[i].addrs[j].name;
115 backup->peer[n].weight = server[i].weight;
116 backup->peer[n].current_weight = server[i].weight;
117 backup->peer[n].max_fails = server[i].max_fails;
118 backup->peer[n].fail_timeout = server[i].fail_timeout;
119 backup->peer[n].down = server[i].down;
120 n++;
121 }
122 }
123
124 peers->next = backup;
125
126 ngx_sort(&backup->peer[0], (size_t) n,
127 sizeof(ngx_http_upstream_rr_peer_t),
128 ngx_http_upstream_cmp_servers);
129
130 return NGX_OK;
131 }
132
133
134 /* an upstream implicitly defined by proxy_pass, etc. */
135
136 if (us->port == 0 && us->default_port == 0) {
137 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
138 "no port in upstream \"%V\" in %s:%ui",
139 &us->host, us->file_name, us->line);
140 return NGX_ERROR;
141 }
142
143 ngx_memzero(&u, sizeof(ngx_url_t));
144
145 u.host = us->host;
146 u.port = (in_port_t) (us->port ? us->port : us->default_port);
147
148 if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
149 if (u.err) {
150 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
151 "%s in upstream \"%V\" in %s:%ui",
152 u.err, &us->host, us->file_name, us->line);
153 }
154
155 return NGX_ERROR;
156 }
157
158 n = u.naddrs;
159
160 peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
161 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
162 if (peers == NULL) {
163 return NGX_ERROR;
164 }
165
166 peers->single = (n == 1);
167 peers->number = n;
168 peers->name = &us->host;
169
170 for (i = 0; i < u.naddrs; i++) {
171 peers->peer[i].sockaddr = u.addrs[i].sockaddr;
172 peers->peer[i].socklen = u.addrs[i].socklen;
173 peers->peer[i].name = u.addrs[i].name;
174 peers->peer[i].weight = 1;
175 peers->peer[i].current_weight = 1;
176 peers->peer[i].max_fails = 1;
177 peers->peer[i].fail_timeout = 10;
178 }
179
180 us->peer.data = peers;
181
182 /* implicitly defined upstream has no backup servers */
183
184 return NGX_OK;
185 }
186
187
188 static int
189 ngx_http_upstream_cmp_servers(const void *one, const void *two)
190 {
191 ngx_http_upstream_rr_peer_t *first, *second;
192
193 first = (ngx_http_upstream_rr_peer_t *) one;
194 second = (ngx_http_upstream_rr_peer_t *) two;
195
196 return (first->weight < second->weight);
197 }
198
199
200 ngx_int_t
201 ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
202 ngx_http_upstream_srv_conf_t *us)
203 {
204 ngx_uint_t n;
205 ngx_http_upstream_rr_peer_data_t *rrp;
206
207 rrp = r->upstream->peer.data;
208
209 if (rrp == NULL) {
210 rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
211 if (rrp == NULL) {
212 return NGX_ERROR;
213 }
214
215 r->upstream->peer.data = rrp;
216 }
217
218 rrp->peers = us->peer.data;
219 rrp->current = 0;
220
221 if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
222 rrp->tried = &rrp->data;
223 rrp->data = 0;
224
225 } else {
226 n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
227 / (8 * sizeof(uintptr_t));
228
229 rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
230 if (rrp->tried == NULL) {
231 return NGX_ERROR;
232 }
233 }
234
235 r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
236 r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
237 r->upstream->peer.tries = rrp->peers->number;
238 #if (NGX_HTTP_SSL)
239 r->upstream->peer.set_session =
240 ngx_http_upstream_set_round_robin_peer_session;
241 r->upstream->peer.save_session =
242 ngx_http_upstream_save_round_robin_peer_session;
243 #endif
244
245 return NGX_OK;
246 }
247
248
249 ngx_int_t
250 ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
251 ngx_http_upstream_resolved_t *ur)
252 {
253 u_char *p;
254 size_t len;
255 ngx_uint_t i, n;
256 struct sockaddr_in *sin;
257 ngx_http_upstream_rr_peers_t *peers;
258 ngx_http_upstream_rr_peer_data_t *rrp;
259
260 rrp = r->upstream->peer.data;
261
262 if (rrp == NULL) {
263 rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
264 if (rrp == NULL) {
265 return NGX_ERROR;
266 }
267
268 r->upstream->peer.data = rrp;
269 }
270
271 peers = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_rr_peers_t)
272 + sizeof(ngx_http_upstream_rr_peer_t) * (ur->naddrs - 1));
273 if (peers == NULL) {
274 return NGX_ERROR;
275 }
276
277 peers->single = (ur->naddrs == 1);
278 peers->number = ur->naddrs;
279 peers->name = &ur->host;
280
281 for (i = 0; i < ur->naddrs; i++) {
282
283 len = INET_ADDRSTRLEN - 1 + 1 + sizeof(":65536") - 1;
284
285 p = ngx_palloc(r->pool, len);
286 if (p == NULL) {
287 return NGX_ERROR;
288 }
289
290 len = ngx_inet_ntop(AF_INET, &ur->addrs[i], p, INET_ADDRSTRLEN);
291 len = ngx_sprintf(&p[len], ":%d", ur->port) - p;
292
293 sin = ngx_pcalloc(r->pool, sizeof(struct sockaddr_in));
294 if (sin == NULL) {
295 return NGX_ERROR;
296 }
297
298 sin->sin_family = AF_INET;
299 sin->sin_port = htons(ur->port);
300 sin->sin_addr.s_addr = ur->addrs[i];
301
302 peers->peer[i].sockaddr = (struct sockaddr *) sin;
303 peers->peer[i].socklen = sizeof(struct sockaddr_in);
304 peers->peer[i].name.len = len;
305 peers->peer[i].name.data = p;
306 peers->peer[i].weight = 1;
307 peers->peer[i].current_weight = 1;
308 peers->peer[i].max_fails = 1;
309 peers->peer[i].fail_timeout = 10;
310 }
311
312 rrp->peers = peers;
313 rrp->current = 0;
314
315 if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
316 rrp->tried = &rrp->data;
317 rrp->data = 0;
318
319 } else {
320 n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
321 / (8 * sizeof(uintptr_t));
322
323 rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
324 if (rrp->tried == NULL) {
325 return NGX_ERROR;
326 }
327 }
328
329 r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
330 r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
331 r->upstream->peer.tries = rrp->peers->number;
332 #if (NGX_HTTP_SSL)
333 r->upstream->peer.set_session =
334 ngx_http_upstream_set_round_robin_peer_session;
335 r->upstream->peer.save_session =
336 ngx_http_upstream_save_round_robin_peer_session;
337 #endif
338
339 return NGX_OK;
340 }
341
342
343 ngx_int_t
344 ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
345 {
346 ngx_http_upstream_rr_peer_data_t *rrp = data;
347
348 time_t now;
349 uintptr_t m;
350 ngx_int_t rc;
351 ngx_uint_t i, n;
352 ngx_connection_t *c;
353 ngx_http_upstream_rr_peer_t *peer;
354 ngx_http_upstream_rr_peers_t *peers;
355
356 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
357 "get rr peer, try: %ui", pc->tries);
358
359 now = ngx_time();
360
361 /* ngx_lock_mutex(rrp->peers->mutex); */
362
363 if (rrp->peers->last_cached) {
364
365 /* cached connection */
366
367 c = rrp->peers->cached[rrp->peers->last_cached];
368 rrp->peers->last_cached--;
369
370 /* ngx_unlock_mutex(ppr->peers->mutex); */
371
372 #if (NGX_THREADS)
373 c->read->lock = c->read->own_lock;
374 c->write->lock = c->write->own_lock;
375 #endif
376
377 pc->connection = c;
378 pc->cached = 1;
379
380 return NGX_OK;
381 }
382
383 pc->cached = 0;
384 pc->connection = NULL;
385
386 if (rrp->peers->single) {
387 peer = &rrp->peers->peer[0];
388
389 } else {
390
391 /* there are several peers */
392
393 if (pc->tries == rrp->peers->number) {
394
395 /* it's a first try - get a current peer */
396
397 i = pc->tries;
398
399 for ( ;; ) {
400 rrp->current = ngx_http_upstream_get_peer(rrp->peers);
401
402 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
403 "get rr peer, current: %ui %i",
404 rrp->current,
405 rrp->peers->peer[rrp->current].current_weight);
406
407 n = rrp->current / (8 * sizeof(uintptr_t));
408 m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
409
410 if (!(rrp->tried[n] & m)) {
411 peer = &rrp->peers->peer[rrp->current];
412
413 if (!peer->down) {
414
415 if (peer->max_fails == 0
416 || peer->fails < peer->max_fails)
417 {
418 break;
419 }
420
421 if (now - peer->accessed > peer->fail_timeout) {
422 peer->fails = 0;
423 break;
424 }
425
426 peer->current_weight = 0;
427
428 } else {
429 rrp->tried[n] |= m;
430 }
431
432 pc->tries--;
433 }
434
435 if (pc->tries == 0) {
436 goto failed;
437 }
438
439 if (--i == 0) {
440 ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
441 "round robin upstream stuck on %ui tries",
442 pc->tries);
443 goto failed;
444 }
445 }
446
447 peer->current_weight--;
448
449 } else {
450
451 i = pc->tries;
452
453 for ( ;; ) {
454 n = rrp->current / (8 * sizeof(uintptr_t));
455 m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
456
457 if (!(rrp->tried[n] & m)) {
458
459 peer = &rrp->peers->peer[rrp->current];
460
461 if (!peer->down) {
462
463 if (peer->max_fails == 0
464 || peer->fails < peer->max_fails)
465 {
466 break;
467 }
468
469 if (now - peer->accessed > peer->fail_timeout) {
470 peer->fails = 0;
471 break;
472 }
473
474 peer->current_weight = 0;
475
476 } else {
477 rrp->tried[n] |= m;
478 }
479
480 pc->tries--;
481 }
482
483 rrp->current++;
484
485 if (rrp->current >= rrp->peers->number) {
486 rrp->current = 0;
487 }
488
489 if (pc->tries == 0) {
490 goto failed;
491 }
492
493 if (--i == 0) {
494 ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
495 "round robin upstream stuck on %ui tries",
496 pc->tries);
497 goto failed;
498 }
499 }
500
501 peer->current_weight--;
502 }
503
504 rrp->tried[n] |= m;
505 }
506
507 pc->sockaddr = peer->sockaddr;
508 pc->socklen = peer->socklen;
509 pc->name = &peer->name;
510
511 /* ngx_unlock_mutex(rrp->peers->mutex); */
512
513 if (pc->tries == 1 && rrp->peers->next) {
514 pc->tries += rrp->peers->next->number;
515
516 n = rrp->peers->next->number / (8 * sizeof(uintptr_t)) + 1;
517 for (i = 0; i < n; i++) {
518 rrp->tried[i] = 0;
519 }
520 }
521
522 return NGX_OK;
523
524 failed:
525
526 peers = rrp->peers;
527
528 if (peers->next) {
529
530 /* ngx_unlock_mutex(peers->mutex); */
531
532 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");
533
534 rrp->peers = peers->next;
535 pc->tries = rrp->peers->number;
536
537 n = rrp->peers->number / (8 * sizeof(uintptr_t)) + 1;
538 for (i = 0; i < n; i++) {
539 rrp->tried[i] = 0;
540 }
541
542 rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);
543
544 if (rc != NGX_BUSY) {
545 return rc;
546 }
547
548 /* ngx_lock_mutex(peers->mutex); */
549 }
550
551 /* all peers failed, mark them as live for quick recovery */
552
553 for (i = 0; i < peers->number; i++) {
554 peers->peer[i].fails = 0;
555 }
556
557 /* ngx_unlock_mutex(peers->mutex); */
558
559 pc->name = peers->name;
560
561 return NGX_BUSY;
562 }
563
564
565 static ngx_uint_t
566 ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers)
567 {
568 ngx_uint_t i, n;
569 ngx_http_upstream_rr_peer_t *peer;
570
571 peer = &peers->peer[0];
572
573 for ( ;; ) {
574
575 for (i = 0; i < peers->number; i++) {
576
577 if (peer[i].current_weight <= 0) {
578 continue;
579 }
580
581 n = i;
582
583 while (i < peers->number - 1) {
584
585 i++;
586
587 if (peer[i].current_weight <= 0) {
588 continue;
589 }
590
591 if (peer[n].current_weight * 1000 / peer[i].current_weight
592 > peer[n].weight * 1000 / peer[i].weight)
593 {
594 return n;
595 }
596
597 n = i;
598 }
599
600 if (peer[i].current_weight > 0) {
601 n = i;
602 }
603
604 return n;
605 }
606
607 for (i = 0; i < peers->number; i++) {
608 peer[i].current_weight = peer[i].weight;
609 }
610 }
611 }
612
613
614 void
615 ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
616 ngx_uint_t state)
617 {
618 ngx_http_upstream_rr_peer_data_t *rrp = data;
619
620 time_t now;
621 ngx_http_upstream_rr_peer_t *peer;
622
623 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
624 "free rr peer %ui %ui", pc->tries, state);
625
626 if (state == 0 && pc->tries == 0) {
627 return;
628 }
629
630 /* TODO: NGX_PEER_KEEPALIVE */
631
632 if (rrp->peers->single) {
633 pc->tries = 0;
634 return;
635 }
636
637 if (state & NGX_PEER_FAILED) {
638 now = ngx_time();
639
640 peer = &rrp->peers->peer[rrp->current];
641
642 /* ngx_lock_mutex(rrp->peers->mutex); */
643
644 peer->fails++;
645 peer->accessed = now;
646
647 peer->current_weight -= peer->weight / peer->max_fails;
648
649 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
650 "free rr peer failed: %ui %i",
651 rrp->current, peer->current_weight);
652
653 if (peer->current_weight < 0) {
654 peer->current_weight = 0;
655 }
656
657 /* ngx_unlock_mutex(rrp->peers->mutex); */
658 }
659
660 rrp->current++;
661
662 if (rrp->current >= rrp->peers->number) {
663 rrp->current = 0;
664 }
665
666 if (pc->tries) {
667 pc->tries--;
668 }
669
670 /* ngx_unlock_mutex(rrp->peers->mutex); */
671 }
672
673
674 #if (NGX_HTTP_SSL)
675
676 ngx_int_t
677 ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
678 void *data)
679 {
680 ngx_http_upstream_rr_peer_data_t *rrp = data;
681
682 ngx_int_t rc;
683 ngx_ssl_session_t *ssl_session;
684 ngx_http_upstream_rr_peer_t *peer;
685
686 peer = &rrp->peers->peer[rrp->current];
687
688 /* TODO: threads only mutex */
689 /* ngx_lock_mutex(rrp->peers->mutex); */
690
691 ssl_session = peer->ssl_session;
692
693 rc = ngx_ssl_set_session(pc->connection, ssl_session);
694
695 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
696 "set session: %p:%d",
697 ssl_session, ssl_session ? ssl_session->references : 0);
698
699 /* ngx_unlock_mutex(rrp->peers->mutex); */
700
701 return rc;
702 }
703
704
705 void
706 ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
707 void *data)
708 {
709 ngx_http_upstream_rr_peer_data_t *rrp = data;
710
711 ngx_ssl_session_t *old_ssl_session, *ssl_session;
712 ngx_http_upstream_rr_peer_t *peer;
713
714 ssl_session = ngx_ssl_get_session(pc->connection);
715
716 if (ssl_session == NULL) {
717 return;
718 }
719
720 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
721 "save session: %p:%d", ssl_session, ssl_session->references);
722
723 peer = &rrp->peers->peer[rrp->current];
724
725 /* TODO: threads only mutex */
726 /* ngx_lock_mutex(rrp->peers->mutex); */
727
728 old_ssl_session = peer->ssl_session;
729 peer->ssl_session = ssl_session;
730
731 /* ngx_unlock_mutex(rrp->peers->mutex); */
732
733 if (old_ssl_session) {
734
735 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
736 "old session: %p:%d",
737 old_ssl_session, old_ssl_session->references);
738
739 /* TODO: may block */
740
741 ngx_ssl_free_session(old_ssl_session);
742 }
743 }
744
745 #endif
746
This page was automatically generated by the
LXR engine.
Visit the LXR main site for more
information.