深入理解 nginx和nginx-lua
2019-01-09 本文已影响61人
磊哥jinleileiking
最近一段时间，公司nginxlua遇到了各种奇怪问题，于是又读了一遍陶辉的书。
主要关注upstream 长连接和 nginx lua的cosocket方面
subrequest和upstream的区别
在开发nginx module时，我们最有可能遇到的一件事就是，在处理一个请求时，我们需要访问其他多个backend server网络资源，拉取到结果后分析整理成一个response，再发给用户。这个过程是无法使用nginx upstream机制的，因为upstream被设计为用来支持nginx reverse proxy功能，所以呢，upstream默认是把其他server的http response body全部返回给client。这与我们的要求不符，这个时候，我们可以考虑subrequest了，nginx http模块提供的这个功能能够帮我们搞定它。
upstream
keepalive
keepalive 指定的 数值 是 Nginx 每个 worker 连接后端的最大长连接数，而不是整个 Nginx 的。 而且这里的后端指的是「所有的后端」，而不是每一个后端(已验证)。
https://skyao.gitbooks.io/learning-nginx/content/documentation/keep_alive.html
buffering
Syntax: proxy_buffering on | off;
Default: proxy_buffering on;
http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering
ngx_http_proxy_pass
clcf->handler = ngx_http_proxy_handler;
content阶段执行ngx_http_proxy_handler,此函数类似test
ngx_http_proxy_handler
891 u->create_request = ngx_http_proxy_create_request;¬
| 892 u->reinit_request = ngx_http_proxy_reinit_request;¬
| 893 u->process_header = ngx_http_proxy_process_status_line;¬
| 894 u->abort_request = ngx_http_proxy_abort_request;¬
| 895 u->finalize_request = ngx_http_proxy_finalize_request;¬
rc = ngx_http_read_client_request_body(r,ngx_http_upstream_init);
145 if(rb->rest == 0){¬
| 146 /* the whole request body was pre-read */¬
| 147 r->request_body_no_buffering = 0;¬
| 148 post_handler(r);¬
| 149 return NGX_OK;¬
| 150 }¬
ngx_http_upstream_init
-> ngx_http_upstream_init_request
-> if(u->create_request(r)!= NGX_OK)
取上游地址
ngx_http_upstream_connect ** 连接上游 **
| 1538 c->write->handler = ngx_http_upstream_handler;¬
| 1539 c->read->handler = ngx_http_upstream_handler;¬
| 1540 ¬
| 1541 u->write_event_handler = ngx_http_upstream_send_request_handler;¬
| 1542 u->read_event_handler = ngx_http_upstream_process_header;¬
ngx_http_upstream_send_request ** 发送下游header+body给上游**
ngx_http_upstream_process_header ** 接收上游header **
2338 if (!r->subrequest_in_memory) {¬
| 2339 ngx_http_upstream_send_response(r, u); !!!!!!!!!!!!!!!!!!!!!!
| 2340 return;¬
| 2341 }¬
u->read_event_handler = ngx_http_upstream_process_body_in_memory;
ngx_http_upstream_process_body_in_memory ** 接收上游body **
发送上游header给下游
ngx_http_upstream_send_response
ngx_http_send_header
ngx_http_top_header_filter ( ngx_http_headers_module)
| 2930 u->read_event_handler = ngx_http_upstream_process_non_buffered_upstream;¬
| 2931 r->write_event_handler =¬
| 2932 ngx_http_upstream_process_non_buffered_downstream;¬
接收上游body
ngx_http_upstream_process_non_buffered_upstream
ngx_http_upstream_process_non_buffered_request
返回上游body 给下游
ngx_http_upstream_process_non_buffered_downstream
ngx_http_upstream_process_non_buffered_request
Q: 为何nginx无法透传上游返回的header？
A: 因为在分析上游headers时，将header放在了headers_in，目前没有module可以直接返回所有的上游header
ngx_http_upstream_send_request_handler
ngx_http_upstream_send_request
-> ngx_http_upstream_reinit
ngx_http_upstream_process_header
-> | 2300 rc = u->process_header(r);¬
#0 ngx_http_upstream_init(r=0x2038cb0)at src/http/ngx_http_upstream.c:503
#1 0x00000000004697d7 in ngx_http_read_client_request_body(r=0x2038cb0,
post_handler=0x4759c0 )at src/http/ngx_http_request_body.c:148
#2 0x000000000049811d in ngx_http_proxy_handler(r=0x2038cb0)at src/http/modules/ngx_http_proxy_module.c:930
#3 0x000000000045b7f4 in ngx_http_core_content_phase(r=0x2038cb0,ph=0x204e210)
at src/http/ngx_http_core_module.c:1173
#4 0x000000000045582d in ngx_http_core_run_phases(r=0x2038cb0)at src/http/ngx_http_core_module.c:862
#5 0x0000000000460fd8 in ngx_http_process_request(r=0x2038cb0)at src/http/ngx_http_request.c:1950
#6 0x0000000000461e96 in ngx_http_process_request_line(rev=0x2054670)at src/http/ngx_http_request.c:1049
#7 0x000000000044a381 in ngx_epoll_process_events(cycle=0x202a460,timer=,
flags=)at src/event/modules/ngx_epoll_module.c:902
#8 0x000000000043fb4a in ngx_process_events_and_timers(cycle=0x202a460)at src/event/ngx_event.c:252
#9 0x0000000000447f65 in ngx_worker_process_cycle(cycle=0x202a460,data=)
at src/os/unix/ngx_process_cycle.c:815
#10 0x00000000004461d4 in ngx_spawn_process(cycle=0x202a460,proc=0x447f20 ,data=0x0,
name=0x55b2ac "worker process",respawn=-3)at src/os/unix/ngx_process.c:198
#11 0x00000000004472bc in ngx_start_worker_processes(cycle=0x202a460,n=1,type=-3)
at src/os/unix/ngx_process_cycle.c:396
上下文
r->module->ctx 一个http请求设置在某个模块的ctx
core
main
ngx_init_cycle
222 for (i = 0; cycle->modules[i]; i++) {¬
| 223 if (cycle->modules[i]->type != NGX_CORE_MODULE) {¬
| 224 continue;¬
| 225 }¬
| 226 ¬
| 227 module = cycle->modules[i]->ctx;¬
| 228 ¬
| 229 if (module->create_conf) {¬
| 230 rv = module->create_conf(cycle);¬
| 231 if (rv == NULL) {¬
| 232 ngx_destroy_pool(pool);¬
| 233 return NULL;¬
| 234 }¬
| 235 cycle->conf_ctx[cycle->modules[i]->index] = rv;¬
| 236 }¬
| 237 }¬
286 for (i = 0; cycle->modules[i]; i++) {¬
| 287 if (cycle->modules[i]->type != NGX_CORE_MODULE) {¬
| 288 continue;¬
| 289 }¬
| 290 ¬
| 291 module = cycle->modules[i]->ctx;¬
| 292 ¬
| 293 if (module->init_conf) {¬
| 294 if (module->init_conf(cycle,¬
| 295 cycle->conf_ctx[cycle->modules[i]->index])¬
| 296 == NGX_CONF_ERROR)¬
| 297 {¬
| 298 environ = senv;¬
| 299 ngx_destroy_cycle_pools(&conf);¬
| 300 return NULL;¬
| 301 }¬
| 302 }¬
| 303 }¬
/* handle the listening sockets */¬
ngx_master_process_cycle
ngx_start_worker_processes
| 357 for (i = 0; i < n; i++) {¬
| 358 ¬
| 359 ngx_spawn_process(cycle, ngx_worker_process_cycle,¬
| 360 (void *) (intptr_t) i, "worker process", type);¬
ngx_worker_process_cycle
ngx_worker_process_init
929 for (i = 0; cycle->modules[i]; i++) {¬
| 930 if (cycle->modules[i]->init_process) {¬
| 931 if (cycle->modules[i]->init_process(cycle) == NGX_ERROR) {¬
| 932 /* fatal */¬
| 933 exit(2);¬
| 934 }¬
| 935 }¬
| 936 }¬
ngx_listening_t --- 每个监听端口一个此结构
-> ngx_connection_handler_pt 监听成功建立tcp连接后
events
#0 ngx_epoll_process_events (cycle=0x202a460, timer=5000, flags=1) at src/event/modules/ngx_epoll_module.c:785
#1 0x000000000043fb4a in ngx_process_events_and_timers (cycle=0x202a460) at src/event/ngx_event.c:252
#2 0x0000000000447f65 in ngx_worker_process_cycle (cycle=0x202a460, data=<value optimized out>)
at src/os/unix/ngx_process_cycle.c:815
#3 0x00000000004461d4 in ngx_spawn_process (cycle=0x202a460, proc=0x447f20 <ngx_worker_process_cycle>, data=0x0,
name=0x55b2ac "worker process", respawn=-3) at src/os/unix/ngx_process.c:198
#4 0x00000000004472bc in ngx_start_worker_processes (cycle=0x202a460, n=1, type=-3)
at src/os/unix/ngx_process_cycle.c:396
#5 0x0000000000448873 in ngx_master_process_cycle (cycle=0x202a460) at src/os/unix/ngx_process_cycle.c:135
#6 0x000000000041fec7 in main (argc=<value optimized out>, argv=<value optimized out>) at src/core/nginx.c:381
ngx_event_accept
ngx_event_accept
c = ngx_get_connection(s, ev->log);
c->fd = s;
ls->handler(c); = ngx_http_init_connection
- 81 static ngx_command_t ngx_events_commands[] = {¬
| 82 ¬
| 83 { ngx_string("events"),¬
| 84 NGX_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS,¬
| 85 ngx_events_block,¬
| 86 0,¬
| 87 0,¬
| 88 NULL },¬
| 89 ¬
| 90 ngx_null_command¬
| 91 };¬
92
ngx_events_block
940 (*ctx)[cf->cycle->modules[i]->ctx_index] =¬
| 941 m->create_conf(cf->cycle);¬
| 969 rv = m->init_conf(cf->cycle,¬
| 970 (*ctx)[cf->cycle->modules[i]->ctx_index]);¬
ngx_event_process_init
734 for (i = 0; i < cycle->listening.nelts; i++) {¬
| 735 ¬
| 736 #if (NGX_HAVE_REUSEPORT)¬
| 737 if (ls[i].reuseport && ls[i].worker != ngx_worker) {¬
| 738 continue;¬
| 739 }¬
| 740 #endif¬
| 741 ¬
| 742 c = ngx_get_connection(ls[i].fd, cycle->log);¬
| 754 rev = c->read;¬
| 821 rev->handler = (c->type == SOCK_STREAM) ? ngx_event_accept¬
| 822 : ngx_event_recvmsg;¬
| 856 if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) {¬
| 857 return NGX_ERROR;¬
| 858 }¬
ngx_event_accept 没有post flag，不进入队列，立即执行
ngx_event_s
-> ngx_event_handler_pt
epoll
ngx_event_use
| 1037 if (module->name->len == value[1].len) {¬
| 1038 if (ngx_strcmp(module->name->data, value[1].data) == 0) {¬
| 1039 ecf->use = cf->cycle->modules[m]->ctx_index;¬
| 1040 ecf->name = module->name->data;¬
| 1041 ¬
| 1042 if (ngx_process == NGX_PROCESS_SINGLE¬
| 1043 && old_ecf¬
| 1044 && old_ecf->use != ecf->use)¬
| 1045 {¬
| 1046 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,¬
| 1047 "when the server runs without a master process "¬
| 1048 "the \"%V\" event type must be the same as "¬
| 1049 "in previous configuration - \"%s\" "¬
| 1050 "and it cannot be changed on the fly, "¬
| 1051 "to change it you need to stop server "¬
| 1052 "and start it again",¬
| 1053 &value[1], old_ecf->name);¬
| 1054 ¬
| 1055 return NGX_CONF_ERROR;¬
| 1056 }¬
| 1057 ¬
| 1058 return NGX_CONF_OK;¬
| 1059 }¬
| 1060 }¬
- 179 static ngx_event_module_t ngx_epoll_module_ctx = {¬
| 180 &epoll_name,¬
| 181 ngx_epoll_create_conf, /* create configuration */¬
| 182 ngx_epoll_init_conf, /* init configuration */¬
| 183 ¬
| 184 {¬
| 185 ngx_epoll_add_event, /* add an event */¬
| 186 ngx_epoll_del_event, /* delete an event */¬
| 187 ngx_epoll_add_event, /* enable an event */¬
| 188 ngx_epoll_del_event, /* disable an event */¬
| 189 ngx_epoll_add_connection, /* add an connection */¬
| 190 ngx_epoll_del_connection, /* delete an connection */¬
| 191 #if (NGX_HAVE_EVENTFD)¬
| 192 ngx_epoll_notify, /* trigger a notify */¬
| 193 #else¬
| 194 NULL, /* trigger a notify */¬
| 195 #endif¬
| 196 ngx_epoll_process_events, /* process the events */¬
| 197 ngx_epoll_init, /* init the events */¬
| 198 ngx_epoll_done, /* done the events */¬
| 199 }¬
| 200 };¬
201 ¬
- 202 ngx_module_t ngx_epoll_module = {¬
| 203 NGX_MODULE_V1,¬
| 204 &ngx_epoll_module_ctx, /* module context */¬
| 205 ngx_epoll_commands, /* module directives */¬
| 206 NGX_EVENT_MODULE, /* module type */¬
| 207 NULL, /* init master */¬
| 208 NULL, /* init module */¬
| 209 NULL, /* init process */¬
| 210 NULL, /* init thread */¬
| 211 NULL, /* exit thread */¬
| 212 NULL, /* exit process */¬
| 213 NULL, /* exit master */¬
| 214 NGX_MODULE_V1_PADDING¬
| 215 };¬
cycle
ngx_cycle_t
conf_ctx
http
ngx_http_block
ngx_http_optimize_servers
ngx_http_init_listening
ngx_http_add_listening
ls->handler = ngx_http_init_connection
ngx_http_init_connection
318 rev = c->read;¬
| 319 rev->handler = ngx_http_wait_request_handler;¬
| 320 c->write->handler = ngx_http_empty_handler;¬
ngx_http_wait_request_handler
| 505 rev->handler = ngx_http_process_request_line;¬
| 506 ngx_http_process_request_line(rev);¬
ngx_http_process_request_headers
ngx_http_process_request
| 1944 c->read->handler = ngx_http_request_handler;¬
| 1945 c->write->handler = ngx_http_request_handler;¬
