Apache actively close tcp connections when keep-alive is set by the client - apache

Question

I'm trying to do Apache performance benchmarking for my course project. But I meet a strange problem. When I use a single client to establish multiple TCP connections (e.g. 100) to an Apache server and send HTTP 1.1 requests with the Connection: keep-alive header, I suppose the TCP connections can be reused. But the Apache server will actively terminate TCP connections, even if Connection: Keep-Alive and Keep-Alive: xxx are included in the HTTP response header.
this is my client code (I obfuscate the IP address):
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <time.h>
#include <errno.h>
#include <sys/time.h>
#define DEBUG
#define MAX_SOCKETS_NUM 100
#define BUF_LEN 4096
int main() {
int i;
struct sockaddr_in server_addr, peer_addr;
int res, len = sizeof(peer_addr);
char *req = "GET /20KB HTTP/1.1\r\n"
"Host: x.x.x.192\r\n"
"Connection: keep-alive\r\n"
"Upgrade-Insecure-Requests: 1\r\n"
"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36\r\n"
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\r\n"
"Accept-Encoding: gzip, deflate\r\n"
"Accept-Language: zh-CN,zh;q=0.9\r\n"
"\r\n";
char buf[BUF_LEN + 1];
int sockets[MAX_SOCKETS_NUM];
int estb_num = 0;
int estb_map[MAX_SOCKETS_NUM];
struct timeval goal, now, interval;
// create sockets
for (i = 0; i < MAX_SOCKETS_NUM; i++) {
if ((sockets[i] = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0)) < 0) {
printf("Socket %d error!\n", i);
return -1;
}
}
// initialize server_addr
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(80);
if (inet_pton(AF_INET, "x.x.x.192", &server_addr.sin_addr) <= 0) {
printf("Invalid address/Address not supported\n");
return -1;
}
// connect to the victim server
for (i = 0; i < MAX_SOCKETS_NUM; i++) {
if ((res = connect(sockets[i], (struct sockaddr *)&server_addr, sizeof(server_addr))) == 0) {
#ifdef DEBUG
printf("Socket %d connected immediately.\n", i);
#endif
estb_map[i] = 1;
estb_num++;
}
else if (res == -1) {
if (errno != EINPROGRESS) {
printf("Error occured when connect() is called on socket %d.\n", i);
return -1;
}
#ifdef DEBUG
printf("Socket %d sends SYN packet but the ACK is not received.\n", i);
#endif
}
}
while (1) {
if (estb_num == MAX_SOCKETS_NUM)
break;
for (i = 0; i < MAX_SOCKETS_NUM; i++) {
while (1) {
res = getpeername(sockets[i], (struct sockaddr *)&peer_addr, &len);
if (res == 0) {
estb_num++;
#ifdef DEBUG
printf("Socket %d connects successfully.\n", i);
#endif
break;
}
}
}
}
interval = (struct timeval) {
.tv_sec = 1,
.tv_usec = 0
};
len = strlen(req);
while (1) {
gettimeofday(&now, NULL);
timeradd(&now, &interval, &goal);
#ifdef DEBUG
printf("%ld.%ld\n", goal.tv_sec, goal.tv_usec);
#endif
// send requests
for (i = 0; i < MAX_SOCKETS_NUM; i++) {
res = send(sockets[i], req, len, 0);
if (res == -1) {
printf("socket:%ld errno:%ld\n", i, errno);
}
}
while (1) {
for (i = 0; i < MAX_SOCKETS_NUM; i++) {
res = recv(sockets[i], buf, BUF_LEN, 0);
if (res == 0) {
struct sockaddr_in dbg_addr;
int dgb_addr_len = sizeof(struct sockaddr_in);
getsockname(sockets[i], &dbg_addr, &dgb_addr_len);
printf("socket:%d port:%d\n", i, ntohs(dbg_addr.sin_port));
goto end;
}
else if (res == -1) {
if (errno != EAGAIN)
printf("Error occurs when recv is called.\n");
}
else {
// do nothing because we don't need the response
}
}
gettimeofday(&now, NULL);
if (timercmp(&now, &goal, >) != 0) {
#ifdef DEBUG
printf("%d.%d\n", now.tv_sec, now.tv_usec);
#endif
break;
}
}
}
end:
return 0;
}
the workflow of the client (with the IP address x.x.x.198) is:
establish 100 tcp connections to the server (with the IP address x.x.x.192) using non-blocking socket.
send the same requests on the 100 tcp connections
invoking non-blocking recv repeatedly on these 100 sockets for 1 second
goto 2.
one execution of the program generates the following output:
Socket 0 sends SYN packet but the ACK is not received.
(some lines are omitted)
Socket 99 sends SYN packet but the ACK is not received.
Socket 0 connects successfully.
(some lines are omitted)
Socket 99 connects successfully.
1639450192.343129
1639450192.343155
1639450193.343163
socket:63 port:56804
The output indicates in the second round of requests (the first round is finished because it prints two timestamps, but the second round only prints one), the recv function on the 63th socket with the local port number 56804 returns 0, which means the Apache server actively terminates tcp connections. And I dumped all the packets on the client using tcpdump, the following figure shows the packet trace of the connection with the local port number 56804:
the packet trace shows the same result that the server actively sends tcp FIN packet to the client to terminate the TCP connection. But we can see Connection: Keep-Alive and Keep-Alive: timeout=10m, max=1999 are included in the response header which means the Apache server handles keep-alive correctly.
The server runs Ubuntu 20.04.3 and Apache 2.4.41.
I's very confused about why this happens, why will Apache close keep-alive connections? I'd be appreciate if you can help me, thanks!

Answer 1

From this document:
A host MAY keep an idle connection open for longer than the time that
it indicates, but it SHOULD attempt to retain a connection for at
least as long as indicated.
Capital letters there are key, and written like this in the document:
Your case matches the "SHOULD" part. E.g. keep-alive is a recommendation - but if the server needs those resources (or is configured to have less open connections than the number of your clients) it's free to close them at will. Your clients will need to deal with this state.
If the described behavior is dependent on the number of parallel sockets that you open (apart from the timeout), you're most likely running into resource limits on your server - either explicitly configured, or implicit, from default values.
Imagine how easy a DDOS attack would be if all that's required was a couple of keep-alive requests to saturate the number of concurrent connections that the server offers.
Also note that timeout is based on different perceptions of time on server (starting with sending the last packet) and client (starting when receiving the last packet)