I have Spring Boot (3.3.2) microservice and do load testing.
My problem is app can’t handle more than 60 RPS.

Inside request I call about 8 endpoint (with same host). Average request processing time 800ms.
When exceeding 60 RPS, the request processing time increases to several seconds.
I use RestClient with Virtual Threads. But if I use RestTemplate then situation is even worse, the application can handle about 40 RPS. Logging and monitoring show that the time it takes to call external systems is increasing, although in reality this is not the case and monitoring on the external systems side shows normal expected values.
App inside Docker in Kubernetes. Limits are not exceeded.

I thought it was a connection pool issue, but increasing or decreasing the pool configuration values ​​doesn’t change much. What could be the problem?

Below is the configuration class:

package com.nc.config;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.nexign.mfactory.ng.handler.RestErrorHandler;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.httpcomponents.hc5.PoolingHttpClientConnectionManagerMetricsBinder;
import org.apache.hc.client5.http.classic.HttpClient;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.socket.ConnectionSocketFactory;
import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
import org.apache.hc.core5.http.config.Registry;
import org.apache.hc.core5.http.config.RegistryBuilder;
import org.apache.hc.core5.util.TimeValue;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.http.MediaType;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.HttpMessageConverter;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.web.client.RestClient;

import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;

@Configuration
public class BaseConfig {
    @Value("${HTTP_CLIENT_MAX_CONN_TOTAL:2000}")
    private Integer maxPoolSize;
    @Value("${HTTP_CLIENT_MAX_PER_ROUTE:1000}")
    private Integer maxPerRoute;
    @Value("${HTTP_CLIENT_MAIN_TIMEOUT:2000}")
    private Integer timeout;

    @Bean
    public ObjectMapper mapper(Jackson2ObjectMapperBuilder builder) {
        return builder.modules(new JavaTimeModule())
                .featuresToDisable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
                .featuresToEnable(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, SerializationFeature.INDENT_OUTPUT)
                .serializationInclusion(JsonInclude.Include.NON_NULL)
                .build();
    }

    @Bean
    public HttpClient httpClient(MeterRegistry meterRegistry){

        PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();

        connectionManager.setMaxTotal(maxPoolSize);
        connectionManager.setDefaultMaxPerRoute(maxPerRoute);
        new PoolingHttpClientConnectionManagerMetricsBinder(connectionManager, "http-client-pool").bindTo(meterRegistry);
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectionRequestTimeout(timeout, TimeUnit.MILLISECONDS)
                .setResponseTimeout(timeout, TimeUnit.MILLISECONDS)
                .build();

        return HttpClientBuilder.create().setDefaultRequestConfig(requestConfig)
                .setConnectionManager(connectionManager)              
                .build();
    }

    @Bean
    public ClientHttpRequestFactory httpRequestFactory(HttpClient httpClient){
        return new HttpComponentsClientHttpRequestFactory(httpClient);
    }

    @Bean
    @Primary
    public RestClient restClient(RestClient.Builder restClientBuilder, @Qualifier("mapper") ObjectMapper mapper,
                                 UrlsConfig urlsConfig, HttpClient httpClient) {

        List<HttpMessageConverter<?>> messageConverters = new ArrayList<>();
        messageConverters.add(0, new StringHttpMessageConverter(StandardCharsets.UTF_8));
        var converter = new MappingJackson2HttpMessageConverter(mapper);
        converter.setSupportedMediaTypes(Collections.singletonList(MediaType.ALL));
        messageConverters.add(1, converter);
        return restClientBuilder
                .defaultStatusHandler(new RestErrorHandler(urlsConfig))
                .requestFactory(httpRequestFactory(httpClient))
                .messageConverters(converters -> converters.addAll(messageConverters))
                .build();
    }
}