fix(filter): enhance load detection logging and memory management

Author: contrueCT

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/LoadDetectFilter.java

@@ -58,6 +58,9 @@ public class LoadDetectFilter implements ContainerRequestFilter {
     private static final RateLimiter GC_RATE_LIMITER =
             RateLimiter.create(1.0 / 30);
 
+    // Log at most 1 request per second to avoid too many logs when server is under heavy load
+    private static final RateLimiter REJECT_LOG_RATE_LIMITER = RateLimiter.create(1.0);
+
     @Context
     private jakarta.inject.Provider<HugeConfig> configProvider;
     @Context
@@ -71,10 +74,12 @@ public static boolean isWhiteAPI(ContainerRequestContext context) {
         return WHITE_API_LIST.contains(rootPath);
     }
 
-    private static void gcIfNeeded() {
+    private static boolean gcIfNeeded() {
         if (GC_RATE_LIMITER.tryAcquire(1)) {
             System.gc();
+            return true;
         }
+        return false;
     }
 
     @Override
@@ -90,10 +95,12 @@ public void filter(ContainerRequestContext context) {
         // There will be a thread doesn't work, dedicated to statistics
         int currentLoad = load.incrementAndGet();
         if (currentLoad >= maxWorkerThreads) {
-            LOG.warn("Rejected request due to high worker load, method={}, path={}, " +
-                     "currentLoad={}, maxWorkerThreads={}",
-                     context.getMethod(), context.getUriInfo().getPath(),
-                     currentLoad, maxWorkerThreads);
+            if (REJECT_LOG_RATE_LIMITER.tryAcquire()) {
+                LOG.warn("Rejected request due to high worker load, method={}, path={}, " +
+                         "currentLoad={}, maxWorkerThreads={}",
+                         context.getMethod(), context.getUriInfo().getPath(),
+                         currentLoad, maxWorkerThreads);
+            }
             throw new ServiceUnavailableException(String.format(
                     "The server is too busy to process the request, " +
                     "you can config %s to adjust it or try again later",
@@ -106,11 +113,17 @@ public void filter(ContainerRequestContext context) {
         long presumableFreeMem = (Runtime.getRuntime().maxMemory() -
                                   allocatedMem) / Bytes.MB;
         if (presumableFreeMem < minFreeMemory) {
-            gcIfNeeded();
+            boolean gcTriggered = gcIfNeeded();
+            long allocatedMemAfterCheck = Runtime.getRuntime().totalMemory() -
+                                          Runtime.getRuntime().freeMemory();
+            long recheckedFreeMem = (Runtime.getRuntime().maxMemory() -
+                                     allocatedMemAfterCheck) / Bytes.MB;
             LOG.warn("Rejected request due to low free memory, method={}, path={}, " +
-                     "presumableFreeMemMB={}, minFreeMemoryMB={}",
+                     "presumableFreeMemMB={}, recheckedFreeMemMB={}, gcTriggered={}, " +
+                     "minFreeMemoryMB={}",
                      context.getMethod(), context.getUriInfo().getPath(),
-                     presumableFreeMem, minFreeMemory);
+                     presumableFreeMem, recheckedFreeMem, gcTriggered,
+                     minFreeMemory);
             throw new ServiceUnavailableException(String.format(
                     "The server available memory %s(MB) is below than " +
                     "threshold %s(MB) and can't process the request, " +

hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/api/filter/LoadDetectFilterTest.java

@@ -27,6 +27,7 @@
 import org.apache.hugegraph.config.ServerOptions;
 import org.apache.hugegraph.define.WorkLoad;
 import org.apache.hugegraph.testutil.Assert;
+import org.apache.hugegraph.testutil.Whitebox;
 import org.apache.hugegraph.unit.BaseUnitTest;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,9 +56,8 @@ public void setup() {
         Mockito.when(this.requestContext.getMethod()).thenReturn("GET");
 
         this.loadDetectFilter = new LoadDetectFilter();
-        injectProvider(this.loadDetectFilter, "loadProvider", () -> this.workLoad);
-        injectProvider(this.loadDetectFilter, "configProvider",
-                       () -> createConfig(8, 0));
+        this.setLoadProvider(this.workLoad);
+        this.setConfigProvider(createConfig(8, 0));
     }
 
     @Test
@@ -74,8 +74,7 @@ public void testFilter_WhiteListPathIgnored() {
     public void testFilter_RejectsWhenWorkerLoadIsTooHigh() {
         setupPath("graphs/hugegraph/vertices",
                   List.of("graphs", "hugegraph", "vertices"));
-        injectProvider(this.loadDetectFilter, "configProvider",
-                       () -> createConfig(2, 0));
+        this.setConfigProvider(createConfig(2, 0));
         this.workLoad.incrementAndGet();
 
         ServiceUnavailableException exception = (ServiceUnavailableException) Assert.assertThrows(
@@ -92,8 +91,7 @@ public void testFilter_RejectsWhenWorkerLoadIsTooHigh() {
     public void testFilter_RejectsWhenFreeMemoryIsTooLow() {
         setupPath("graphs/hugegraph/vertices",
                   List.of("graphs", "hugegraph", "vertices"));
-        injectProvider(this.loadDetectFilter, "configProvider",
-                       () -> createConfig(8, Integer.MAX_VALUE));
+        this.setConfigProvider(createConfig(8, Integer.MAX_VALUE));
 
         ServiceUnavailableException exception = (ServiceUnavailableException) Assert.assertThrows(
                 ServiceUnavailableException.class,
@@ -109,8 +107,7 @@ public void testFilter_RejectsWhenFreeMemoryIsTooLow() {
     public void testFilter_AllowsRequestWhenLoadAndMemoryAreHealthy() {
         setupPath("graphs/hugegraph/vertices",
                   List.of("graphs", "hugegraph", "vertices"));
-        injectProvider(this.loadDetectFilter, "configProvider",
-                       () -> createConfig(8, 0));
+        this.setConfigProvider(createConfig(8, 0));
 
         this.loadDetectFilter.filter(this.requestContext);
 
@@ -138,14 +135,13 @@ private PathSegment createPathSegment(String path) {
         return segment;
     }
 
-    private <T> void injectProvider(LoadDetectFilter filter, String fieldName,
-                                    Provider<T> provider) {
-        try {
-            java.lang.reflect.Field field = LoadDetectFilter.class.getDeclaredField(fieldName);
-            field.setAccessible(true);
-            field.set(filter, provider);
-        } catch (Exception e) {
-            throw new RuntimeException("Failed to inject provider: " + fieldName, e);
-        }
+    private void setLoadProvider(WorkLoad workLoad) {
+        Whitebox.setInternalState(this.loadDetectFilter, "loadProvider",
+                                  (Provider<WorkLoad>) () -> workLoad);
+    }
+
+    private void setConfigProvider(HugeConfig config) {
+        Whitebox.setInternalState(this.loadDetectFilter, "configProvider",
+                                  (Provider<HugeConfig>) () -> config);
     }
 }