From d8c24b9d08fa8c56fdb9f9fc1d3d8adbbf2f150d Mon Sep 17 00:00:00 2001 From: cyforkk <1139009771@qq.com> Date: Wed, 29 Jul 2026 22:39:12 +0800 Subject: [PATCH] feat: Add task result chunking support (issue #3071, Chunk 1-4) - Add TASK_RESULT_CHUNK_SIZE config option to CoreOptions - Add chunkKey() and isChunkedProperty() helper methods to P class - Implement chunked write in asArray() method - Add RESULT_CHUNK_COUNT constant Co-Authored-By: Claude --- .../apache/hugegraph/config/CoreOptions.java | 13 +++++ .../org/apache/hugegraph/task/HugeTask.java | 49 +++++++++++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java index c10de4bf43..50a0022e09 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java @@ -306,6 +306,19 @@ public class CoreOptions extends OptionHolder { rangeInt(0L, Bytes.GB), 16 * Bytes.MB ); + /** + * The chunk size for task results in bytes. + * When the compressed result exceeds this size, it will be split into + * multiple properties (task_result_0, task_result_1, ...). + * Set to 0 to disable chunking. + */ + public static final ConfigOption TASK_RESULT_CHUNK_SIZE = + new ConfigOption<>( + "task.result_chunk_size", + "The chunk size for task results in bytes. Set 0 to disable chunking.", + rangeInt(0, BytesBuffer.BYTES_LEN_MAX), + 1048576 + ); public static final ConfigOption TASK_TTL_DELETE_BATCH = new ConfigOption<>( "task.ttl_delete_batch", diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java index f63d4c0762..5e37e4372e 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java @@ -575,9 +575,29 @@ protected synchronized Object[] asArray() { if (this.result != null) { byte[] bytes = StringEncoding.compress(this.result); - checkPropertySize(bytes.length, P.RESULT); - list.add(P.RESULT); - list.add(bytes); + int chunkSize = CoreOptions.instance().get(CoreOptions.TASK_RESULT_CHUNK_SIZE); + + if (chunkSize > 0 && bytes.length > chunkSize) { + // Split into chunks + int numChunks = (int) Math.ceil((double) bytes.length / chunkSize); + for (int i = 0; i < numChunks; i++) { + int start = i * chunkSize; + int end = Math.min(start + chunkSize, bytes.length); + byte[] chunk = new byte[end - start]; + System.arraycopy(bytes, start, chunk, 0, end - start); + checkPropertySize(chunk.length, P.chunkKey(i)); + list.add(P.chunkKey(i)); + list.add(chunk); + } + // Write chunk count + list.add(P.RESULT_CHUNK_COUNT); + list.add(String.valueOf(numChunks)); + } else { + // Single property (original behavior) + checkPropertySize(bytes.length, P.RESULT); + list.add(P.RESULT); + list.add(bytes); + } } if (this.server != null) { @@ -859,9 +879,32 @@ public static final class P { public static final String RETRIES = "~task_retries"; public static final String INPUT = "~task_input"; public static final String RESULT = "~task_result"; + public static final String RESULT_CHUNK_COUNT = "~task_result_chunk_count"; public static final String DEPENDENCIES = "~task_dependencies"; public static final String SERVER = "~task_server"; + /** + * Get the chunk key for a given chunk index. + * @param index the chunk index, or -1 for the non-chunked result key + * @return the property key for the chunk + */ + public static String chunkKey(int index) { + if (index < 0) { + return RESULT; + } + return RESULT + "_" + index; + } + + /** + * Check if a property key is a chunked result property. + * @param key the property key to check + * @return true if the key matches the chunk pattern + */ + public static boolean isChunkedProperty(String key) { + return key.startsWith(RESULT + "_") && key.length() > RESULT.length() + 1 && + key.substring(RESULT.length() + 1).matches("\\d+"); + } + private static final String[] METADATA_PROPERTIES = new String[]{ TYPE, NAME, CALLABLE, DESCRIPTION, CONTEXT, STATUS, PROGRESS, CREATE, UPDATE, RETRIES, DEPENDENCIES, INPUT, SERVER