From e928d8f48c8cf0f936a74ae7f676502fcce1fb6e Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Mon, 11 May 2026 16:47:17 +0800 Subject: [PATCH 1/2] Fix typo. --- graph_net/tools/generate_subgraph_dataset.sh | 12 ++++++------ sqlite/graphsample_insert.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/graph_net/tools/generate_subgraph_dataset.sh b/graph_net/tools/generate_subgraph_dataset.sh index e29a0cbb3..ccab4adf7 100755 --- a/graph_net/tools/generate_subgraph_dataset.sh +++ b/graph_net/tools/generate_subgraph_dataset.sh @@ -3,7 +3,7 @@ set -x MIN_SEQ_OPS=${1:-4} MAX_SEQ_OPS=${2:-64} -GPU_ID=${3:-0} +GPU_ID=${3:-7} OP_RANGE=$MIN_SEQ_OPS-$MAX_SEQ_OPS @@ -12,9 +12,9 @@ export CUDA_VISIBLE_DEVICES="${GPU_ID}" GRAPH_NET_ROOT=$(python3 -c "import graph_net; import os; print(os.path.dirname(os.path.dirname(graph_net.__file__)))") RESUME="true" -DECOMPOSE_WORKSPACE=/tmp/subgraph_dataset_workspace +#DECOMPOSE_WORKSPACE=/tmp/subgraph_dataset_workspace +DECOMPOSE_WORKSPACE=/work/graphnet_test_workspace/subgraph_dataset_20260203 OUTPUT_DIR=$DECOMPOSE_WORKSPACE/outputs -DB_PATH=$OUTPUT_DIR/GraphNet.db DEVICE_REWRITED_SAMPLE_DIR=$DECOMPOSE_WORKSPACE/01_device_rewrited_samples DIM_GENERALIZED_SAMPLE_DIR=$DECOMPOSE_WORKSPACE/02_dimension_generalized_samples @@ -131,7 +131,7 @@ function dimension_generalizer(){ } EOF ) - cp -rf $DEVICE_REWRITED_SAMPLE_DIR $DIM_GENERALIZED_SAMPLE_DIR/9 + cp -rf $DEVICE_REWRITED_SAMPLE_DIR/* $DIM_GENERALIZED_SAMPLE_DIR/9/ } function generate_op_names() { @@ -599,7 +599,7 @@ function generate_fusible_subgraphs() { generate_generalized_subgraph_list ${DTYPE_GENERALIZED_FUSIBLE_SUBGRAPH_DIR} ${dtype_generalized_subgraphs_list} # extract backward graphs (train_forward, train_backward, eval_forward) - backward_graph_extractor 2>&1 | tee ${DECOMPOSE_WORKSPACE}/log_backward_graph_extractor_fusible_${suffix}.txt + # backward_graph_extractor 2>&1 | tee ${DECOMPOSE_WORKSPACE}/log_backward_graph_extractor_${suffix}.txt # generate kernelbench format unittest # generate_unittests 2>&1 | tee ${DECOMPOSE_WORKSPACE}/log_unittests_${suffix}.txt @@ -622,7 +622,7 @@ function generate_typical_subgraphs() { generate_generalized_subgraph_list ${DTYPE_GENERALIZED_TYPICAL_SUBGRAPH_DIR} ${dtype_generalized_typical_subgraph_list} # extract backward graphs (train_forward, train_backward, eval_forward) - backward_graph_extractor_typical 2>&1 | tee ${DECOMPOSE_WORKSPACE}/log_backward_graph_extractor_typical_${suffix}.txt + #backward_graph_extractor_typical 2>&1 | tee ${DECOMPOSE_WORKSPACE}/log_backward_graph_extractor_typical_${suffix}.txt # generate kernelbench format unittest # generate_unittest_for_typical_subgraphs 2>&1 | tee ${DECOMPOSE_WORKSPACE}/log_unittests_typical_subgraphs_${suffix}.txt diff --git a/sqlite/graphsample_insert.py b/sqlite/graphsample_insert.py index c321c569c..27d1f3b5f 100755 --- a/sqlite/graphsample_insert.py +++ b/sqlite/graphsample_insert.py @@ -457,7 +457,7 @@ def insert_one_sample( except sqlite3.IntegrityError as e: session.rollback() print( - "insert {sample_type} failed: integrity error (possible duplicate uuid or graph_hash)" + f"insert {sample_type} failed: integrity error (possible duplicate uuid or graph_hash)" ) print(f"error info: {e}") successed = False From 2d33859e7caaab32f830c98a2cec3476111c0004 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Wed, 13 May 2026 15:43:38 +0800 Subject: [PATCH 2/2] Rename and allow to config more arguments. --- graph_net/tools/generate_subgraph_dataset.sh | 54 ++++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/graph_net/tools/generate_subgraph_dataset.sh b/graph_net/tools/generate_subgraph_dataset.sh index ccab4adf7..52a650d88 100755 --- a/graph_net/tools/generate_subgraph_dataset.sh +++ b/graph_net/tools/generate_subgraph_dataset.sh @@ -1,19 +1,21 @@ #!/bin/bash set -x +GRAPH_NET_ROOT=$(python3 -c "import graph_net; import os; print(os.path.dirname(os.path.dirname(graph_net.__file__)))") + MIN_SEQ_OPS=${1:-4} MAX_SEQ_OPS=${2:-64} GPU_ID=${3:-7} +MODEL_LIST=${5:-"$GRAPH_NET_ROOT/graph_net/config/torch100_samples_list.txt"} +DECOMPOSE_WORKSPACE=${4:-"/tmp/subgraph_dataset_workspace"} +USE_SUBPROCESS=${5:-1} OP_RANGE=$MIN_SEQ_OPS-$MAX_SEQ_OPS export CUDA_VISIBLE_DEVICES="${GPU_ID}" -GRAPH_NET_ROOT=$(python3 -c "import graph_net; import os; print(os.path.dirname(os.path.dirname(graph_net.__file__)))") RESUME="true" -#DECOMPOSE_WORKSPACE=/tmp/subgraph_dataset_workspace -DECOMPOSE_WORKSPACE=/work/graphnet_test_workspace/subgraph_dataset_20260203 OUTPUT_DIR=$DECOMPOSE_WORKSPACE/outputs DEVICE_REWRITED_SAMPLE_DIR=$DECOMPOSE_WORKSPACE/01_device_rewrited_samples @@ -25,15 +27,15 @@ RENAMED_TYPICAL_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/06_renamed_typical_subgraphs DEDUP_TYPICAL_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/07_deduplicated_typical_subgraphs # fusible_subgraphs -CUMSUM_NUM_KERNELS_DIR=$DECOMPOSE_WORKSPACE/08_cumsum_num_kernels -FUSIBLE_SUBGRAPH_RANGE_DIR=$DECOMPOSE_WORKSPACE/09_fusible_subgraph_ranges -GROUPED_FUSIBLE_SUBGRAPH_RANGE_DIR=$DECOMPOSE_WORKSPACE/10_grouped_fusible_subgraph_ranges -DIM_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/11_dimension_generalized_fusible_subgraphs -RENAMED_DIM_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/12_renamed_dimension_generalized_fusible_subgraphs -DEDUP_DIM_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/13_deduplicated_dimension_generalized_fusible_subgraphs -DTYPE_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/14_dtype_generalized_fusible_subgraphs -BACKWARD_GRAPH_OUTPUT_DIR=$DECOMPOSE_WORKSPACE/15_backward_fusible_subgraphs -# FUSIBLE_SUBGRAPH_UNITTEST_DIR=$DECOMPOSE_WORKSPACE/16_fusible_subgraphs_unittests +CUMSUM_NUM_KERNELS_DIR=$DECOMPOSE_WORKSPACE/1-08_cumsum_num_kernels +FUSIBLE_SUBGRAPH_RANGE_DIR=$DECOMPOSE_WORKSPACE/1-09_fusible_subgraph_ranges +GROUPED_FUSIBLE_SUBGRAPH_RANGE_DIR=$DECOMPOSE_WORKSPACE/1-10_grouped_fusible_subgraph_ranges +DIM_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/1-11_dimension_generalized_fusible_subgraphs +RENAMED_DIM_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/1-12_renamed_dimension_generalized_fusible_subgraphs +DEDUP_DIM_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/1-13_deduplicated_dimension_generalized_fusible_subgraphs +DTYPE_GENERALIZED_FUSIBLE_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/1-14_dtype_generalized_fusible_subgraphs +BACKWARD_GRAPH_OUTPUT_DIR=$DECOMPOSE_WORKSPACE/1-15_backward_fusible_subgraphs +# FUSIBLE_SUBGRAPH_UNITTEST_DIR=$DECOMPOSE_WORKSPACE/16_fusible_kernelbench_unittests # typical_subgraphs DIM_GENERALIZED_TYPICAL_SUBGRAPH_DIR=$DECOMPOSE_WORKSPACE/2-08_dimension_generalized_typical_subgraphs @@ -46,16 +48,14 @@ BACKWARD_GRAPH_TYPICAL_OUTPUT_DIR=$DECOMPOSE_WORKSPACE/2-12_backward_typical_sub mkdir -p $DECOMPOSE_WORKSPACE mkdir -p $OUTPUT_DIR -model_list="$GRAPH_NET_ROOT/graph_net/config/torch100_samples_list.txt" - device_rewrited_sample_list=${DECOMPOSE_WORKSPACE}/device_rewrited_sample_list.txt -range_decomposed_subgraph_list=${DECOMPOSE_WORKSPACE}/range_decomposed_subgraph_sample_list.txt -deduplicated_subgraph_list=${DECOMPOSE_WORKSPACE}/deduplicated_subgraph_sample_list.txt +range_decomposed_subgraph_list=${DECOMPOSE_WORKSPACE}/range_decomposed_subgraph_list.txt +deduplicated_subgraph_list=${DECOMPOSE_WORKSPACE}/deduplicated_subgraph_list.txt # fusible_subgraphs -dimension_generalized_subgraph_list=${DECOMPOSE_WORKSPACE}/dimension_generalized_subgraph_sample_list.txt -deduplicated_fusible_subgraphs_list=${DECOMPOSE_WORKSPACE}/deduplicated_dimension_generalized_subgraph_sample_list.txt -dtype_generalized_subgraphs_list=${DECOMPOSE_WORKSPACE}/dtype_generalized_subgraphs_sample_list.txt +dimension_generalized_subgraph_list=${DECOMPOSE_WORKSPACE}/dimension_generalized_fusible_subgraph_list.txt +deduplicated_fusible_subgraphs_list=${DECOMPOSE_WORKSPACE}/deduplicated_dimension_generalized_fusible_subgraph_list.txt +dtype_generalized_subgraphs_list=${DECOMPOSE_WORKSPACE}/dtype_generalized_fusible_subgraph_list.txt # typical_subgraphs dimension_generalized_typical_subgraph_list=${DECOMPOSE_WORKSPACE}/dimension_generalized_typical_subgraph_list.txt @@ -63,7 +63,7 @@ deduplicated_typical_subgraph_list=${DECOMPOSE_WORKSPACE}/deduplicated_dimension dtype_generalized_typical_subgraph_list=${DECOMPOSE_WORKSPACE}/dtype_generalized_typical_subgraph_list.txt -if [[ "$model_list" == *"/torch_samples_list.txt" ]]; then +if [[ "$USE_SUBPROCESS" == "1" ]]; then USE_SUBPROCESS_ARGS="--use-subprocess" else USE_SUBPROCESS_ARGS="" @@ -85,7 +85,7 @@ function generate_subgraph_list() { local sample_list="$2" echo ">>> Generate subgraph_sample_list for samples under ${target_dir}." echo ">>>" - cat $model_list \ + cat $MODEL_LIST \ | grep -v '# ' \ | xargs -I {} find ${target_dir}/{} -name "model.py" \ | xargs dirname \ @@ -97,7 +97,7 @@ function rewrite_device() { echo ">>> [1] Rewrite devices for subgraph samples under ${GRAPH_NET_ROOT}." echo ">>>" python3 -m graph_net.model_path_handler \ - --model-path-list ${model_list} \ + --model-path-list ${MODEL_LIST} \ --handler-config=$(base64 -w 0 <>> [3] Generate op_names.txt for samples in ${model_list}." + echo ">>> [3] Generate op_names.txt for samples in ${MODEL_LIST}." echo ">>>" python3 -m graph_net.model_path_handler ${USE_SUBPROCESS_ARGS} \ - --model-path-list $model_list \ + --model-path-list $MODEL_LIST \ --handler-config=$(base64 -w 0 <>> [4] Generate subgraph_ranges.json for samples in ${model_list}." + echo ">>> [4] Generate subgraph_ranges.json for samples in ${MODEL_LIST}." echo ">>> MIN_SEQ_OPS: ${MIN_SEQ_OPS}, MAX_SEQ_OPS: ${MAX_SEQ_OPS}" echo ">>>" python3 -m graph_net.apply_sample_pass \ - --model-path-list $model_list \ + --model-path-list $MODEL_LIST \ --sample-pass-file-path $GRAPH_NET_ROOT/graph_net/torch/sample_pass/typical_sequence_split_points.py \ --sample-pass-class-name TypicalSequenceSplitPointsGenerator \ --sample-pass-config=$(base64 -w 0 <