trpc-agent-python/examples/optimization/quickstart/optimizer.json at main · Adonis-a233/trpc-agent-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
{
  "evaluate": {
    "metrics": [
      {
        "metric_name": "final_response_avg_score",
        "threshold": 1.0,
        "criterion": {
          "final_response": {
            "text": {
              "match": "contains",
              "case_insensitive": true
            }
          }
        }
      },
      {
        "metric_name": "llm_rubric_response",
        "threshold": 0.66,
        "criterion": {
          "llm_judge": {
            "judge_model": {
              "model_name": "${TRPC_AGENT_MODEL_NAME}",
              "base_url": "${TRPC_AGENT_BASE_URL}",
              "api_key": "${TRPC_AGENT_API_KEY}",
              "num_samples": 1,
              "generation_config": {
                "max_tokens": 1024,
                "temperature": 0.2
              }
            },
            "rubrics": [
              {
                "id": "numeric_correct",
                "content": {
                  "text": "最终给出的数字答案是否与参考答案一致（数值相同即视为通过）。"
                },
                "description": "数字答案与参考答案一致",
                "type": "FINAL_RESPONSE_QUALITY"
              },
              {
                "id": "reasoning_clear",
                "content": {
                  "text": "回答中是否给出清晰、可追溯的推理或计算步骤，让读者能复核结果。"
                },
                "description": "推理步骤清晰",
                "type": "FINAL_RESPONSE_QUALITY"
              },
              {
                "id": "units_present",
                "content": {
                  "text": "最终数字答案是否带有正确的单位（例如：个、元、公里、千克、人 等）。"
                },
                "description": "答案带正确单位",
                "type": "FINAL_RESPONSE_QUALITY"
              }
            ]
          }
        }
      }
    ],
    "num_runs": 1
  },
  "optimize": {
    "eval_case_parallelism": 2,
    "stop": {
      "required_metrics": "all"
    },
    "algorithm": {
      "name": "gepa_reflective",
      "seed": 42,
      "reflection_lm": {
        "model_name": "${TRPC_AGENT_MODEL_NAME}",
        "base_url": "${TRPC_AGENT_BASE_URL}",
        "api_key": "${TRPC_AGENT_API_KEY}",
        "generation_config": {
          "max_tokens": 4096,
          "temperature": 0.6
        }
      },
      "candidate_selection_strategy": "pareto",
      "module_selector": "round_robin",
      "reflection_minibatch_size": 3,
      "skip_perfect_score": false,
      "max_metric_calls": 60,
      "max_iterations_without_improvement": 8
    }
  }
}