Skip to content

Rename thread to discussion in pull request dashboard scripts #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
trask wants to merge 1 commit into from
+56 −56
Closed

Rename thread to discussion in pull request dashboard scripts #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions .github/scripts/pull-request-dashboard/classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@
from utils import truncate


LLM_THREAD_TIMEOUT_SECONDS = 180
LLM_DISCUSSION_TIMEOUT_SECONDS = 180
CLASSIFICATION_CACHE_DIR = Path(__file__).resolve().parent / ".cache" / "classifications"
THREAD_RECENT_COMMENTS_LIMIT = 20
THREAD_COMMENT_BODY_MAX_CHARS = 500
DISCUSSION_RECENT_COMMENTS_LIMIT = 20
DISCUSSION_COMMENT_BODY_MAX_CHARS = 500
MAX_PROMPT_CHARS = 18_000

THREAD_PROMPT_TEMPLATE = """You are triaging one pull request discussion thread.
DISCUSSION_PROMPT_TEMPLATE = """You are triaging one pull request discussion thread.

Classify ONLY this one thread. You are not deciding the final dashboard section.
The final routing is computed later from deterministic facts and all thread
Expand Down Expand Up @@ -119,7 +119,7 @@ def extract_json_object(s: str) -> dict[str, Any] | None:
return objects[-1] if objects else None


def normalize_thread_action(action: str) -> str:
def normalize_discussion_action(action: str) -> str:
action = (action or "").lower().strip()
if action in ("author", "reviewer", "external", "none", "unclear"):
return action
Expand All @@ -128,12 +128,12 @@ def normalize_thread_action(action: str) -> str:
return "unclear"


def parse_thread_decision(response_text: str) -> tuple[dict[str, str], bool]:
def parse_discussion_decision(response_text: str) -> tuple[dict[str, str], bool]:
obj = extract_json_object(response_text) if response_text else None
if not obj:
return {"thread_action": "unclear", "reason": "LLM did not return valid JSON"}, False
raw_action = str(obj.get("thread_action") or obj.get("route") or "")
action = normalize_thread_action(raw_action)
action = normalize_discussion_action(raw_action)
valid_action = raw_action.lower().strip() in (
"author",
"reviewer",
Expand Down Expand Up @@ -161,7 +161,7 @@ def participant_role(actor_role: str) -> str:
return "reviewer"


def thread_prompt_input(thread: dict[str, Any]) -> dict[str, Any]:
def discussion_prompt_input(thread: dict[str, Any]) -> dict[str, Any]:
prompt_thread = {
key: value
for key, value in thread.items()
Expand All @@ -180,23 +180,23 @@ def thread_prompt_input(thread: dict[str, Any]) -> dict[str, Any]:
return prompt_thread


def thread_prompt(thread: dict[str, Any]) -> str:
prompt_thread = thread_prompt_input(thread)
def discussion_prompt(thread: dict[str, Any]) -> str:
prompt_thread = discussion_prompt_input(thread)
thread_text = json.dumps(prompt_thread, indent=2, sort_keys=True)
prompt = THREAD_PROMPT_TEMPLATE.format(thread=thread_text)
prompt = DISCUSSION_PROMPT_TEMPLATE.format(thread=thread_text)
if len(prompt) <= MAX_PROMPT_CHARS:
return prompt
trimmed = dict(prompt_thread)
comments = [dict(c) for c in prompt_thread.get("comments") or []]
for c in comments:
c["body"] = truncate(c.get("body") or "", THREAD_COMMENT_BODY_MAX_CHARS)
trimmed["comments"] = comments[-THREAD_RECENT_COMMENTS_LIMIT:]
c["body"] = truncate(c.get("body") or "", DISCUSSION_COMMENT_BODY_MAX_CHARS)
trimmed["comments"] = comments[-DISCUSSION_RECENT_COMMENTS_LIMIT:]
thread_text = json.dumps(trimmed, indent=2, sort_keys=True)
return THREAD_PROMPT_TEMPLATE.format(thread=thread_text)
return DISCUSSION_PROMPT_TEMPLATE.format(thread=thread_text)


def run_llm_for_thread(thread: dict[str, Any], model: str) -> dict[str, Any]:
prompt = thread_prompt(thread)
def run_llm_for_discussion(thread: dict[str, Any], model: str) -> dict[str, Any]:
prompt = discussion_prompt(thread)
with tempfile.TemporaryDirectory(prefix="copilot-otel-") as otel_dir:
otel_path = Path(otel_dir) / "copilot-otel.jsonl"
env = os.environ.copy()
Expand All @@ -208,12 +208,12 @@ def run_llm_for_thread(thread: dict[str, Any], model: str) -> dict[str, Any]:
text=True,
encoding="utf-8",
errors="replace",
timeout=LLM_THREAD_TIMEOUT_SECONDS,
timeout=LLM_DISCUSSION_TIMEOUT_SECONDS,
env=env,
)
print_copilot_otel_file(otel_path)
response_text = proc.stdout
decision, valid_response = parse_thread_decision(response_text)
decision, valid_response = parse_discussion_decision(response_text)
failed = proc.returncode != 0 or not valid_response
record = {
"thread_id": thread["thread_id"],
Expand All @@ -236,12 +236,12 @@ def run_llm_for_thread(thread: dict[str, Any], model: str) -> dict[str, Any]:
return record


def thread_cache_key(thread: dict[str, Any], model: str) -> str:
def discussion_cache_key(thread: dict[str, Any], model: str) -> str:
cache_key_json = json.dumps(
{
"model": model,
"prompt_template": THREAD_PROMPT_TEMPLATE,
"thread": thread_prompt_input(thread),
"prompt_template": DISCUSSION_PROMPT_TEMPLATE,
"thread": discussion_prompt_input(thread),
Comment thread
trask marked this conversation as resolved.
Comment on lines +243 to +244
},
sort_keys=True,
separators=(",", ":"),
Expand Down Expand Up @@ -285,12 +285,12 @@ def prune_classification_cache(open_pr_numbers: set[int]) -> None:
path.unlink()


def classify_threads(number: int, threads: list[dict[str, Any]], model: str) -> list[dict[str, Any]]:
def classify_discussions(number: int, threads: list[dict[str, Any]], model: str) -> list[dict[str, Any]]:
cache_in = load_classification_cache(number)
cache_out: dict[str, dict[str, Any]] = {}
classifications: list[dict[str, Any]] = []
for thread in threads:
key = thread_cache_key(thread, model)
key = discussion_cache_key(thread, model)
cached = cache_in.get(key)
if isinstance(cached, dict):
record = cached_classification_record(cached)
Expand All @@ -300,14 +300,14 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) ->
cache_out[key] = record
continue
try:
record = run_llm_for_thread(thread, model)
record = run_llm_for_discussion(thread, model)
except subprocess.TimeoutExpired as e:
record = {
"thread_id": thread["thread_id"],
"thread_kind": thread["thread_kind"],
"_copilot_cli_call": True,
"failed": True,
"error": f"Copilot CLI timed out after {LLM_THREAD_TIMEOUT_SECONDS}s",
"error": f"Copilot CLI timed out after {LLM_DISCUSSION_TIMEOUT_SECONDS}s",
"decision": {"thread_action": "unclear", "reason": "LLM timeout"},
}
if e.stdout and e.stdout.strip():
Expand Down
62 changes: 31 additions & 31 deletions .github/scripts/pull-request-dashboard/dashboard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,10 +158,10 @@
repo_state_key,
)
from classification import (
THREAD_RECENT_COMMENTS_LIMIT,
classify_threads,
DISCUSSION_RECENT_COMMENTS_LIMIT,
classify_discussions,
is_conflict_resolution_comment,
normalize_thread_action,
normalize_discussion_action,
prune_classification_cache,
)
from state import (
Expand Down Expand Up @@ -477,7 +477,7 @@ def compute_facts(
return facts


def thread_comment(
def discussion_comment(
timestamp: str,
actor: str,
author: str,
Expand All @@ -494,7 +494,7 @@ def thread_comment(
}


def add_thread_facts(
def add_discussion_facts(
thread: dict[str, Any],
comments: list[dict[str, Any]],
facts: dict[str, Any],
Expand Down Expand Up @@ -535,7 +535,7 @@ def group_review_threads(
comments = []
for c in ((thread.get("comments") or {}).get("nodes") or []):
actor = reviewer_actor_login(c.get("author") or {})
comments.append(thread_comment(
comments.append(discussion_comment(
c.get("updatedAt") or c.get("createdAt") or "",
actor,
author,
Expand All @@ -547,7 +547,7 @@ def group_review_threads(
comments.sort(key=lambda c: c["timestamp"])
if not comments:
continue
threads.append(add_thread_facts({
threads.append(add_discussion_facts({
"thread_id": thread.get("id") or f"review-thread-{len(threads) + 1}",
"thread_kind": "review-comment-thread",
"path": thread.get("path"),
Expand Down Expand Up @@ -581,7 +581,7 @@ def group_pr_conversation(
comments = []
for c in raw["issue_comments"]:
actor = reviewer_actor_login(c.get("user") or {})
comment = thread_comment(c.get("updated_at") or c.get("created_at") or "", actor, author, reviewers, c.get("body") or "")
comment = discussion_comment(c.get("updated_at") or c.get("created_at") or "", actor, author, reviewers, c.get("body") or "")
if comment["timestamp"] and comment["actor_role"] != "bot" and comment["body"]:
comments.append(comment)
# GitHub renders top-level review bodies inline in the PR conversation,
Expand All @@ -595,7 +595,7 @@ def group_pr_conversation(
if not body:
continue
actor = reviewer_actor_login(r.get("user") or {})
comment = thread_comment(
comment = discussion_comment(
r.get("submitted_at") or "", actor, author, reviewers, f"[review: {state}] {body}",
)
if comment["timestamp"] and comment["actor_role"] != "bot":
Expand All @@ -616,10 +616,10 @@ def group_pr_conversation(

if facts.get("conflicts") == "no":
selected = [c for c in selected if not is_conflict_resolution_comment(c.get("body") or "")]
selected = selected[-THREAD_RECENT_COMMENTS_LIMIT:]
selected = selected[-DISCUSSION_RECENT_COMMENTS_LIMIT:]
if not selected:
return []
return [add_thread_facts({
return [add_discussion_facts({
"thread_id": "pr-conversation",
"thread_kind": "pr-conversation",
"path": None,
Expand All @@ -629,7 +629,7 @@ def group_pr_conversation(
}, selected, facts)]


def group_discussion_threads(
def group_discussions(
raw: dict[str, Any],
events: list[dict[str, Any]],
author: str,
Expand All @@ -643,7 +643,7 @@ def group_discussion_threads(
# ---------------------------------------------------------------- routing


ROUTE_THREAD_ACTIONS = {
ROUTE_DISCUSSION_ACTIONS = {
"author": "author",
"approver": "reviewer",
"maintainer": "reviewer",
Expand All @@ -654,14 +654,14 @@ def group_discussion_threads(
def action_counts(classifications: list[dict[str, Any]]) -> dict[str, int]:
counts = {"author": 0, "reviewer": 0, "external": 0, "none": 0, "unclear": 0}
for c in classifications:
action = normalize_thread_action((c.get("decision") or {}).get("thread_action") or "")
action = normalize_discussion_action((c.get("decision") or {}).get("thread_action") or "")
counts[action] += 1
return counts


def has_blocking_review_thread(classifications: list[dict[str, Any]]) -> bool:
def has_blocking_discussion(classifications: list[dict[str, Any]]) -> bool:
for c in classifications:
action = normalize_thread_action((c.get("decision") or {}).get("thread_action") or "")
action = normalize_discussion_action((c.get("decision") or {}).get("thread_action") or "")
if action in ("reviewer", "unclear") and c.get("thread_kind") != "pr-conversation":
return True
return False
Expand All @@ -685,32 +685,32 @@ def route_pr(facts: dict[str, Any], classifications: list[dict[str, Any]], requi
return "author"
if counts["external"]:
return "external"
if facts.get("approval_count", 0) >= approval_threshold and not has_blocking_review_thread(classifications):
if facts.get("approval_count", 0) >= approval_threshold and not has_blocking_discussion(classifications):
return "maintainer"
return "approver"


def threads_by_id(threads: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
def discussions_by_id(threads: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {t["thread_id"]: t for t in threads}


def thread_latest_comment_ts(thread: dict[str, Any] | None) -> datetime | None:
def discussion_latest_comment_ts(thread: dict[str, Any] | None) -> datetime | None:
comments = (thread or {}).get("comments") or []
if not comments:
return None
return parse_ts(comments[-1].get("timestamp") or "")


def oldest_thread_wait_ts(
def oldest_discussion_wait_ts(
threads: list[dict[str, Any]],
classifications: list[dict[str, Any]],
action: str,
) -> datetime | None:
by_id = threads_by_id(threads)
by_id = discussions_by_id(threads)
timestamps = [
thread_latest_comment_ts(by_id.get(c.get("thread_id") or ""))
discussion_latest_comment_ts(by_id.get(c.get("thread_id") or ""))
for c in classifications
if normalize_thread_action((c.get("decision") or {}).get("thread_action") or "") == action
if normalize_discussion_action((c.get("decision") or {}).get("thread_action") or "") == action
]
timestamps = [ts for ts in timestamps if ts is not None]
return min(timestamps) if timestamps else None
Expand All @@ -732,8 +732,8 @@ def add_wait_age_facts(
threads: list[dict[str, Any]],
classifications: list[dict[str, Any]],
) -> None:
action = ROUTE_THREAD_ACTIONS.get(route)
wait_ts = oldest_thread_wait_ts(threads, classifications, action) if action else None
action = ROUTE_DISCUSSION_ACTIONS.get(route)
wait_ts = oldest_discussion_wait_ts(threads, classifications, action) if action else None
basis = "oldest_pending_thread" if wait_ts else ""
if wait_ts is None:
wait_ts, basis = fallback_wait_ts(route, facts)
Expand All @@ -747,22 +747,22 @@ def add_wait_age_facts(
# Thread actions that count as an open, unresolved discussion. A reviewer who
# commented in such a thread is not yet satisfied, even if they have approved.
# "none" means no follow-up is needed, so it does not block a clear check.
OPEN_THREAD_ACTIONS = {"author", "reviewer", "external", "unclear"}
OPEN_DISCUSSION_ACTIONS = {"author", "reviewer", "external", "unclear"}


def reviewers_with_open_threads(
threads: list[dict[str, Any]],
classifications: list[dict[str, Any]],
) -> set[str]:
by_id = threads_by_id(threads)
by_id = discussions_by_id(threads)
logins: set[str] = set()
for c in classifications:
# The synthetic PR conversation contributes to the PR's routing bucket,
# but it is not a reviewer-owned discussion thread for badges.
if c.get("thread_kind") == "pr-conversation":
continue
action = normalize_thread_action((c.get("decision") or {}).get("thread_action") or "")
if action not in OPEN_THREAD_ACTIONS:
action = normalize_discussion_action((c.get("decision") or {}).get("thread_action") or "")
if action not in OPEN_DISCUSSION_ACTIONS:
continue
thread = by_id.get(c.get("thread_id") or "")
if not thread:
Expand Down Expand Up @@ -833,8 +833,8 @@ def build_pr_result(
author = effective_author(raw)
events = normalize_events(raw, author, reviewers)
facts = compute_facts(raw, author, events)
threads = group_discussion_threads(raw, events, author, reviewers, facts)
classifications = classify_threads(number, threads, model)
threads = group_discussions(raw, events, author, reviewers, facts)
classifications = classify_discussions(number, threads, model)
failed_classifications = [c for c in classifications if c.get("failed")]
if failed_classifications:
return {
Expand Down