diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index a973320b4..56843ecc3 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -1189,10 +1189,14 @@ static bool is_comment_node(const char *kind) { } // Extract comment text, truncating to MAX_COMMENT_LEN. +// #1017: snap the cut point back to a complete UTF-8 codepoint boundary. static char *extract_comment_text(CBMArena *a, TSNode node, const char *source) { char *text = cbm_node_text(a, node, source); if (text && strlen(text) > MAX_COMMENT_LEN) { - text[MAX_COMMENT_LEN] = '\0'; + size_t cut = MAX_COMMENT_LEN; + while (cut > 0 && ((unsigned char)text[cut] & 0xC0) == 0x80) + cut--; + text[cut] = '\0'; } return text; } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index fb2b6c795..8cfc565fb 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -3559,6 +3559,81 @@ TEST(extract_rust_test_attr_marks_is_test_issue855) { PASS(); } +/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) can split a + * multi-byte UTF-8 character, leaving an incomplete byte sequence. + * Craft a Go comment whose 498th-500th bytes are a 3-byte CJK character + * (U+6210 = 成 = e6 88 90). The raw byte truncation at offset 500 lands + * one byte past the character start, splitting it. After the fix the + * truncated string must end on a complete codepoint boundary. */ +TEST(docstring_utf8_truncation_boundary_issue1017) { + /* Build a comment: "// " (3 bytes) + 495 ASCII 'A' + "成成成" (9 bytes) + * Total comment text = 3 + 495 + 9 = 507 bytes. + * MAX_COMMENT_LEN = 500. The first kanji (成 = e6 88 90) occupies + * offsets 498-500, so text[500] = '\0' keeps bytes 0-499: the lead + * byte 0xe6 plus one continuation 0x88 — an incomplete 2-of-3 sequence. + * Before fix: the truncated string ended with that broken pair. */ + char comment[600]; + int off = 0; + comment[off++] = '/'; + comment[off++] = '/'; + comment[off++] = ' '; + for (int i = 0; i < 495; i++) + comment[off++] = 'A'; + /* U+6210 (成) = 0xe6 0x88 0x90 — 3-byte UTF-8 */ + const char *kanji = "\xe6\x88\x90"; + for (int k = 0; k < 3; k++) { + memcpy(comment + off, kanji, 3); + off += 3; + } + comment[off] = '\0'; + + /* Wrap in a Go function so the comment becomes the docstring. */ + char src[800]; + snprintf(src, sizeof(src), "package main\n\n%s\nfunc Compute() {}\n", comment); + + CBMFileResult *r = extract(src, CBM_LANG_GO, "test", "main.go"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Function", "Compute")); + + const char *doc = NULL; + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].name, "Compute") == 0) { + doc = r->defs.items[i].docstring; + break; + } + } + ASSERT_NOT_NULL(doc); + + /* Verify every byte in the truncated docstring is valid UTF-8: + * no trailing incomplete multi-byte sequence. */ + size_t len = strlen(doc); + ASSERT_TRUE(len <= 500); + const unsigned char *u = (const unsigned char *)doc; + size_t i = 0; + while (i < len) { + unsigned char c = u[i]; + int seq_len; + if (c < 0x80) + seq_len = 1; + else if ((c & 0xE0) == 0xC0) + seq_len = 2; + else if ((c & 0xF0) == 0xE0) + seq_len = 3; + else if ((c & 0xF8) == 0xF0) + seq_len = 4; + else + FAIL("invalid UTF-8 lead byte"); + ASSERT_TRUE(i + (size_t)seq_len <= len); + for (int j = 1; j < seq_len; j++) + ASSERT_TRUE((u[i + (size_t)j] & 0xC0) == 0x80); + i += (size_t)seq_len; + } + + cbm_free_result(r); + PASS(); +} + /* Reproduce-first (ms-typescript reallyLargeFile.ts, 2026-07-07): a file * whose root node has hundreds of thousands of FLAT SIBLINGS (580k //// * comment lines in the 3.5 MB fourslash fixture) hung extraction for over @@ -4680,6 +4755,7 @@ SUITE(extraction) { RUN_TEST(extract_c_clean_file_no_recovery_duplicates_issue961); RUN_TEST(walk_defs_no_truncation_over_4096_issue668); RUN_TEST(extract_rust_test_attr_marks_is_test_issue855); + RUN_TEST(docstring_utf8_truncation_boundary_issue1017); cbm_shutdown(); }