From 714d594e552f52f539e0087921a02158473c8ba0 Mon Sep 17 00:00:00 2001
From: Tim <code@tim.life>
Date: Tue, 18 Aug 2026 11:15:21 -0700
Subject: [PATCH] fix: parse heredocs with an empty body (#309)

`a = <<EOF\nEOF` failed to parse. The v8 rewrite turned the optional
newline after the opening marker into a mandatory one, so a marker
immediately followed by its closing delimiter no longer matched the
HEREDOC_TEMPLATE terminal.

The failure is not confined to the empty heredoc. Having failed to match
at that position, the lexer scans on and matches a *later* delimiter, so
an empty heredoc followed by more attributes silently swallows them:

    locals {
      a = <<EOF
    EOF

      b = <<EOF
    real body
    EOF
    }

serialized to a single attribute `a` whose value ran to the end of `b`,
with `b` absent from the output. Only a file whose empty heredoc had no
later delimiter to latch onto surfaced as a parse error.

Make the body and its trailing newline a single optional group rather
than restoring the bare `\n?` the rewrite dropped. The group has to be
lazy: a greedy one prefers to match a body, which reintroduces the
run-on above. The delimiter still has to begin its own line, so #194's
rule holds and a word merely ending in the delimiter does not terminate
the body.

Cover empty, empty-trimmed and blank-line-only heredocs in the
specialized suite, along with an attribute after them that proves the
lexer stops where it should. `test/helpers/terraform-config/` carried an
empty-heredoc fixture for this before the v8 test reorganisation; nothing
in the current suite replaced it. Without the fix, 6 of the new tests
fail.
---
 CHANGELOG.md                                  |  1 +
 hcl2/hcl2.lark                                |  4 +-
 test/integration/specialized/heredocs.tf      | 12 +++++
 .../specialized/heredocs_flattened.json       |  4 ++
 .../specialized/heredocs_preserved.json       |  4 ++
 .../specialized/heredocs_restored.tf          |  4 ++
 test/unit/test_api.py                         | 44 +++++++++++++++++++
 7 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5b0148ee..1b8f8fed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Fixed
 
 - Restore `py.typed` marker so type checkers recognize `hcl2` (and `cli`) as typed packages. ([#298](https://github.com/amplify-education/python-hcl2/issues/298))
+- Parse heredocs with an empty body again. A marker immediately followed by its closing delimiter failed to match, and the lexer then ran on to a later delimiter, silently absorbing the attributes in between. ([#309](https://github.com/amplify-education/python-hcl2/issues/309))
 
 ## \[8.1.2\] - 2026-04-10
 
diff --git a/hcl2/hcl2.lark b/hcl2/hcl2.lark
index 7603ec2c..ee875319 100644
--- a/hcl2/hcl2.lark
+++ b/hcl2/hcl2.lark
@@ -82,8 +82,8 @@ ELLIPSIS : "..."
 COLONS: "::"
 
 // Heredocs
-HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:.|\n)*?\n\s*(?P=heredoc)\n/
-HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:.|\n)*?\n\s*(?P=heredoc_trim)\n/
+HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:(?:.|\n)*?\n)??\s*(?P=heredoc)\n/
+HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:(?:.|\n)*?\n)??\s*(?P=heredoc_trim)\n/
 
 // Ignore whitespace (but not newlines, as they're significant in HCL)
 %ignore /[ \t]+/
diff --git a/test/integration/specialized/heredocs.tf b/test/integration/specialized/heredocs.tf
index 9fc16498..6a10897b 100644
--- a/test/integration/specialized/heredocs.tf
+++ b/test/integration/specialized/heredocs.tf
@@ -31,4 +31,16 @@ EOF
   json_content = <<EOF
 {"key": "value"}
 EOF
+
+  empty = <<EOF
+EOF
+
+  empty_trimmed = <<-EOF
+  EOF
+
+  blank_line_only = <<EOF
+
+EOF
+
+  after_empty = "still parsed"
 }
diff --git a/test/integration/specialized/heredocs_flattened.json b/test/integration/specialized/heredocs_flattened.json
index a8c8b945..95fb4e55 100644
--- a/test/integration/specialized/heredocs_flattened.json
+++ b/test/integration/specialized/heredocs_flattened.json
@@ -8,6 +8,10 @@
       "trimmed": "\"indented1\\nindented2\"",
       "trimmed_mixed": "\"line1\\n  line2\\nline3\"",
       "json_content": "\"{\\\"key\\\": \\\"value\\\"}\"",
+      "empty": "\"\"",
+      "empty_trimmed": "\"\"",
+      "blank_line_only": "\"\"",
+      "after_empty": "\"still parsed\"",
       "__is_block__": true
     }
   ]
diff --git a/test/integration/specialized/heredocs_preserved.json b/test/integration/specialized/heredocs_preserved.json
index 6e8ee2f2..f9094838 100644
--- a/test/integration/specialized/heredocs_preserved.json
+++ b/test/integration/specialized/heredocs_preserved.json
@@ -8,6 +8,10 @@
       "trimmed": "\"<<-EOF\n    indented1\n    indented2\n  EOF\"",
       "trimmed_mixed": "\"<<-EOF\n    line1\n      line2\n    line3\n  EOF\"",
       "json_content": "\"<<EOF\n{\"key\": \"value\"}\nEOF\"",
+      "empty": "\"<<EOF\nEOF\"",
+      "empty_trimmed": "\"<<-EOF\n  EOF\"",
+      "blank_line_only": "\"<<EOF\n\nEOF\"",
+      "after_empty": "\"still parsed\"",
       "__is_block__": true
     }
   ]
diff --git a/test/integration/specialized/heredocs_restored.tf b/test/integration/specialized/heredocs_restored.tf
index 5d07e1b0..05832d52 100644
--- a/test/integration/specialized/heredocs_restored.tf
+++ b/test/integration/specialized/heredocs_restored.tf
@@ -17,4 +17,8 @@ line1
 line3
 EOF
   json_content     = "{\"key\": \"value\"}"
+  empty            = ""
+  empty_trimmed    = ""
+  blank_line_only  = ""
+  after_empty      = "still parsed"
 }
diff --git a/test/unit/test_api.py b/test/unit/test_api.py
index 6af029a5..8814acbd 100644
--- a/test/unit/test_api.py
+++ b/test/unit/test_api.py
@@ -288,3 +288,47 @@ def test_query_file_object(self):
         self.assertIsInstance(result, DocumentView)
         attr = result.attribute("x")
         self.assertIsNotNone(attr)
+
+
+class TestEmptyHeredocs(TestCase):
+    """A heredoc with no body parses, and does not swallow what follows.
+
+    The v8 grammar made the newline before the closing delimiter mandatory, so
+    a marker immediately followed by its delimiter could not match. The lexer
+    then scanned on to a later delimiter, which silently absorbed the
+    intervening attributes rather than reporting an error.
+    """
+
+    def test_empty_heredoc_parses(self):
+        self.assertEqual(loads("a = <<EOF\nEOF\n"), {"a": '"<<EOF\nEOF"'})
+
+    def test_empty_trimmed_heredoc_parses(self):
+        self.assertEqual(loads("a = <<-EOF\n  EOF\n"), {"a": '"<<-EOF\n  EOF"'})
+
+    def test_empty_heredoc_flattens_to_empty_string(self):
+        options = SerializationOptions(preserve_heredocs=False)
+        self.assertEqual(loads("a = <<EOF\nEOF\n", serialization_options=options), {"a": '""'})
+
+    def test_empty_heredoc_does_not_swallow_following_attributes(self):
+        source = "a = <<EOF\nEOF\n\nb = <<EOF\nreal body\nEOF\n\nc = 1\n"
+        self.assertEqual(
+            loads(source),
+            {"a": '"<<EOF\nEOF"', "b": '"<<EOF\nreal body\nEOF"', "c": 1},
+        )
+
+    def test_heredoc_with_a_blank_body_line_is_distinct_from_empty(self):
+        self.assertEqual(loads("a = <<EOF\n\nEOF\n"), {"a": '"<<EOF\n\nEOF"'})
+
+    def test_delimiter_must_start_its_own_line(self):
+        """A word merely ending in the delimiter does not terminate the body."""
+        self.assertEqual(loads("a = <<EOF\nsayEOF\nEOF\n"), {"a": '"<<EOF\nsayEOF\nEOF"'})
+
+    def test_body_containing_the_delimiter_as_a_prefix(self):
+        self.assertEqual(loads("a = <<EOF\nEOF_NOT\nEOF\n"), {"a": '"<<EOF\nEOF_NOT\nEOF"'})
+
+    def test_consecutive_heredocs_stay_separate(self):
+        source = "a = <<EOF\nx\nEOF\nb = <<EOF\ny\nEOF\n"
+        self.assertEqual(loads(source), {"a": '"<<EOF\nx\nEOF"', "b": '"<<EOF\ny\nEOF"'})
+
+    def test_indented_closing_delimiter_still_allowed(self):
+        self.assertEqual(loads("a = <<EOF\nx\n   EOF\n"), {"a": '"<<EOF\nx\n   EOF"'})