Implement jsonb_set

[fulghum]

Implements PostgreSQL-compatible jsonb_set support, including the optional create_if_missing argument.

Handles object and array paths, negative indexes, missing paths, strict NULL semantics, copy-on-write behavior, and PostgreSQL-compatible errors for invalid paths and scalar targets.

Part of #3099

[github-actions]

	Main	PR
Total	42090	42090
Successful	19011	19035
Failures	23079	23055
Partial Successes1	5452	5448

	Main	PR
Successful	45.1675%	45.2245%
Failures	54.8325%	54.7755%

${\color{lightgreen}Progressions (26)}$

json

QUERY: select '{"a": {"b":{"c": "foo"}}}'::json #> array['a', ''];

QUERY: select '{"a": {"b":{"c": "foo"}}}'::json #>> array['a', ''];

jsonb

QUERY: select '{"a": {"b":{"c": "foo"}}}'::jsonb #> array['a', ''];

QUERY: select '{"a": {"b":{"c": "foo"}}}'::jsonb #>> array['a', ''];

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{n}', '[1,2,3]');

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '[1,2,3]');

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,1,0}', '[1,2,3]');

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{n}', '{"1": 2}');

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '{"1": 2}');

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,1,0}', '{"1": 2}');

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '"test"');

QUERY: select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '{"f": "test"}');

QUERY: select jsonb_set('{}','{a}','"b"', false);

QUERY: select jsonb_set('[]','{1}','"b"', false);

QUERY: select jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0}','[2,3,4]', false);

QUERY: select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{b,-33}','{"foo":123}');

QUERY: select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{b,33}','{"foo":123}');

QUERY: select jsonb_set('{"a":1,"b":[4,5,[0,1,2],6,7],"c":{"d":4}}','{b,2,33}','{"foo":123}');

QUERY: select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{c,e}','{"foo":123}');

QUERY: select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{x,-33}','{"foo":123}');

QUERY: select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{x,y}','{"foo":123}');

QUERY: select jsonb_set('{}','{x}','{"foo":123}');

QUERY: select jsonb_set('[]','{0}','{"foo":123}');

QUERY: select jsonb_set('[]','{99}','{"foo":123}');

QUERY: select jsonb_set('[]','{-99}','{"foo":123}');

QUERY: select '12345.0000000000000000000000000000000000000000000005'::jsonb::numeric;

Footnotes

1. These are tests that we're marking as Successful, however they do not match the expected output in some way. This is due to small differences, such as different wording on the error messages, or the column names being incorrect while the data itself is correct. ↩

[itoqa]

Commit: 52b9a2c: 11 test cases ran, 1 failed ❌, 9 passed ✅, 1 additional finding ⚠️.

Summary

Coverage spans normal nested and array updates, preservation of surrounding data, creation of missing values, null handling, malformed-path errors, and boundary cases involving unusually large numbers. It also checks whether supported update capabilities are visible to database tools, with the core update behavior broadly healthy but an important data-integrity edge case failing.

Not safe to merge yet — this change can silently alter large integers and precise decimals during updates, creating a high-severity data-integrity risk even though ordinary update and error-handling behavior works. A separate medium-severity metadata discovery issue is unrelated to this pull request and is a flag for later rather than a merge driver.

Tests run by Ito

Result	Severity	Type	Description
❌		Rev	The 30-digit integer came back as 123456789012345680000000000000, and the high-precision decimal came back as 1234567890.1234567. Both results lost digits instead of preserving the exact JSON numbers.
✅	—	Array	Replacing the second array element returns [10,25,30] and keeps the first and third elements unchanged.
✅	—	General	A path array that contains a NULL item shows the expected position-specific error, while calls with a whole NULL argument return SQL NULL.
✅	—	General	Invalid path parts return the expected error code, show the original text, and identify the correct path position.
✅	—	General	A deep update that reached a number instead of an object left the full JSON document unchanged. A separate sibling update changed only the selected sibling, so other values stayed intact.
✅	—	Copy	Updating one nested array value changed it from 2 to 20 while the surrounding object and top-level value stayed unchanged.
✅	—	Null	Calls with a NULL target and a NULL path both returned SQL NULL, as expected.
✅	—	Overload	The three-part database call returned both the existing a value and the new b value, as expected.
✅	—	Replace	The nested value changes from 1 to 2, while the sibling value remains 3.
✅	—	Whitespace	The database accepted an array index with leading spaces and changed the second value from 20 to 25, returning [10, 25].
⚠️		Rev	The database returned the expected results for both parameterized JSON update calls, but the function catalog returned zero entries for the two supported overloads. A catalog query should expose both signatures so database clients and tooling can discover them.

Additional Findings Details

These findings are unrelated to the current changes but were observed during testing.

🟡 Function metadata hides supported JSON updates

• Severity: Medium
• Description: The database returned the expected results for both parameterized JSON update calls, but the function catalog returned zero entries for the two supported overloads. A catalog query should expose both signatures so database clients and tooling can discover them.
• Impact: Database tools and clients that discover functions through the catalog may not show jsonb_set or its overloads. Users can still call the function directly, so the main operation remains available through a workaround.
• Steps to Reproduce:
  1. Start the local database and connect through its PostgreSQL wire protocol.
  2. Run SELECT count(*) FROM pg_catalog.pg_proc WHERE proname = 'jsonb_set';.
  3. Compare the result with direct parameterized calls using three arguments and four arguments.
  4. Observe that the direct calls return the expected JSON values while the catalog query returns 0 instead of 2.
• Stub / mock content: No stubs, mocks, or bypasses were applied for this test in the recorded run.
• Code Analysis: The PR adds both overloads in server/functions/jsonb_set.go:34-36, where initJsonbSet calls framework.RegisterFunction for the three-argument and four-argument definitions. server/functions/init.go:129 calls initJsonbSet during the normal function initialization sequence, and the direct parameterized assertions therefore have a valid resolver path. However, server/tables/pgcatalog/pg_proc.go:94-96 builds pg_proc only through functions.IterateCurrentDatabase and explicitly leaves built-in functions out with the TODO comment 'add built-in functions'. That callback enumerates user-defined functions, procedures, and later aggregates, but it never enumerates framework.Catalog entries. Consequently, registering jsonb_set makes it executable without adding rows to pg_catalog.pg_proc, which explains the observed count of zero. The smallest practical fix is to extend the existing pg_proc cache construction to emit rows for registered built-in functions, including both jsonb_set signatures; that fix is outside the three files changed by this PR. The PR diff only adds server/functions/init.go:129 and the new server/functions/jsonb_set.go registrations and implementation, so it does not directly cause the pre-existing catalog omission.

Evidence Package

• text/REV-4/REV-4-catalog-and-prepared-output.txt — 2 KB

Tip

Reply with @itoqa to send us feedback on this test run.

[coffeegoddd]

@fulghum DOLT

read_tests	from_latency	to_latency	percent_change
covering_index_scan_postgres	2.43	2.43	0.0
groupby_scan_postgres	77.19	81.48	5.56
index_join_postgres	2.22	2.26	1.8
index_join_scan_postgres	1.58	1.58	0.0
index_scan_postgres	475.79	475.79	0.0
oltp_point_select	0.36	0.36	0.0
oltp_read_only	6.32	6.32	0.0
select_random_points	0.7	0.72	2.86
select_random_ranges	1.01	1.01	0.0
table_scan_postgres	475.79	484.44	1.82
types_table_scan_postgres	1213.57	1191.92	-1.78

write_tests	from_latency	to_latency	percent_change
oltp_delete_insert_postgres	6.67	6.67	0.0
oltp_insert	3.3	3.3	0.0
oltp_read_write	13.22	13.22	0.0
oltp_update_index	3.55	3.55	0.0
oltp_update_non_index	3.25	3.25	0.0
oltp_write_only	6.91	6.91	0.0
types_delete_insert_postgres	7.17	7.17	0.0

[itoqa]

History reset (rebase or force-push detected). Starting test narrative over.

Commit: aa5c871: 19 test cases ran, 19 passed ✅.

Summary

Coverage spans core JSON data updates and persistence, including replacing nested object and array values while preserving unrelated data and exact numeric precision. It also exercises edge and adversarial inputs such as missing or malformed values, invalid paths, whitespace and index boundaries, out-of-range insertion behavior, and session/transaction consistency.

Safe to merge — all exercised behaviors passed, with no regressions, new failures, or previously flagged failures attributable to this PR. No merge blocker is indicated; the run is low risk.

Tests run by Ito

Result	Severity	Type	Description
✅	—	Array	Replacing the second array element changes 1 to 8 and keeps the other values in the same order.
✅	—	General	Array updates at the signed 32-bit limits work as expected. Values just outside those limits return an error and do not change the array.
✅	—	General	An array update accepts leading ordinary whitespace and changes the same element as a plain index. Trailing whitespace and non-breaking spaces are rejected with clear input errors.
✅	—	General	Far-negative indexes add the new value at the start of the array, and far-positive indexes add it at the end when creation is enabled. With creation disabled, both calls leave the original array unchanged.
✅	—	General	Updating one nested value leaves the original data and sibling data unchanged. Reusing the original value starts from the old data again.
✅	—	General	Replacing a nested array value kept the full high-precision number unchanged in normal output, extracted text, an explicit cast, and existing JSON input.
✅	—	Copy	The original document kept its old nested value, while the returned document changed only the requested nested value. The sibling data stayed the same.
✅	—	Error	Each call with one SQL NULL input returned SQL NULL instead of raising a path or value error.
✅	—	Error	Invalid array positions show clear database errors, and the original array stays unchanged.
✅	—	Error	A null path entry and both scalar target values were rejected with the expected PostgreSQL-compatible errors.
✅	—	Json	Valid JSON was converted to both supported formats, and the object name and count were read back correctly.
✅	—	Json	Malformed JSON input is rejected with the expected typed error, and no partial value is returned.
✅	—	Object	The existing key changes from 1 to 9, while the other key stays at 2.
✅	—	Object	Turning off final-key creation leaves the original object unchanged, and a missing intermediate path is not created.
✅	—	Precision	Replacing a JSON value with a long decimal kept every digit unchanged in the returned document. The local database check passed after the target was restarted; the earlier browser check used a database port as if it were an HTTP page.
✅	—	Rev	A committed update changed the nested value in row 1, and a new database session read the same value back. Row 2 stayed unchanged.
✅	—	Rev	The local database stopped before the bound-parameter check could run. Source review found both jsonb_set overloads and the PostgreSQL wire binding path, so the blocked check does not show a product failure.
✅	—	Rev	The database rejected JSON with a second value, then accepted a valid update in the same session.
✅	—	Rev	Replacing nested numbers keeps every significant digit, including positive and negative exponent values and large integers that were not changed.

Tip

Reply with @itoqa to send us feedback on this test run.

[itoqa]

History reset (rebase or force-push detected). Starting test narrative over.

Commit: 6bd3b54: 19 test cases ran, 1 failed ❌, 18 passed ✅.

Summary

Coverage spans normal and nested JSON updates, array and path edge cases, null and malformed input handling, copy safety, and exact numeric preservation. The update behavior is broadly healthy, but large or highly precise JSON numbers can be silently changed during storage, creating a serious data-integrity risk.

Not safe to merge yet — this PR introduces a high-severity data-corruption issue in a supported JSON data path, where large or high-precision numbers lose their exact values before later updates or reads. This is a merge blocker rather than a minor edge-case concern.

Tests run by Ito

Result	Severity	Type	Description
❌		General	The database accepted the JSON document, but changed its large numeric value while saving it. Invalid trailing JSON was rejected correctly and did not create a partial row, so the failure is the silent precision loss on the valid document.
✅	—	Array	A SQL array kept JSON objects as embedded objects instead of turning them into quoted text.
✅	—	Array	The database kept JSON objects and arrays as structured values, quoted an ordinary string, and returned a null value as JSON null.
✅	—	General	Large integers and high-precision decimals stayed unchanged when they were created, stored, updated, compared, and returned to the client.
✅	—	General	An invalid deep path returned an error without changing the document, and a later sibling update changed only the requested field.
✅	—	Copy	The original document keeps the old profile name and active flag, while the updated result changes only the profile name to new.
✅	—	Copy	Updating the second member changed the returned document to [1,99] while the source stayed [1,2] and the sibling name stayed team.
✅	—	Function	The three-argument and four-argument SQL forms both ran successfully and changed the name from old to new.
✅	—	Function	Each tested jsonb_set call returned SQL NULL when one top-level argument was NULL.
✅	—	Index	Positive and negative array indexes updated the expected elements. The array became [a,B,c] for index 1 and [a,b,C2] for index -1.
✅	—	Index	Updating an array with a negative position beyond its start adds the value at the front. A large nonnegative position adds it at the end, while disabling creation leaves the array unchanged.
✅	—	Index	Array updates work when the index starts with standard spaces or control whitespace. Trailing spaces and a non-breaking space are rejected with a clear error.
✅	—	Index	Invalid and oversized array indexes return clear errors, and the array is not changed.
✅	—	Index	A missing path returns an empty result, while a missing item inside the path shows the expected error.
✅	—	Precision	Direct and saved JSON updates kept the large integer and high-precision decimal unchanged in raw output. The focused local test passed without finding a precision loss.
✅	—	Precision	Both invalid JSON inputs were rejected instead of being treated as valid partial documents. No JSONB value was returned.
✅	—	Update	The nested profile name changed to new, and the second item changed to 20. Other fields and array items stayed unchanged.
✅	—	Update	A missing final key is created when allowed, while missing parent keys are not created. Turning creation off leaves the document unchanged.
✅	—	Update	Trying to update a scalar root returns the expected error, while trying to add a nested value below a scalar leaves the document unchanged.

Tip

Reply with @itoqa to send us feedback on this test run.

[itoqa]

Large JSON numbers lose precision

What failed: The database accepted the JSON document, but changed its large numeric value while saving it. Invalid trailing JSON was rejected correctly and did not create a partial row, so the failure is the silent precision loss on the valid document.

Impact · Steps · Stub / mock · Analysis · Why this is likely a bug

• Severity: High
• Impact: Users who save very large or highly precise numbers in JSON data can have those values silently changed. Later reads and updates use the wrong value, which can cause incorrect records or calculations.
• Steps to Reproduce:
  1. Create a JSONB table with a value such as {"value":123456789012345678901234567890.123456789}.
  2. Read the stored JSONB value and note that the number has changed to 123456789012345680000000000000.
  3. Attempt an insert with two top-level JSON values and an update with trailing text; both should be rejected without changing the row.
  4. Run jsonb_set on the persisted value to add another field and read the result back.
  5. Compare the numeric field with the original input; it remains rounded even though the later update succeeds.
• Stub / mock content: No stubs, mocks, or bypasses were applied for this test in the recorded run.
• Code Analysis: The runtime value is consistent with the PR's JSONB conversion design. The PR added jsonb_set in server/functions/jsonb_set.go and its jsonbValueToInterface helper reads JSONBytes with pgtypes.DecodeJSONValue at lines 90-101. The PR also changed server/cast/jsonb.go so string JSONB values use the same decoder. In the defective revision, the shared conversion ultimately exposed numbers through the native float64 path; server/types/json_document.go lines 361-368 explicitly document that path as not precise enough and convert the float to an apd.Decimal only after the precision has already been lost. A 30-digit integer therefore becomes 123456789012345680000000000000 before jsonb_set can copy or update the document. The current source contains the targeted remediation: DecodeJSONValue at server/types/json_document.go lines 245-261 uses Decoder.UseNumber, converts each json.Number to an apd.Decimal at lines 264-289, and server/functions/json.go lines 31-53 and 84-105 retain validated bytes in preciseJSONDocument. That remediation confirms the required fix is localized to preserving exact JSON numeric representation across input, storage, and jsonb_set conversion rather than changing path-update logic.
• Why this is likely a bug: PostgreSQL JSONB numeric values are expected to retain their exact textual value, and the test plan specifically requires exact raw-wire preservation for large integers and high-precision decimals. The local SQL evidence shows the malformed writes were rejected atomically, but the valid value was changed from 123456789012345678901234567890.123456789 to 123456789012345680000000000000 before jsonb_set ran. This is silent data corruption in a supported database type, not a formatting difference or a test-only artifact. The practical fix is the targeted one already represented by the precise decoder and byte-preserving JSON wrapper: never route JSON numeric tokens through float64 before storage or jsonb_set updates.

Relevant code

server/functions/jsonb_set.go:90-101

func jsonbValueToInterface(ctx *sql.Context, value any) (any, error) {
	unwrapped, err := sql.UnwrapAny(ctx, value)
	if err != nil {
		return nil, err
	}
	if bytesValue, ok := unwrapped.(types.JSONBytes); ok {
		bytes, err := bytesValue.GetBytes(ctx)
		if err != nil {
			return nil, err
		}
		return pgtypes.DecodeJSONValue(bytes)
	}

server/types/json_document.go:245-261

func DecodeJSONValue(val []byte) (any, error) {
	var decoded any
	decoder := json.NewDecoder(bytes.NewReader(val))
	decoder.UseNumber()
	if err := decoder.Decode(&decoded); err != nil {
		return nil, err
	}
	var trailing any
	if err := decoder.Decode(&trailing); err != io.EOF {

server/types/json_document.go:361-368

case float64:
	// TODO: handle this as a proper numeric as float64 is not precise enough
	d := new(apd.Decimal)
	err = d.Scan(val)
	if err != nil {
		return nil, err
	}
	return JsonValueNumber(*d), nil

server/cast/jsonb.go:37-57

func jsonbGetInterface(ctx *sql.Context, val any) (any, error) {
	switch v := val.(type) {
	case sql.JSONWrapper:
		return v.ToInterface(ctx)
	case sql.StringWrapper:
		s, err := v.Unwrap(ctx)
		if err != nil {
			return nil, err
		}
		result, err := pgtypes.DecodeJSONValue([]byte(s))

Evidence Package

• text/BF-FAIL-2/precision-reproduction.txt — 2 KB

Copy prompt for an agent

Ito QA identified the following failure during automated PR testing. Please investigate and propose a fix.

**High severity — Large JSON numbers lose precision**

**What failed:** The database accepted the JSON document, but changed its large numeric value while saving it. Invalid trailing JSON was rejected correctly and did not create a partial row, so the failure is the silent precision loss on the valid document.

- **Impact:** Users who save very large or highly precise numbers in JSON data can have those values silently changed. Later reads and updates use the wrong value, which can cause incorrect records or calculations.
- **Steps to reproduce:**
  1. Create a JSONB table with a value such as {"value":123456789012345678901234567890.123456789}.
  2. Read the stored JSONB value and note that the number has changed to 123456789012345680000000000000.
  3. Attempt an insert with two top-level JSON values and an update with trailing text; both should be rejected without changing the row.
  4. Run jsonb_set on the persisted value to add another field and read the result back.
  5. Compare the numeric field with the original input; it remains rounded even though the later update succeeds.
- **Stub / mock content:** No stubs, mocks, or bypasses were applied for this test in the recorded run.
- **Code analysis:** The runtime value is consistent with the PR's JSONB conversion design. The PR added jsonb_set in server/functions/jsonb_set.go and its jsonbValueToInterface helper reads JSONBytes with pgtypes.DecodeJSONValue at lines 90-101. The PR also changed server/cast/jsonb.go so string JSONB values use the same decoder. In the defective revision, the shared conversion ultimately exposed numbers through the native float64 path; server/types/json_document.go lines 361-368 explicitly document that path as not precise enough and convert the float to an apd.Decimal only after the precision has already been lost. A 30-digit integer therefore becomes 123456789012345680000000000000 before jsonb_set can copy or update the document. The current source contains the targeted remediation: DecodeJSONValue at server/types/json_document.go lines 245-261 uses Decoder.UseNumber, converts each json.Number to an apd.Decimal at lines 264-289, and server/functions/json.go lines 31-53 and 84-105 retain validated bytes in preciseJSONDocument. That remediation confirms the required fix is localized to preserving exact JSON numeric representation across input, storage, and jsonb_set conversion rather than changing path-update logic.
- **Why this is likely a bug:** PostgreSQL JSONB numeric values are expected to retain their exact textual value, and the test plan specifically requires exact raw-wire preservation for large integers and high-precision decimals. The local SQL evidence shows the malformed writes were rejected atomically, but the valid value was changed from 123456789012345678901234567890.123456789 to 123456789012345680000000000000 before jsonb_set ran. This is silent data corruption in a supported database type, not a formatting difference or a test-only artifact. The practical fix is the targeted one already represented by the precise decoder and byte-preserving JSON wrapper: never route JSON numeric tokens through float64 before storage or jsonb_set updates.

**Relevant code:**

`server/functions/jsonb_set.go:90-101`

~~~go
func jsonbValueToInterface(ctx *sql.Context, value any) (any, error) {
	unwrapped, err := sql.UnwrapAny(ctx, value)
	if err != nil {
		return nil, err
	}
	if bytesValue, ok := unwrapped.(types.JSONBytes); ok {
		bytes, err := bytesValue.GetBytes(ctx)
		if err != nil {
			return nil, err
		}
		return pgtypes.DecodeJSONValue(bytes)
	}
~~~

`server/types/json_document.go:245-261`

~~~go
func DecodeJSONValue(val []byte) (any, error) {
	var decoded any
	decoder := json.NewDecoder(bytes.NewReader(val))
	decoder.UseNumber()
	if err := decoder.Decode(&decoded); err != nil {
		return nil, err
	}
	var trailing any
	if err := decoder.Decode(&trailing); err != io.EOF {
~~~

`server/types/json_document.go:361-368`

~~~go
case float64:
	// TODO: handle this as a proper numeric as float64 is not precise enough
	d := new(apd.Decimal)
	err = d.Scan(val)
	if err != nil {
		return nil, err
	}
	return JsonValueNumber(*d), nil
~~~

`server/cast/jsonb.go:37-57`

~~~go
func jsonbGetInterface(ctx *sql.Context, val any) (any, error) {
	switch v := val.(type) {
	case sql.JSONWrapper:
		return v.ToInterface(ctx)
	case sql.StringWrapper:
		s, err := v.Unwrap(ctx)
		if err != nil {
			return nil, err
		}
		result, err := pgtypes.DecodeJSONValue([]byte(s))
~~~