[GR-75457] Lazy native unicode data.

docs/contributor/IMPLEMENTATION_DETAILS.md

@@ -315,6 +315,24 @@ When a managed object is passed to a native extension code:
   We then clear the `PythonObjectReference.strongReference` field, and the
   memory management is then again left solely to the Java tracing GC.
 
+##### Unicode Objects
+
+Managed strings exposed to native code use `GraalPyUnicodeObject` as their
+native companion. Like CPython, GraalPy supports two allocation layouts:
+
+* **Compact:** The native allocation is over-allocated, and the character data
+  follows the structure at `sizeof(GraalPyUnicodeObject)`.
+* **Non-compact:** The structure's `data` field points to a separately allocated
+  buffer.
+
+Native character data is stored only in this companion, not duplicated in
+hidden managed attributes. `PyUnicode_New` delegates to
+`GraalPyPrivate_Unicode_New` and eagerly creates a compact companion because a
+native-created string is expected to need native data. In contrast, when an
+existing managed `PString` is passed to native code, its character data remains
+uninitialized until requested through the C API. GraalPy then allocates and
+initializes a non-compact buffer lazily.
+
 #### Native Objects
 
 Native objects allocated using `PyObject_GC_New` in the native code are backed

graalpython/com.oracle.graal.python.cext/include/cpython/unicodeobject.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2020, 2025, Oracle and/or its affiliates.
+/* Copyright (c) 2020, 2026, Oracle and/or its affiliates.
  * Copyright (C) 1996-2020 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -254,9 +254,9 @@ PyAPI_FUNC(int) GraalPyUnicode_KIND(PyObject*);
 #define PyUnicode_KIND(op) ((enum PyUnicode_Kind)GraalPyUnicode_KIND(_PyObject_CAST(op)))
 
 /* Return a void pointer to the raw unicode buffer. */
-static inline void* _PyUnicode_COMPACT_DATA(PyObject *Py_UNUSED(op)) {
-    // strings are never compact in GraalPy
-    return NULL;
+PyAPI_FUNC(void*) GraalPyUnicode_COMPACT_DATA(PyObject *op);
+static inline void* _PyUnicode_COMPACT_DATA(PyObject *op) {
+    return GraalPyUnicode_COMPACT_DATA(op);
 }
 
 PyAPI_FUNC(void*) GraalPyUnicode_NONCOMPACT_DATA(PyObject *op);

graalpython/com.oracle.graal.python.cext/src/unicodeobject.c

@@ -1221,12 +1221,9 @@ resize_compact(PyObject *unicode, Py_ssize_t length)
 }
 
 /*
- * A managed GraalPyUnicodeObject is physically compact: a single native
- * allocation contains its header and the character data immediately after it.
- * PyUnicode_IS_COMPACT nevertheless reports false because it denotes
- * CPython's PyASCIIObject/PyCompactUnicodeObject layout, which a handle-space
- * GraalPyUnicodeObject does not use. Its tagged handle and NativeMemory
- * allocation therefore cannot be passed to PyObject_Realloc.
+ * The unicode objects used by the writer are created using PyUnicode_New. Those
+ * objects are unmaterialized strings backed by a compact (i.e. over-allocated)
+ * GraalPyUnicodeObject. Therefore, it cannot be passed to PyObject_Realloc.
  *
  * A unicode writer owns an unmaterialized string and only needs to shrink its
  * logical size when it finishes. This function does not resize the allocation:
@@ -1241,7 +1238,7 @@ graalpy_resize_compact(PyObject *unicode, Py_ssize_t length)
     int kind;
 
     assert(unicode_modifiable(unicode));
-    assert(!PyUnicode_IS_COMPACT(unicode));
+    assert(PyUnicode_IS_COMPACT(unicode));
     assert(points_to_py_handle_space(unicode));
 
     native_unicode = (GraalPyUnicodeObject *)pointer_to_stub(unicode);
@@ -1421,27 +1418,35 @@ PyObject *
 PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
 {
     // GraalPy change: different implementation
-    if (size < 0) {
-        PyErr_SetString(PyExc_SystemError,
-                        "Negative size passed to PyUnicode_New");
-        return NULL;
-    }
+    int kind;
+    int is_ascii;
     if (maxchar < 128) {
-        return GraalPyPrivate_Unicode_New(size, PyUnicode_1BYTE_KIND, 1);
+        kind = PyUnicode_1BYTE_KIND;
+        is_ascii = 1;
     } else if (maxchar < 256) {
-        return GraalPyPrivate_Unicode_New(size, PyUnicode_1BYTE_KIND, 0);
+        kind = PyUnicode_1BYTE_KIND;
+        is_ascii = 0;
     } else if (maxchar < 65536) {
-        return GraalPyPrivate_Unicode_New(size, PyUnicode_2BYTE_KIND, 0);
+        kind = PyUnicode_2BYTE_KIND;
+        is_ascii = 0;
     } else {
         if (maxchar > MAX_UNICODE) {
             PyErr_SetString(PyExc_SystemError,
                             "invalid maximum character passed to PyUnicode_New");
             return NULL;
         }
-        return GraalPyPrivate_Unicode_New(size, PyUnicode_4BYTE_KIND, 0);
+        kind = PyUnicode_4BYTE_KIND;
+        is_ascii = 0;
     }
-    /* should never be reached */
-    return NULL;
+
+    /* Ensure we won't overflow the size. */
+    if (size < 0) {
+        PyErr_SetString(PyExc_SystemError,
+                        "Negative size passed to PyUnicode_New");
+        return NULL;
+    }
+
+    return GraalPyPrivate_Unicode_New(size, kind, is_ascii);
 }
 
 #if SIZEOF_WCHAR_T == 2
@@ -15384,6 +15389,32 @@ GraalPyUnicodeObject_GetKind(GraalPyUnicodeObject *unicode)
     return GraalPyUnicodeObject_GetKindFromState(unicode->state);
 }
 
+/* Keep in sync with CApiTransitions.isGraalPyUnicodeObjectCompactFromState. */
+static inline unsigned int
+GraalPyUnicodeObject_IsCompactFromState(uint64_t state)
+{
+    return (state & GRAALPY_UNICODE_IS_COMPACT_FLAG) != 0;
+}
+
+/* Keep in sync with CApiTransitions.isGraalPyUnicodeObjectCompactFromState. */
+static inline unsigned int
+GraalPyUnicodeObject_IsCompact(GraalPyUnicodeObject *unicode)
+{
+    return GraalPyUnicodeObject_IsCompactFromState(unicode->state);
+}
+
+static inline GraalPyUnicodeObject *
+GraalpyUnicodeObject_EnsureNativeData(PyObject *op) {
+    GraalPyUnicodeObject *raw = (GraalPyUnicodeObject *) pointer_to_stub(op);
+    /* 'kind == 0' is not a valid kind for any unicode object. We use it to indicate that
+     * the native data was not yet initialized. */
+    if (GraalPyUnicodeObject_GetKind(raw) == 0) {
+        GraalPyPrivate_Unicode_FillNativeData(op);
+    }
+    return raw;
+}
+
+
 unsigned int GraalPyUnicode_CHECK_INTERNED(PyObject *op) {
     if (points_to_py_handle_space(op)) {
         GraalPyUnicodeObject *unicode = (GraalPyUnicodeObject *) pointer_to_stub(op);
@@ -15399,39 +15430,50 @@ unsigned int GraalPyUnicode_CHECK_INTERNED(PyObject *op) {
 
 Py_ssize_t GraalPyUnicode_GET_LENGTH(PyObject* op) {
     if (points_to_py_handle_space(op)) {
+        /* The length is initialized eagerly when the managed object gets its native stub. */
         return ((GraalPyUnicodeObject *) pointer_to_stub(op))->length;
     }
     return _PyASCIIObject_CAST(op)->length;
 }
 
 unsigned int GraalPyUnicode_IS_ASCII(PyObject* op) {
     if (points_to_py_handle_space(op)) {
-        return GraalPyUnicodeObject_IsAscii((GraalPyUnicodeObject *) pointer_to_stub(op));
+        return GraalPyUnicodeObject_IsAscii(GraalpyUnicodeObject_EnsureNativeData(op));
     }
     return _PyASCIIObject_CAST(op)->state.ascii;
 }
 
 unsigned int GraalPyUnicode_IS_COMPACT(PyObject* op) {
     if (points_to_py_handle_space(op)) {
-        return 0;
+        return GraalPyUnicodeObject_IsCompact(GraalpyUnicodeObject_EnsureNativeData(op));
     }
     return _PyASCIIObject_CAST(op)->state.compact;
 }
 
 int GraalPyUnicode_KIND(PyObject* op) {
     if (points_to_py_handle_space(op)) {
-        return GraalPyUnicodeObject_GetKind((GraalPyUnicodeObject *) pointer_to_stub(op));
+        return GraalPyUnicodeObject_GetKind(GraalpyUnicodeObject_EnsureNativeData(op));
     }
     return _PyASCIIObject_CAST(op)->state.kind;
 }
 
 void* GraalPyUnicode_NONCOMPACT_DATA(PyObject* op) {
     if (points_to_py_handle_space(op)) {
-        return ((GraalPyUnicodeObject *) pointer_to_stub(op))->data;
+        return GraalpyUnicodeObject_EnsureNativeData(op)->data;
     }
     return _PyUnicodeObject_CAST(op)->data.any;
 }
 
+void* GraalPyUnicode_COMPACT_DATA(PyObject* op) {
+    if (points_to_py_handle_space(op)) {
+        return _Py_STATIC_CAST(void*, GraalpyUnicodeObject_EnsureNativeData(op) + 1);
+    }
+    if (PyUnicode_IS_ASCII(op)) {
+        return _Py_STATIC_CAST(void*, (_PyASCIIObject_CAST(op) + 1));
+    }
+    return _Py_STATIC_CAST(void*, (_PyCompactUnicodeObject_CAST(op) + 1));
+}
+
 #ifdef __cplusplus
 }
 #endif

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_unicode.py

@@ -1191,6 +1191,87 @@ def test_intern(self):
         assert tester.check_is_same_str_ptr(s2)
 
 
+    def test_intern_state_preserved_during_lazy_initialization(self):
+        TestLazyIntern = CPyExtType(
+            "TestLazyIntern",
+            '''
+            static PyObject* check_intern_state_after_data(PyObject* Py_UNUSED(self), PyObject* str) {
+                Py_INCREF(str);
+                PyUnicode_InternInPlace(&str);
+                if (str == NULL) {
+                    return NULL;
+                }
+
+                unsigned int before = PyUnicode_CHECK_INTERNED(str);
+                void *data = PyUnicode_DATA(str);
+                unsigned int after = PyUnicode_CHECK_INTERNED(str);
+                int preserved = data != NULL && before != SSTATE_NOT_INTERNED && after == before;
+                Py_DECREF(str);
+                return PyBool_FromLong(preserved);
+            }
+            ''',
+            tp_methods='''
+            {"check_intern_state_after_data", (PyCFunction)check_intern_state_after_data, METH_O, ""}
+            ''',
+        )
+        tester = TestLazyIntern()
+        string = b'lazy intern state'.decode('ascii')
+        assert tester.check_intern_state_after_data(string)
+
+
+    def test_unicode_data(self):
+        TestUnicodeData = CPyExtType(
+            "TestUnicodeData",
+            '''
+            #define NCHARS ((Py_ssize_t) 5)
+            static PyObject* create_compact(void) {
+                PyObject* obj = PyUnicode_New(NCHARS, (Py_UCS4) 128);
+                if (obj == NULL) {
+                    return NULL;
+                }
+                void* data = PyUnicode_DATA(obj);
+                memcpy(data, "hello", NCHARS);
+                PyUnicode_READY(obj);
+                return obj;
+            }
+
+            static PyObject* compact_unicode_as_bytes(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg)) {
+                // PyUnicode_New always returns a compact string (on GraalPy and CPython)
+                PyObject *compact = create_compact();
+                if (!PyUnicode_IS_COMPACT(compact)) {
+                    PyErr_SetString(PyExc_TypeError, "expected compact unicode object");
+                    return NULL;
+                }
+                void *data = PyUnicode_DATA(compact);
+                Py_ssize_t n = PyUnicode_GET_LENGTH(compact);
+                return PyBytes_FromStringAndSize(data, n);
+            }
+
+            static PyObject* noncompact_unicode_as_bytes(PyObject *Py_UNUSED(self), PyObject *arg) {
+                if (PyUnicode_IS_COMPACT(arg)) {
+                    PyErr_SetString(PyExc_TypeError, "expected non-compact unicode object");
+                    return NULL;
+                }
+                void *data = PyUnicode_DATA(arg);
+                Py_ssize_t n = PyUnicode_GET_LENGTH(arg);
+                return PyBytes_FromStringAndSize(data, n);
+            }
+            ''',
+            tp_methods='''
+            {"compact_unicode_as_bytes", (PyCFunction)compact_unicode_as_bytes, METH_NOARGS, ""},
+            {"noncompact_unicode_as_bytes", (PyCFunction)noncompact_unicode_as_bytes, METH_O, ""}
+            ''',
+        )
+        tester = TestUnicodeData()
+
+        # on CPython: unicode subclasses are always non-compact
+        # on GraalPy: managed unicode objects are always non-compact
+        noncompact_unicode = CustomString("hello")
+
+        assert tester.compact_unicode_as_bytes() == b'hello'
+        assert tester.noncompact_unicode_as_bytes(noncompact_unicode) == b'hello'
+
+
 class TestNativeUnicodeSubclass(unittest.TestCase):
     def test_builtins(self):
         s = UnicodeSubclass("asdf")