[ENG-11015] [ENG-11692] IDS Phase 1 & 2 Project PR - BE Part

CHANGELOG

@@ -6,7 +6,7 @@ We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 ====================
 
 - Bug-fixes & improvements
-- ES2 code cleanup 
+- ES2 code cleanup
 
 26.14.0 (2026-07-06)
 ====================
@@ -20,7 +20,6 @@ We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 - add migrate_osfmetrics_fix_6to8 command to correct monthly usage reports
 - temporarily re-add elasticsearch6 and some es6-based osf-metrics models
 
-
 26.12.0 (2026-06-23)
 ====================
 
@@ -32,7 +31,6 @@ We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 - Stop using elasticsearch6
 - Change all osfmetrics-consuming code to use es8 from data migration in 26.10.0
 
-
 26.10.7 (2026-06-10)
 ====================
 

addons/osfstorage/tests/test_views.py

@@ -67,16 +67,6 @@ def send_hook(self, view_name, view_kwargs, payload, target, method='get', **kwa
 @pytest.mark.django_db
 class TestGetMetadataHook(HookTestCase):
 
-    def test_empty(self):
-        res = self.send_hook(
-            'osfstorage_get_children',
-            {'fid': self.node_settings.get_root()._id, 'user_id': self.user._id},
-            {},
-            self.node
-        )
-        assert isinstance(res.json, list)
-        assert res.json == []
-
     def test_file_metadata(self):
         path = 'kind/of/magíc.mp3'
         record = recursively_create_file(self.node_settings, path)
@@ -107,6 +97,95 @@ def test_preprint_primary_file_metadata(self):
         assert isinstance(res.json, dict)
         assert res.json == record.parent.serialize(True)
 
+    def test_osf_storage_root(self):
+        auth = Auth(self.project.creator)
+        result = osf_storage_root(self.node_settings.config, self.node_settings, auth)
+        node = self.project
+        expected = rubeus.build_addon_root(
+            node_settings=self.node_settings,
+            name='',
+            permissions=auth,
+            user=auth.user,
+            nodeUrl=node.url,
+            nodeApiUrl=node.api_url,
+        )
+        root = result[0]
+        assert root == expected
+
+    def test_root_default(self):
+        res = self.send_hook('osfstorage_get_metadata', {}, {}, self.node)
+
+        assert res.json['fullPath'] == '/'
+        assert res.json['id'] == self.node_settings.get_root()._id
+
+    def test_root_preprint_default(self):
+        preprint = PreprintFactory()
+        res = self.send_hook('osfstorage_get_metadata', {}, {}, preprint)
+
+        assert res.json['fullPath'] == '/'
+        assert res.json['id'] == preprint.root_folder._id
+
+    def test_metadata_not_found(self):
+        res = self.send_hook(
+            'osfstorage_get_metadata',
+            {'fid': 'somebogusid'}, {},
+            self.node,
+        )
+        assert res.status_code == 404
+
+    def test_metadata_not_found_lots_of_slashes(self):
+        res = self.send_hook(
+            'osfstorage_get_metadata',
+            {'fid': '/not/fo/u/nd/'}, {},
+            self.node,
+        )
+        assert res.status_code == 302
+        assert '/login?service=' in res.location
+
+        self.node.is_public = True
+        self.node.save()
+        res = self.send_hook(
+            'osfstorage_get_metadata',
+            {'fid': '/not/fo/u/nd/'}, {},
+            self.node,
+        )
+        assert res.status_code == 404
+
+
+@pytest.mark.django_db
+class TestGetChildrenHook(HookTestCase):
+
+    def get_children(self, parent, target=None, **query_params):
+        params = {
+            'fid': parent._id,
+            'user_id': self.user._id,
+            'minimal': 'true',
+            **query_params,
+        }
+        return self.send_hook(
+            'osfstorage_get_children',
+            params,
+            {},
+            target or self.node,
+        )
+
+    def _create_child_file(self, parent, name):
+        record = parent.append_file(name)
+        version = factories.FileVersionFactory()
+        record.add_version(version)
+        record.save()
+        return record
+
+    def test_empty(self):
+        res = self.send_hook(
+            'osfstorage_get_children',
+            {'fid': self.node_settings.get_root()._id, 'user_id': self.user._id},
+            {},
+            self.node
+        )
+        assert isinstance(res.json, list)
+        assert res.json == []
+
     def test_children_metadata(self):
         path = 'kind/of/magíc.mp3'
         record = recursively_create_file(self.node_settings, path)
@@ -176,59 +255,60 @@ def test_children_metadata_preprint(self):
         assert res_date_modified == expected_date_modified
         assert res_date_created == expected_date_created
 
-    def test_osf_storage_root(self):
-        auth = Auth(self.project.creator)
-        result = osf_storage_root(self.node_settings.config, self.node_settings, auth)
-        node = self.project
-        expected = rubeus.build_addon_root(
-            node_settings=self.node_settings,
-            name='',
-            permissions=auth,
-            user=auth.user,
-            nodeUrl=node.url,
-            nodeApiUrl=node.api_url,
-        )
-        root = result[0]
-        assert root == expected
-
-    def test_root_default(self):
-        res = self.send_hook('osfstorage_get_metadata', {}, {}, self.node)
-
-        assert res.json['fullPath'] == '/'
-        assert res.json['id'] == self.node_settings.get_root()._id
-
-    def test_root_preprint_default(self):
-        preprint = PreprintFactory()
-        res = self.send_hook('osfstorage_get_metadata', {}, {}, preprint)
-
-        assert res.json['fullPath'] == '/'
-        assert res.json['id'] == preprint.root_folder._id
-
-    def test_metadata_not_found(self):
-        res = self.send_hook(
-            'osfstorage_get_metadata',
-            {'fid': 'somebogusid'}, {},
-            self.node,
-        )
-        assert res.status_code == 404
-
-    def test_metadata_not_found_lots_of_slashes(self):
-        res = self.send_hook(
-            'osfstorage_get_metadata',
-            {'fid': '/not/fo/u/nd/'}, {},
-            self.node,
-        )
-        assert res.status_code == 302
-        assert '/login?service=' in res.location
-
-        self.node.is_public = True
-        self.node.save()
-        res = self.send_hook(
-            'osfstorage_get_metadata',
-            {'fid': '/not/fo/u/nd/'}, {},
-            self.node,
+    def test_minimal_child_fields(self):
+        parent = self.node_settings.get_root().append_folder('minimal-child-fields')
+        record = self._create_child_file(parent, 'magíc.mp3')
+        folder = parent.append_folder('nested')
+        res = self.get_children(parent)
+        assert res.status_code == 200
+        assert len(res.json) == 2
+        assert {item['name'] for item in res.json} == {record.name, folder.name}
+        for item in res.json:
+            assert set(item.keys()) == {'id', 'kind', 'name', 'path', 'storage'}
+            if item['kind'] == 'file':
+                assert item['storage'] == {
+                    'data': {
+                        'name': record.name,
+                        'path': record.versions.first().location_hash,
+                    },
+                    'settings': {
+                        storage_settings.WATERBUTLER_RESOURCE: record.versions.first().location[
+                            storage_settings.WATERBUTLER_RESOURCE
+                        ],
+                    },
+                }
+                assert item['path'] == f'/{record._id}'
+                assert item['id'] == record.id
+            else:
+                assert item['storage'] is None
+                assert item['path'] == f'/{folder._id}/'
+                assert item['id'] == folder.id
+
+    def test_minimal_pagination(self):
+        parent = self.node_settings.get_root().append_folder('pagination')
+        children = [
+            self._create_child_file(parent, name)
+            for name in ('a.mp3', 'b.mp3', 'c.mp3')
+        ]
+        children.sort(key=lambda c: c.id)
+        first_page = self.get_children(parent, limit=2)
+        assert first_page.status_code == 200
+        assert [item['id'] for item in first_page.json] == [children[0].id, children[1].id]
+        second_page = self.get_children(
+            parent,
+            limit='2',
+            after=str(children[1].id),
         )
-        assert res.status_code == 404
+        assert second_page.status_code == 200
+        assert len(second_page.json) == 1
+        assert second_page.json[0]['id'] == children[2].id
+
+    def test_minimal_pagination_after_beyond_last_child(self):
+        parent = self.node_settings.get_root()
+        record = self._create_child_file(parent, 'only.mp3')
+        res = self.get_children(parent, after=str(record.id))
+        assert res.status_code == 200
+        assert res.json == []
 
 
 @pytest.mark.django_db

addons/osfstorage/views.py

@@ -2,18 +2,22 @@
 import logging
 
 from django.core.exceptions import ValidationError
+from django.contrib.contenttypes.models import ContentType
 from django.db import IntegrityError
 from django.db import connection
 from django.db import transaction
-
+from django.db.models import Case, CharField, F, JSONField, OuterRef, Subquery, Value, When
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Coalesce, Concat, JSONObject
 from flask import request
 
 from framework.auth import Auth
 from framework.exceptions import HTTPError
 from framework.auth.decorators import must_be_signed, must_be_logged_in
 
+from api.base.utils import is_truthy
 from osf.exceptions import InvalidTagError, TagNotFoundError
-from osf.models import FileVersion, Node, OSFUser
+from osf.models import BaseFileVersionsThrough, FileVersion, Node, OSFUser
 from osf.utils.permissions import WRITE
 from osf.utils.requests import check_select_for_update
 from website.project.decorators import (
@@ -25,7 +29,7 @@
 from website.settings import StorageLimits
 from addons.osfstorage import utils
 from addons.osfstorage import decorators
-from addons.osfstorage.models import OsfStorageFolder
+from addons.osfstorage.models import OsfStorageFileNode, OsfStorageFolder
 from addons.osfstorage import settings as osf_storage_settings
 
 
@@ -183,12 +187,64 @@ def osfstorage_get_metadata(file_node, **kwargs):
     return file_node.serialize(version=version, include_full=True)
 
 
-@must_be_signed
-@decorators.autoload_filenode(must_be='folder')
-def osfstorage_get_children(file_node, **kwargs):
-    from django.contrib.contenttypes.models import ContentType
+def _osfstorage_minimal_metadata(file_node, *, limit=None, after=None):
+    waterbutler_resource = osf_storage_settings.WATERBUTLER_RESOURCE
+    latest_version = BaseFileVersionsThrough.objects.filter(
+        basefilenode_id=OuterRef('pk'),
+    ).order_by('-fileversion__created')
+
+    file_storage = Subquery(
+        latest_version.annotate(
+            _storage=JSONObject(
+                data=JSONObject(
+                    name=Coalesce(F('version_name'), OuterRef('name')),
+                    path=KeyTextTransform('object', 'fileversion__location'),
+                ),
+                settings=JSONObject(**{
+                    waterbutler_resource: KeyTextTransform(
+                        waterbutler_resource, 'fileversion__location',
+                    ),
+                }),
+            ),
+        ).values('_storage')[:1],
+    )
+
+    qs = OsfStorageFileNode.objects.filter(
+        parent_id=file_node.id,
+        type__in=('osf.osfstoragefile', 'osf.osfstoragefolder'),
+    ).annotate(
+        kind=Case(
+            When(type='osf.osfstoragefile', then=Value('file')),
+            default=Value('folder'),
+            output_field=CharField(),
+        ),
+        path=Case(
+            When(type='osf.osfstoragefile', then=Concat(Value('/'), F('_id'))),
+            default=Concat(Value('/'), F('_id'), Value('/')),
+            output_field=CharField(),
+        ),
+        storage=Case(
+            When(type='osf.osfstoragefile', then=file_storage),
+            default=Value(None, output_field=JSONField()),
+            output_field=JSONField(),
+        ),
+    ).values('id', 'name', 'kind', 'path', 'storage')
+
+    if after is not None:
+        qs = qs.filter(id__gt=after)
+
+    # Always return files/folders in the same order
+    # It helps make waterbutler file streaming more stable
+    qs = qs.order_by('id')
+
+    if limit is not None:
+        qs = qs[:limit]
+
+    return list(qs)
+
+
+def _osfstorage_full_metadata(file_node, user_id):
     from osf.models.preprint import Preprint
-    user_id = request.args.get('user_id')
     user_content_type_id = ContentType.objects.get_for_model(OSFUser).id
     user_pk = OSFUser.objects.filter(guids___id=user_id, guids___id__isnull=False).values_list('pk', flat=True).first()
     guid_id = file_node.target.get_guid().id if isinstance(file_node.target, Preprint) else file_node.target.guids.first().id,
@@ -294,6 +350,18 @@ def osfstorage_get_children(file_node, **kwargs):
         return cursor.fetchone()[0] or []
 
 
+@must_be_signed
+@decorators.autoload_filenode(must_be='folder')
+def osfstorage_get_children(file_node, **kwargs):
+    if is_truthy(request.args.get('minimal')):
+        return _osfstorage_minimal_metadata(
+            file_node,
+            limit=request.args.get('limit', type=int, default=None),
+            after=request.args.get('after', type=int, default=None),
+        )
+    return _osfstorage_full_metadata(file_node, request.args.get('user_id'))
+
+
 @must_be_signed
 @decorators.autoload_filenode(must_be='folder')
 def osfstorage_create_child(file_node, payload, **kwargs):