diff --git a/CHANGELOG b/CHANGELOG index fdcff46c99b..e17369f4e2b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,7 +6,7 @@ We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO. ==================== - Bug-fixes & improvements -- ES2 code cleanup +- ES2 code cleanup 26.14.0 (2026-07-06) ==================== @@ -20,7 +20,6 @@ We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO. - add migrate_osfmetrics_fix_6to8 command to correct monthly usage reports - temporarily re-add elasticsearch6 and some es6-based osf-metrics models - 26.12.0 (2026-06-23) ==================== @@ -32,7 +31,6 @@ We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO. - Stop using elasticsearch6 - Change all osfmetrics-consuming code to use es8 from data migration in 26.10.0 - 26.10.7 (2026-06-10) ==================== diff --git a/addons/osfstorage/tests/test_views.py b/addons/osfstorage/tests/test_views.py index 825b68b2671..9aa4b2bec58 100644 --- a/addons/osfstorage/tests/test_views.py +++ b/addons/osfstorage/tests/test_views.py @@ -67,16 +67,6 @@ def send_hook(self, view_name, view_kwargs, payload, target, method='get', **kwa @pytest.mark.django_db class TestGetMetadataHook(HookTestCase): - def test_empty(self): - res = self.send_hook( - 'osfstorage_get_children', - {'fid': self.node_settings.get_root()._id, 'user_id': self.user._id}, - {}, - self.node - ) - assert isinstance(res.json, list) - assert res.json == [] - def test_file_metadata(self): path = 'kind/of/magíc.mp3' record = recursively_create_file(self.node_settings, path) @@ -107,6 +97,95 @@ def test_preprint_primary_file_metadata(self): assert isinstance(res.json, dict) assert res.json == record.parent.serialize(True) + def test_osf_storage_root(self): + auth = Auth(self.project.creator) + result = osf_storage_root(self.node_settings.config, self.node_settings, auth) + node = self.project + expected = rubeus.build_addon_root( + node_settings=self.node_settings, + name='', + permissions=auth, + user=auth.user, + nodeUrl=node.url, + nodeApiUrl=node.api_url, + ) + root = result[0] + assert root == expected + + def test_root_default(self): + res = self.send_hook('osfstorage_get_metadata', {}, {}, self.node) + + assert res.json['fullPath'] == '/' + assert res.json['id'] == self.node_settings.get_root()._id + + def test_root_preprint_default(self): + preprint = PreprintFactory() + res = self.send_hook('osfstorage_get_metadata', {}, {}, preprint) + + assert res.json['fullPath'] == '/' + assert res.json['id'] == preprint.root_folder._id + + def test_metadata_not_found(self): + res = self.send_hook( + 'osfstorage_get_metadata', + {'fid': 'somebogusid'}, {}, + self.node, + ) + assert res.status_code == 404 + + def test_metadata_not_found_lots_of_slashes(self): + res = self.send_hook( + 'osfstorage_get_metadata', + {'fid': '/not/fo/u/nd/'}, {}, + self.node, + ) + assert res.status_code == 302 + assert '/login?service=' in res.location + + self.node.is_public = True + self.node.save() + res = self.send_hook( + 'osfstorage_get_metadata', + {'fid': '/not/fo/u/nd/'}, {}, + self.node, + ) + assert res.status_code == 404 + + +@pytest.mark.django_db +class TestGetChildrenHook(HookTestCase): + + def get_children(self, parent, target=None, **query_params): + params = { + 'fid': parent._id, + 'user_id': self.user._id, + 'minimal': 'true', + **query_params, + } + return self.send_hook( + 'osfstorage_get_children', + params, + {}, + target or self.node, + ) + + def _create_child_file(self, parent, name): + record = parent.append_file(name) + version = factories.FileVersionFactory() + record.add_version(version) + record.save() + return record + + def test_empty(self): + res = self.send_hook( + 'osfstorage_get_children', + {'fid': self.node_settings.get_root()._id, 'user_id': self.user._id}, + {}, + self.node + ) + assert isinstance(res.json, list) + assert res.json == [] + def test_children_metadata(self): path = 'kind/of/magíc.mp3' record = recursively_create_file(self.node_settings, path) @@ -176,59 +255,60 @@ def test_children_metadata_preprint(self): assert res_date_modified == expected_date_modified assert res_date_created == expected_date_created - def test_osf_storage_root(self): - auth = Auth(self.project.creator) - result = osf_storage_root(self.node_settings.config, self.node_settings, auth) - node = self.project - expected = rubeus.build_addon_root( - node_settings=self.node_settings, - name='', - permissions=auth, - user=auth.user, - nodeUrl=node.url, - nodeApiUrl=node.api_url, - ) - root = result[0] - assert root == expected - - def test_root_default(self): - res = self.send_hook('osfstorage_get_metadata', {}, {}, self.node) - - assert res.json['fullPath'] == '/' - assert res.json['id'] == self.node_settings.get_root()._id - - def test_root_preprint_default(self): - preprint = PreprintFactory() - res = self.send_hook('osfstorage_get_metadata', {}, {}, preprint) - - assert res.json['fullPath'] == '/' - assert res.json['id'] == preprint.root_folder._id - - def test_metadata_not_found(self): - res = self.send_hook( - 'osfstorage_get_metadata', - {'fid': 'somebogusid'}, {}, - self.node, - ) - assert res.status_code == 404 - - def test_metadata_not_found_lots_of_slashes(self): - res = self.send_hook( - 'osfstorage_get_metadata', - {'fid': '/not/fo/u/nd/'}, {}, - self.node, - ) - assert res.status_code == 302 - assert '/login?service=' in res.location - - self.node.is_public = True - self.node.save() - res = self.send_hook( - 'osfstorage_get_metadata', - {'fid': '/not/fo/u/nd/'}, {}, - self.node, + def test_minimal_child_fields(self): + parent = self.node_settings.get_root().append_folder('minimal-child-fields') + record = self._create_child_file(parent, 'magíc.mp3') + folder = parent.append_folder('nested') + res = self.get_children(parent) + assert res.status_code == 200 + assert len(res.json) == 2 + assert {item['name'] for item in res.json} == {record.name, folder.name} + for item in res.json: + assert set(item.keys()) == {'id', 'kind', 'name', 'path', 'storage'} + if item['kind'] == 'file': + assert item['storage'] == { + 'data': { + 'name': record.name, + 'path': record.versions.first().location_hash, + }, + 'settings': { + storage_settings.WATERBUTLER_RESOURCE: record.versions.first().location[ + storage_settings.WATERBUTLER_RESOURCE + ], + }, + } + assert item['path'] == f'/{record._id}' + assert item['id'] == record.id + else: + assert item['storage'] is None + assert item['path'] == f'/{folder._id}/' + assert item['id'] == folder.id + + def test_minimal_pagination(self): + parent = self.node_settings.get_root().append_folder('pagination') + children = [ + self._create_child_file(parent, name) + for name in ('a.mp3', 'b.mp3', 'c.mp3') + ] + children.sort(key=lambda c: c.id) + first_page = self.get_children(parent, limit=2) + assert first_page.status_code == 200 + assert [item['id'] for item in first_page.json] == [children[0].id, children[1].id] + second_page = self.get_children( + parent, + limit='2', + after=str(children[1].id), ) - assert res.status_code == 404 + assert second_page.status_code == 200 + assert len(second_page.json) == 1 + assert second_page.json[0]['id'] == children[2].id + + def test_minimal_pagination_after_beyond_last_child(self): + parent = self.node_settings.get_root() + record = self._create_child_file(parent, 'only.mp3') + res = self.get_children(parent, after=str(record.id)) + assert res.status_code == 200 + assert res.json == [] @pytest.mark.django_db diff --git a/addons/osfstorage/views.py b/addons/osfstorage/views.py index e387f34a768..c8494ae6a48 100644 --- a/addons/osfstorage/views.py +++ b/addons/osfstorage/views.py @@ -2,18 +2,22 @@ import logging from django.core.exceptions import ValidationError +from django.contrib.contenttypes.models import ContentType from django.db import IntegrityError from django.db import connection from django.db import transaction - +from django.db.models import Case, CharField, F, JSONField, OuterRef, Subquery, Value, When +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import Coalesce, Concat, JSONObject from flask import request from framework.auth import Auth from framework.exceptions import HTTPError from framework.auth.decorators import must_be_signed, must_be_logged_in +from api.base.utils import is_truthy from osf.exceptions import InvalidTagError, TagNotFoundError -from osf.models import FileVersion, Node, OSFUser +from osf.models import BaseFileVersionsThrough, FileVersion, Node, OSFUser from osf.utils.permissions import WRITE from osf.utils.requests import check_select_for_update from website.project.decorators import ( @@ -25,7 +29,7 @@ from website.settings import StorageLimits from addons.osfstorage import utils from addons.osfstorage import decorators -from addons.osfstorage.models import OsfStorageFolder +from addons.osfstorage.models import OsfStorageFileNode, OsfStorageFolder from addons.osfstorage import settings as osf_storage_settings @@ -183,12 +187,64 @@ def osfstorage_get_metadata(file_node, **kwargs): return file_node.serialize(version=version, include_full=True) -@must_be_signed -@decorators.autoload_filenode(must_be='folder') -def osfstorage_get_children(file_node, **kwargs): - from django.contrib.contenttypes.models import ContentType +def _osfstorage_minimal_metadata(file_node, *, limit=None, after=None): + waterbutler_resource = osf_storage_settings.WATERBUTLER_RESOURCE + latest_version = BaseFileVersionsThrough.objects.filter( + basefilenode_id=OuterRef('pk'), + ).order_by('-fileversion__created') + + file_storage = Subquery( + latest_version.annotate( + _storage=JSONObject( + data=JSONObject( + name=Coalesce(F('version_name'), OuterRef('name')), + path=KeyTextTransform('object', 'fileversion__location'), + ), + settings=JSONObject(**{ + waterbutler_resource: KeyTextTransform( + waterbutler_resource, 'fileversion__location', + ), + }), + ), + ).values('_storage')[:1], + ) + + qs = OsfStorageFileNode.objects.filter( + parent_id=file_node.id, + type__in=('osf.osfstoragefile', 'osf.osfstoragefolder'), + ).annotate( + kind=Case( + When(type='osf.osfstoragefile', then=Value('file')), + default=Value('folder'), + output_field=CharField(), + ), + path=Case( + When(type='osf.osfstoragefile', then=Concat(Value('/'), F('_id'))), + default=Concat(Value('/'), F('_id'), Value('/')), + output_field=CharField(), + ), + storage=Case( + When(type='osf.osfstoragefile', then=file_storage), + default=Value(None, output_field=JSONField()), + output_field=JSONField(), + ), + ).values('id', 'name', 'kind', 'path', 'storage') + + if after is not None: + qs = qs.filter(id__gt=after) + + # Always return files/folders in the same order + # It helps make waterbutler file streaming more stable + qs = qs.order_by('id') + + if limit is not None: + qs = qs[:limit] + + return list(qs) + + +def _osfstorage_full_metadata(file_node, user_id): from osf.models.preprint import Preprint - user_id = request.args.get('user_id') user_content_type_id = ContentType.objects.get_for_model(OSFUser).id user_pk = OSFUser.objects.filter(guids___id=user_id, guids___id__isnull=False).values_list('pk', flat=True).first() guid_id = file_node.target.get_guid().id if isinstance(file_node.target, Preprint) else file_node.target.guids.first().id, @@ -294,6 +350,18 @@ def osfstorage_get_children(file_node, **kwargs): return cursor.fetchone()[0] or [] +@must_be_signed +@decorators.autoload_filenode(must_be='folder') +def osfstorage_get_children(file_node, **kwargs): + if is_truthy(request.args.get('minimal')): + return _osfstorage_minimal_metadata( + file_node, + limit=request.args.get('limit', type=int, default=None), + after=request.args.get('after', type=int, default=None), + ) + return _osfstorage_full_metadata(file_node, request.args.get('user_id')) + + @must_be_signed @decorators.autoload_filenode(must_be='folder') def osfstorage_create_child(file_node, payload, **kwargs):