Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions backend/file_management/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,10 @@
"get": "list_ide",
}
)
file_delete = FileManagementViewSet.as_view(
{
"get": "delete",
}
)
urlpatterns = format_suffix_patterns(
[
path("file", file_list, name="file-list"),
path("file/download", file_downlaod, name="download"),
path("file/upload", file_upload, name="upload"),
path("file/delete", file_delete, name="delete"),
]
)
41 changes: 2 additions & 39 deletions backend/file_management/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,10 @@
from connector_v2.models import ConnectorInstance
from django.http import HttpRequest
from oauth2client.client import HttpAccessTokenRefreshError
from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManager
from rest_framework import serializers, status, viewsets
from rest_framework import serializers, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from utils.user_session import UserSessionUtils

from file_management.exceptions import (
ConnectorInstanceNotFound,
Expand All @@ -18,21 +16,19 @@
)
from file_management.file_management_helper import FileManagerHelper
from file_management.serializer import (
FileInfoIdeSerializer,
FileInfoSerializer,
FileListRequestSerializer,
FileUploadSerializer,
)
from unstract.connectors.exceptions import ConnectorError
from unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFS

logger = logging.getLogger(__name__)


class FileManagementViewSet(viewsets.ModelViewSet):
"""FileManagement view.

Handles GET,POST,PUT,PATCH and DELETE
Handles GET, POST, PUT and PATCH

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Low] [Lens 16] — prior finding NOT fixed: this still claims PUT and PATCH

The PR edited this exact line (Handles GET,POST,PUT,PATCH and DELETE -> Handles GET, POST, PUT and PATCH), removing only the DELETE half. urls.py:37-42 binds three paths, all get/post, and FileManagementViewSet defines only list, download (get) and upload (post) — there is no update/partial_update and no PUT/PATCH route anywhere in the repo.

Suggest Handles GET and POST., or dropping the line — the routes are two files away and it will rot again on the next route change.

"""

versioning_class = URLPathVersioning
Expand Down Expand Up @@ -99,36 +95,3 @@ def upload(self, request: HttpRequest) -> Response:
logger.info(f"Uploading file: {file_name}" if file_name else "Uploading file")
FileManagerHelper.upload_file(file_system, path, uploaded_file, file_name)
return Response({"message": "Files are uploaded successfully!"})

@action(detail=True, methods=["get"])
def delete(self, request: HttpRequest) -> Response:
serializer = FileInfoIdeSerializer(data=request.GET)
serializer.is_valid(raise_exception=True)
document_id: str = serializer.validated_data.get("document_id")
document: DocumentManager = DocumentManager.objects.get(pk=document_id)
file_name: str = document.document_name
tool_id: str = serializer.validated_data.get("tool_id")
file_path = FileManagerHelper.handle_sub_directory_for_tenants(
UserSessionUtils.get_organization_id(request),
is_create=False,
user_id=request.user.user_id,
tool_id=tool_id,
)
path = file_path
file_system = LocalStorageFS(settings={"path": path})
try:
# Delete the document record
document.delete()

# Delete the file
FileManagerHelper.delete_file(file_system, path, file_name)
return Response(
{"data": "File deleted succesfully."},
status=status.HTTP_200_OK,
)
except Exception as exc:
logger.error(f"Exception thrown from file deletion, error {exc}")
return Response(
{"data": "File deletion failed."},
status=status.HTTP_400_BAD_REQUEST,
)
4 changes: 3 additions & 1 deletion backend/notification_v2/internal_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ class WebhookInternalViewSet(viewsets.ReadOnlyModelViewSet):

serializer_class = NotificationSerializer
lookup_field = "id"
# Backward compat: remove once all workers pass X-Organization-ID.
# OrganizationFilterBackend is off here; get_queryset() scopes instead, via
# filter_queryset_by_organization. That helper fails closed, so a caller
# without X-Organization-ID gets zero rows.
skip_org_filter = True

def get_queryset(self):
Expand Down
4 changes: 3 additions & 1 deletion backend/pipeline_v2/internal_api_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@


class PipelineInternalViewSet(ViewSet):
# Backward compat: remove once all workers pass X-Organization-ID.
# OrganizationFilterBackend is off here; scoping runs through
# filter_queryset_by_organization, which fails closed, so a caller without
# X-Organization-ID gets zero rows.
skip_org_filter = True

def retrieve(self, request, pk=None):
Expand Down
5 changes: 3 additions & 2 deletions backend/prompt_studio/prompt_profile_manager_v2/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@
from django.db import models
from django.db.models import Q
from tenant_account_v2.organization_member_service import OrganizationMemberService
from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.base_model import BaseModel
from utils.models.org_aware_manager import OrgAwareManager
from utils.user_context import UserContext

from prompt_studio.prompt_studio_core_v2.exceptions import DefaultProfileError
from prompt_studio.prompt_studio_core_v2.models import CustomTool


class ProfileManagerModelManager(BaseModelManager):
class ProfileManagerModelManager(OrgAwareManager):
def for_user(self, user):
"""Read visibility: profile's own share fields OR parent CustomTool sharing.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Medium] [Lens 16, 3] — for_user scopes through the exact path the pin comment says was rejected

org_path_discovery.py:58-59 justifies pinning to vector_store__organization with: "Deliberately not prompt_studio_tool__organization: that FK is nullable, so it would drop tool-less profiles."

But for_user — the read path behind the profile list endpoint — builds Q(prompt_studio_tool__organization=UserContext.get_organization()) at :26 and ANDs it onto a queryset OrgAwareManager.get_queryset has already filtered by vector_store__organization. So on the primary read path tool-less profiles are dropped, the property the pin claims to preserve does not hold, and every query carries two redundant org joins. Someone reading the pin comment will conclude tool-less profiles are visible and debug the wrong layer.

There is also a null-policy split: when UserContext.get_organization() returns None, get_queryset applies no filter at all while :26 applies prompt_studio_tool__organization IS NULL — two opposite behaviours composed into one queryset from the same None.

Fix: drop org_scope from for_user entirely — get_queryset now supplies org scoping, which was the point of changing the base class on this line. for_user should express sharing only. If the tool-based scope must stay, correct the pin comment.

Expand Down
21 changes: 20 additions & 1 deletion backend/prompt_studio/prompt_studio_core_v2/internal_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,26 @@ def extraction_status(request):
extracted=extracted,
error_message=error_message,
)
return JsonResponse({"success": success})
if not success:
# A 200 here is indistinguishable from a write that landed: the
# worker only wraps this call in try/except and never reads the
# body, so the status would be silently dropped and every later
# Answer Prompt would re-run the full extraction. Non-2xx makes
# the worker's existing handler log it.
logger.error(
"extraction_status not recorded for document %s profile %s",
document_id,
profile_manager_id,
)
return JsonResponse(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of returning a JsonResponse, can you raise it as an error to keep it consistent with the rest of the codebase and let DRF validation handle it?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NOT RESOLVED — no push since this was raised, so just re-flagging it rather than treating it as ignored. Still worth doing alongside the other changes on this handler; note the retryable-500 point at internal_views.py:232 interacts with it, so the two are best decided together.

{
"success": False,
"error": "Extraction status could not be recorded",
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Medium] [Lens 8] — surfacing the failure is right; mapping a permanent condition onto a retryable status is not

mark_extraction_status returns False for two very different reasons: a genuine write failure, and DocumentManager.DoesNotExist (prompt_studio_index_helper.py:161-170) — now reachable as "the org filter hid it", or simply because the user deleted the document mid-indexing. Both land on 500.

base_client.py:303 sets retry_statuses = {500, 502, 503, 504} with 3 attempts and backoff_factor=1.0, so a deterministic "document is gone" burns 4 round trips and ~7s of in-worker sleep before ide_callback/tasks.py's handler finally logs it. A normal user action becomes a retry storm plus an ERROR-level alert.

Fix: have mark_extraction_status distinguish its two failure reasons (an enum, or let DoesNotExist propagate) and map "target missing" to 404 — outside retry_statuses — keeping 500 for the genuine write failure.

This is a new consequence of the correct fix to prior finding #5, not a regression on it.

return JsonResponse({"success": True})

except Exception as e:
logger.exception("extraction_status internal API failed")
Expand Down
34 changes: 28 additions & 6 deletions backend/prompt_studio/prompt_studio_core_v2/migration_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,35 @@ def migrate_tool_to_adapter_based(

# Re-fetch the summarize profile with lock within transaction
try:
summarize_profile = ProfileManager.objects.select_for_update().get(
prompt_studio_tool=tool_instance, is_summarize_llm=True
)
# of=("self",): the org-scoped manager joins through
# AdapterInstance, which would otherwise be locked too.
summarize_profile = ProfileManager.objects.select_for_update(
of=("self",)
).get(prompt_studio_tool=tool_instance, is_summarize_llm=True)
Comment thread
athul-rs marked this conversation as resolved.
except ObjectDoesNotExist:
logger.info(
f"No summarize profile found for tool {tool_instance.tool_id}, skipping migration"
)
# Two different situations reach here now that
# ProfileManager.objects is scoped through
# vector_store__organization: the profile genuinely does
# not exist, or it exists and the org filter hid it. The
# second is a misconfiguration that never self-heals — this
# lazy migration re-runs and re-skips on every invocation —
# so it must not share an INFO line with the first.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Medium] [Lens 3, 10] — this diagnostic is dead code: the unfixed lookup above short-circuits first (prior finding #13, NOT RESOLVED)

An identical, unguarded ProfileManager.objects.get(...) runs at :42-50, outside the transaction. When the org filter hides the profile, that except ObjectDoesNotExist fires and returns False with the old ambiguous INFO line at :47-49. Execution never reaches here.

So the _base_manager probe and the ERROR branch are reachable only via a TOCTOU race in which the profile vanishes between the two get() calls — never for the misconfiguration they were written for. An on-call reading "no ERROR in the logs" concludes the profile genuinely does not exist, which is precisely the inference this comment promises is now safe.

Fix: hoist the exists_unscoped probe into the first except, or delete the redundant pre-transaction lookup entirely — the in-transaction fetch already repeats it.

exists_unscoped = ProfileManager._base_manager.filter(
prompt_studio_tool=tool_instance, is_summarize_llm=True
).exists()
if exists_unscoped:
logger.error(
"Summarize profile for tool %s exists but is not "
"visible in the current organization context; "
"migration skipped and will keep being skipped.",
tool_instance.tool_id,
)
else:
logger.info(
"No summarize profile found for tool %s, skipping "
"migration",
tool_instance.tool_id,
)
return False

# Check if profile has an LLM adapter
Expand Down
76 changes: 68 additions & 8 deletions backend/prompt_studio/prompt_studio_core_v2/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
from account_v2.custom_exceptions import DuplicateData
from celery import signature
from celery.result import AsyncResult
from django.db import IntegrityError
from django.db import IntegrityError, transaction
from django.db.models import Count, OuterRef, QuerySet, Subquery
from django.http import HttpRequest, HttpResponse
from django.shortcuts import get_object_or_404
from django.utils import timezone
from file_management.constants import FileInformationKey as FileKey
from file_management.exceptions import FileNotFound
Expand All @@ -24,6 +25,7 @@
from plugins import get_plugin
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import ValidationError
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
Expand Down Expand Up @@ -389,13 +391,39 @@
self.get_object()
) # Assuming you have a get_object method in your viewset

ProfileManager.objects.filter(prompt_studio_tool=prompt_tool).update(
is_default=False
# Validate before looking anything up. A missing key raised KeyError
# and a non-UUID value raised Django's ValidationError; neither is
# mapped by drf_standardized_errors, so both surfaced as 500s next to
# the 404 this action already returns for a valid-but-unmatched id.
default_profile = request.data.get("default_profile")
if not default_profile:
raise ValidationError(detail="'default_profile' is required.")
try:
default_profile = uuid.UUID(str(default_profile))
except (ValueError, AttributeError, TypeError):
raise ValidationError(detail="'default_profile' must be a valid UUID.")

# Resolve the target before clearing anything: the id comes straight
# from the request body, and clearing first would leave the tool with no
# default at all when it does not match. Scoped to the same tool the
# caller already passed authz on, so another tool's id is a 404.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Low] [Lens 16] — this comment and its own test's docstring disagree on whether resolve-before-clear is load-bearing

Here: "clearing first would leave the tool with no default at all when it does not match".

tests/test_cross_org_isolation.py:255-262, for the test that pins this: "Reverting just the ordering, with transaction.atomic() still in place, is genuinely safe and does not fail here."

Your correction in the previous thread was right, and I re-confirmed it by mutation — ordering-only revert with the transaction retained -> 13 passed; full revert to main's clear-first + .objects.get + no atomic -> 1 failed, exactly this test. So the docstring is accurate and the code comment overstates.

Suggest qualifying the comment — the ordering is belt-and-braces given the surrounding transaction — or dropping the causal claim and keeping "resolve before clearing".

profile_manager = get_object_or_404(
ProfileManager,
pk=default_profile,
prompt_studio_tool=prompt_tool,
Comment thread
athul-rs marked this conversation as resolved.
)

profile_manager = ProfileManager.objects.get(pk=request.data["default_profile"])
profile_manager.is_default = True
profile_manager.save()
# Both writes in one transaction so a failure between them cannot leave
# the tool with zero defaults or two. update_fields so the second write
# touches one column: profile_manager was read before the transaction
# opened, and a bare save() would write every column from that snapshot
# back over any concurrent edit.
with transaction.atomic():
ProfileManager.objects.filter(prompt_studio_tool=prompt_tool).update(
is_default=False
)
profile_manager.is_default = True
profile_manager.save(update_fields=["is_default"])

return Response(
status=status.HTTP_200_OK,
Expand Down Expand Up @@ -1130,11 +1158,30 @@
document_id: str = serializer.validated_data.get(ToolStudioPromptKeys.DOCUMENT_ID)
org_id = UserSessionUtils.get_organization_id(request)
user_id = custom_tool.created_by.user_id
document: DocumentManager = DocumentManager.objects.get(pk=document_id)
# Scope to the tool the caller already passed authz on — tighter than
# org scope. self.get_object() above is filtered by the backend, but
# this lookup is a raw .objects query and would not be.
# get_object_or_404 keeps a non-matching id a 404 rather than an
# unhandled DoesNotExist, which the DRF handler turns into a 500.
document: DocumentManager = get_object_or_404(
DocumentManager, pk=document_id, tool=custom_tool
)
Comment thread
athul-rs marked this conversation as resolved.

try:
# Delete indexed flags in redis
index_managers = IndexManager.objects.filter(document_manager=document_id)
if not index_managers.exists():
# Empty means either "never indexed" or "the org filter hid the
# rows". In the second case the Redis indexing flags outlive the
# document, and a re-upload of the same file is treated as
# already indexed — with a 200 telling the user it all worked.
logger.warning(
"No index managers visible for document %s (tool %s, org %s); "
"deleting without clearing Redis indexing flags.",
document_id,
custom_tool.tool_id,
org_id,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Medium] [Lens 10] — this warning fires on the ordinary never-indexed path, and the case it warns about still returns 200 (prior finding #12, PARTIALLY RESOLVED)

if not index_managers.exists() cannot separate "never indexed" from "the org filter hid the rows" — and in practice it is always the former, because IndexManager's pin (document_manager__tool__organization) traverses the same CustomTool.organization as the DocumentManager lookup at :1161-1163 that already succeeded. So every delete of an uploaded-but-not-yet-indexed document emits a WARNING naming an org id on a completely normal path, drowning the signal you added it for.

And in the case it does warn about, the handler still returns {"data": "File deleted succesfully."} with 200 — the Redis flags outlive the document and the user is told everything worked.

Fix: probe IndexManager._base_manager.filter(document_manager=document_id).exists() — the pattern migration_utils.py:60-66 already uses — and warn only when the unscoped probe finds rows the scoped query did not. That also drops the extra round trip on the common path.

)
for index_manager in index_managers:
raw_index_id = index_manager.raw_index_id
DocumentIndexingService.remove_document_indexing(
Expand All @@ -1155,7 +1202,20 @@
status=status.HTTP_200_OK,
)
except Exception as exc:
logger.error("Exception thrown from file deletion, error: %s", exc)
# Deliberately broad. Three subsystems are in play — Redis via
# DocumentIndexingService, the object store via
# PromptStudioFileHelper, and the database — and their failures do
# not share a base class, so narrowing to any list turns a
# reachable outage in whichever one was missed into a 500. The
# diagnosability problem was the log line, not the catch: it now
# carries the exception type, the document and a stack.
logger.error(

Check failure on line 1212 in backend/prompt_studio/prompt_studio_core_v2/views.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use "logging.exception()" instead.

See more on https://sonarcloud.io/project/issues?id=Zipstack_unstract&issues=AZ_1zpG5RR8WeJbnwUag&open=AZ_1zpG5RR8WeJbnwUag&pullRequest=2213
"File deletion failed for document %s (tool %s): %s",
document_id,
custom_tool.tool_id,
exc,
exc_info=True,
)
return Response(
{"data": "File deletion failed."},
status=status.HTTP_400_BAD_REQUEST,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,21 @@
from account_v2.models import User
from django.db import models
from utils.models.base_model import BaseModel
from utils.models.org_aware_manager import OrgAwareManager

from prompt_studio.prompt_studio_core_v2.models import CustomTool


class DocumentManager(BaseModel):
"""Model to store the document details."""

# Org scoping lives at the manager because OrganizationFilterBackend only
# scopes querysets routed through filter_queryset(). A raw Model.objects
# lookup inside a view bypasses it — including inside a custom @action,
# whose own self.get_object() *is* filtered but whose hand-written queries
# are not.
objects = OrgAwareManager()

document_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)

document_name = models.CharField(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from django.db.models.signals import pre_delete
from django.dispatch import receiver
from utils.models.base_model import BaseModel
from utils.models.org_aware_manager import OrgAwareManager
from utils.user_context import UserContext

from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
Expand All @@ -21,6 +22,9 @@
class IndexManager(BaseModel):
"""Model to store the index details."""

# See DocumentManager.objects for why scoping lives at the manager.
objects = OrgAwareManager()

index_manager_id = models.UUIDField(
primary_key=True, default=uuid.uuid4, editable=False
)
Expand Down
Loading
Loading