Feat/s3 large comments

comments/admin.py

@@ -1,8 +1,12 @@
 from admin_auto_filters.filters import AutocompleteFilterFactory
+from django.conf import settings
 from django.contrib import admin
 from django.contrib.postgres.search import SearchQuery
+from django.utils.html import format_html
 
-from utils.models import CustomTranslationAdmin
+from comments.services.text_archive import get_full_text
+from utils.models import CustomTranslationAdmin, uniques_ordered_list
+from utils.translation import build_supported_localized_fieldname
 from .models import Comment, KeyFactor, KeyFactorDriver
 
 
@@ -30,20 +34,22 @@ class CommentAdmin(CustomTranslationAdmin):
         "created_at",
         "is_soft_deleted",
         "is_private",
+        "is_text_archived",
     ]
     list_filter = [
         AutocompleteFilterFactory("Author", "author"),
         AutocompleteFilterFactory("Post", "on_post"),
         "is_soft_deleted",
         "is_private",
+        "is_text_archived",
         AutocompleteFilterFactory("Project", "on_project"),
     ]
     autocomplete_fields = [
         "author",
         "on_post",
         "on_project",
     ]
-    readonly_fields = ["included_forecast"]
+    readonly_fields = ["included_forecast", "is_text_archived"]
     fields = [
         "author",
         "text",
@@ -52,6 +58,7 @@ class CommentAdmin(CustomTranslationAdmin):
         "is_soft_deleted",
         "included_forecast",
         "is_private",
+        "is_text_archived",
     ]
     # `search_fields` must be non-empty for Django admin to render the search box
     # and dispatch to `get_search_results`, but its contents are unused because we
@@ -62,6 +69,55 @@ class CommentAdmin(CustomTranslationAdmin):
     def should_update_translations(self, obj):
         return not obj.on_post.is_private()
 
+    @admin.display(description="Archived text (read-only, fetched from S3)")
+    def archived_text(self, obj):
+        """
+        The full text of an archived comment, read back from the archive.
+
+        The admin is where staff investigate a comment, and the row itself
+        now holds nothing but a 200-character stub. Reading this costs an S3
+        round trip per change-page load, which is why `get_fields` only adds
+        it for rows that are actually archived.
+        """
+
+        text = get_full_text(obj)
+
+        if text is None:
+            return format_html(
+                "<em>{}</em>",
+                "The archived text could not be retrieved from S3. "
+                "Only the stub above remains in the database.",
+            )
+
+        return format_html(
+            '<pre style="white-space: pre-wrap; max-width: 60em; '
+            'max-height: 30em; overflow: auto;">{}</pre>',
+            text,
+        )
+
+    def get_fields(self, request, obj=None):
+        fields = list(super().get_fields(request, obj))
+
+        if obj and obj.is_text_archived:
+            fields.append("archived_text")
+
+        return uniques_ordered_list(fields)
+
+    def get_readonly_fields(self, request, obj=None):
+        readonly_fields = list(super().get_readonly_fields(request, obj))
+
+        if obj and obj.is_text_archived:
+            # Only a stub of the text is left in the db, so editing it here
+            # would bypass the `update_comment` guard and leave the row out of
+            # sync with the archived original. `archived_text` is not a model
+            # field at all, so it has to be declared read-only to appear.
+            readonly_fields += ["text", "archived_text"] + [
+                build_supported_localized_fieldname("text", lang)
+                for lang, _label in settings.LANGUAGES
+            ]
+
+        return uniques_ordered_list(readonly_fields)
+
     def get_search_results(self, request, queryset, search_term):
         search_term = search_term.strip()
         if not search_term:

comments/management/commands/archive_bot_comment_texts.py

@@ -0,0 +1,180 @@
+import time
+from collections.abc import Callable
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from comments.services.text_archive import (
+    ARCHIVE_AGE_DAYS,
+    ARCHIVE_MIN_TEXT_LENGTH,
+    ARCHIVE_STUB_LENGTH,
+    DEFAULT_BATCH_SIZE,
+    DEFAULT_CONCURRENCY,
+    S3_KEY_PREFIX,
+    ArchiveStats,
+    archive_bot_comment_texts,
+    check_is_enabled,
+)
+
+
+def format_duration(seconds: float) -> str:
+    seconds = int(seconds)
+    hours, remainder = divmod(seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+
+    if hours:
+        return f"{hours}h{minutes:02d}m"
+    if minutes:
+        return f"{minutes}m{seconds:02d}s"
+
+    return f"{seconds}s"
+
+
+class ProgressWriter:
+    """
+    Prints a running one-line summary with a rate and an ETA.
+
+    The daily run has only a day of new comments to clear, but the first
+    backfill works through hundreds of thousands of rows over hours, so the
+    point is to make a long run observable rather than to look pretty. Output
+    is one line per batch, not a redrawn line, so it survives being piped to a
+    log file.
+    """
+
+    def __init__(self, stdout, total: int = 0):
+        self.stdout = stdout
+        self.total = total
+        self.started = time.monotonic()
+
+    @property
+    def elapsed(self) -> float:
+        return time.monotonic() - self.started
+
+    def write(self, line: str) -> None:
+        self.stdout.write(line)
+        self.stdout.flush()
+
+    def update(self, done: int, summary: str, detail: str = "") -> None:
+        elapsed = self.elapsed
+        rate = done / elapsed if elapsed else 0
+        percent = (done / self.total * 100) if self.total else 0
+        remaining = max(self.total - done, 0)
+        eta = format_duration(remaining / rate) if rate else "?"
+
+        line = (
+            f"  {done:,}/{self.total:,} ({percent:.1f}%)  {summary}  "
+            f"{rate:.1f}/s  elapsed {format_duration(elapsed)}  eta {eta}"
+        )
+
+        if detail:
+            line += f"  [{detail}]"
+
+        self.write(line)
+
+
+class Command(BaseCommand):
+    help = (
+        "Moves the full text of private bot comments older than "
+        f"{ARCHIVE_AGE_DAYS} days and longer than {ARCHIVE_MIN_TEXT_LENGTH} "
+        f"characters to S3, leaving a {ARCHIVE_STUB_LENGTH}-character stub in "
+        "the database. Runs daily as a cron job; the full text stays "
+        "readable through the comment-detail endpoint."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Report what would be archived without writing to S3 or the database",
+        )
+        parser.add_argument(
+            "--limit",
+            type=int,
+            default=None,
+            help="Maximum number of comments to archive (useful for the first backfill)",
+        )
+        parser.add_argument(
+            "--batch-size",
+            type=int,
+            default=DEFAULT_BATCH_SIZE,
+            help=f"Comments per database update (default: {DEFAULT_BATCH_SIZE})",
+        )
+        parser.add_argument(
+            "--concurrency",
+            type=int,
+            default=DEFAULT_CONCURRENCY,
+            help=(
+                "Uploads to keep in flight at once. S3 has no multi-object PUT, "
+                "so this is what makes a large backfill finish in minutes "
+                f"rather than hours (default: {DEFAULT_CONCURRENCY})"
+            ),
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options["dry_run"]
+
+        if not check_is_enabled():
+            raise CommandError(
+                "AWS_STORAGE_BUCKET_COMMENTS_TEXT is not configured, "
+                "comment text archiving is disabled."
+            )
+
+        progress = ProgressWriter(self.stdout)
+        on_progress: Callable[[ArchiveStats], None] | None = None
+
+        if not dry_run:
+            progress.write(
+                f"Archiving to {settings.AWS_STORAGE_BUCKET_COMMENTS_TEXT}/"
+                f"{S3_KEY_PREFIX}/ with concurrency {options['concurrency']}, "
+                f"batches of {options['batch_size']}"
+            )
+            progress.write("Finding eligible comments...")
+
+            def write_progress(stats: ArchiveStats) -> None:
+                progress.total = stats.total
+                detail = ", ".join(
+                    f"{count} {label}"
+                    for label, count in (
+                        ("failed", stats.failed),
+                        ("skipped", stats.skipped),
+                    )
+                    if count
+                )
+                progress.update(
+                    stats.archived + stats.failed + stats.skipped,
+                    f"{stats.chars_reclaimed:,} chars reclaimed",
+                    detail,
+                )
+
+            on_progress = write_progress
+
+        stats = archive_bot_comment_texts(
+            dry_run=dry_run,
+            limit=options["limit"],
+            batch_size=options["batch_size"],
+            concurrency=options["concurrency"],
+            on_progress=on_progress,
+        )
+
+        verb = "Would archive" if dry_run else "Archived"
+        elapsed = "" if dry_run else f" in {format_duration(progress.elapsed)}"
+        progress.write(
+            f"{verb} {stats.archived:,} comment(s), "
+            f"reclaiming {stats.chars_reclaimed:,} characters{elapsed}"
+        )
+
+        if stats.sample_ids:
+            sample = ", ".join(str(pk) for pk in stats.sample_ids)
+            progress.write(f"Sample comment ids: {sample}")
+
+        if stats.skipped:
+            self.stdout.write(
+                self.style.WARNING(
+                    f"Skipped {stats.skipped} comment(s) edited during the run"
+                )
+            )
+
+        if stats.failed:
+            self.stdout.write(
+                self.style.ERROR(f"Failed to upload {stats.failed} comment(s)")
+            )

comments/migrations/0027_comment_is_text_archived.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.15 on 2026-08-19 17:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('comments', '0026_comment_key_factor_votes_score'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='comment',
+            name='is_text_archived',
+            field=models.BooleanField(db_index=True, default=False, editable=False, help_text='True if the full text has been moved to S3 and only a truncated stub remains in the text columns. Archived comments cannot be edited; use the comment-detail endpoint to read them.'),
+        ),
+    ]

comments/models.py

@@ -99,6 +99,18 @@ class Comment(TimeStampedModel, TranslatedModel):
     is_soft_deleted = models.BooleanField(default=False, db_index=True)
     # Some comments with KeyFactors can have empty text
     text = models.TextField(max_length=150_000, blank=True)
+    # Set by the `archive_bot_comment_texts` command. The full text lives in S3
+    # under a key derived from the comment id, so no pointer is stored here.
+    # example path:
+    #     /s3/buckets/metaculus-web-content-blobs?prefix=comments_text/<comment_id>.json
+    is_text_archived = models.BooleanField(
+        default=False,
+        editable=False,
+        help_text="True if the full text has been moved to S3 and only a "
+        "truncated stub remains in the text columns. Archived comments "
+        "cannot be edited; use the comment-detail endpoint to read them.",
+        db_index=True,
+    )
     on_post = models.ForeignKey(
         Post, models.CASCADE, null=True, related_name="comments"
     )