From 840951dd77da513f8bff52a2f7c6dc95c400452c Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 22 Jan 2025 09:55:43 +0100 Subject: [PATCH 01/13] Add `published` column --- .../6cac7b706953_add_published_field.py | 42 +++++++++++++++++++ warehouse/packaging/models.py | 3 +- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 warehouse/migrations/versions/6cac7b706953_add_published_field.py diff --git a/warehouse/migrations/versions/6cac7b706953_add_published_field.py b/warehouse/migrations/versions/6cac7b706953_add_published_field.py new file mode 100644 index 000000000000..e00c3646cfb5 --- /dev/null +++ b/warehouse/migrations/versions/6cac7b706953_add_published_field.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +add published field + +Revision ID: 6cac7b706953 +Revises: 2a2c32c47a8f +Create Date: 2025-01-22 08:49:17.030343 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "6cac7b706953" +down_revision = "2a2c32c47a8f" + + +def upgrade(): + conn = op.get_bind() + conn.execute(sa.text("SET statement_timeout = 120000")) + conn.execute(sa.text("SET lock_timeout = 120000")) + + op.add_column( + "releases", + sa.Column( + "published", sa.Boolean(), server_default=sa.text("true"), nullable=False + ), + ) + + +def downgrade(): + op.drop_column("releases", "published") diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 476638de019b..8f6fa0d63bd3 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -79,7 +79,7 @@ from warehouse.sitemap.models import SitemapMixin from warehouse.utils import dotted_navigator, wheel from warehouse.utils.attrs import make_repr -from warehouse.utils.db.types import bool_false, datetime_now +from warehouse.utils.db.types import bool_false, bool_true, datetime_now if typing.TYPE_CHECKING: from warehouse.oidc.models import OIDCPublisher @@ -633,6 +633,7 @@ def __table_args__(cls): # noqa _pypi_ordering: Mapped[int | None] requires_python: Mapped[str | None] = mapped_column(Text) created: Mapped[datetime_now] = mapped_column() + published: Mapped[bool_true] = mapped_column() description_id: Mapped[UUID] = mapped_column( ForeignKey("release_descriptions.id", onupdate="CASCADE", ondelete="CASCADE"), From deddcc1d5d006e7137d4ed7c2dbf330e4785bdb7 Mon Sep 17 00:00:00 2001 From: Alan Velasco Date: Mon, 2 May 2022 14:24:40 -0600 Subject: [PATCH 02/13] Add `published` to the `ReleaseFactory` --- tests/common/db/packaging.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 3b97b20cbd92..94bb396407f2 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -100,6 +100,9 @@ class Meta: uploader = factory.SubFactory(UserFactory) description = factory.SubFactory(DescriptionFactory) + published = factory.Faker( + "date_time_between_dates", datetime_start=datetime.datetime(2008, 1, 1) + ) class FileFactory(WarehouseFactory): From c8affcefbd506c778f4d37cd52bc807d8c0c3224 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 6 Dec 2024 14:26:19 +0100 Subject: [PATCH 03/13] Add migrations --- .../3e7bf3217166_add_published_in_release.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py diff --git a/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py b/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py new file mode 100644 index 000000000000..7303a6f00062 --- /dev/null +++ b/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +add published in Release + +Revision ID: 3e7bf3217166 +Revises: f7720656a33c +Create Date: 2024-12-06 11:04:21.907167 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "3e7bf3217166" +down_revision = "f7720656a33c" + + +def upgrade(): + op.add_column( + "releases", + sa.Column( + "published", sa.DateTime(), server_default=sa.text("now()"), nullable=True + ), + ) + + op.execute( + """ + UPDATE releases + SET published = created + """ + ) + + +def downgrade(): + op.drop_column("releases", "published") From f830b63aa8e6c3cb54eaf16d3b07a5e432d6e728 Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 9 Dec 2024 14:47:59 +0100 Subject: [PATCH 04/13] Add a default value for Release.published field. --- tests/common/db/packaging.py | 3 +- tests/unit/packaging/test_views.py | 10 ++ warehouse/forklift/legacy.py | 2 + .../3e7bf3217166_add_published_in_release.py | 4 +- warehouse/packaging/models.py | 112 +++++++----------- 5 files changed, 57 insertions(+), 74 deletions(-) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 94bb396407f2..68f728f44a93 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -123,7 +123,8 @@ class Meta: lambda o: hashlib.blake2b(o.filename.encode("utf8"), digest_size=32).hexdigest() ) upload_time = factory.Faker( - "date_time_between_dates", datetime_start=datetime.datetime(2008, 1, 1) + "date_time_between_dates", + datetime_start=datetime.datetime(2008, 1, 1), ) path = factory.LazyAttribute( lambda o: "/".join( diff --git a/tests/unit/packaging/test_views.py b/tests/unit/packaging/test_views.py index f914c3e97b1b..fe070d95ef67 100644 --- a/tests/unit/packaging/test_views.py +++ b/tests/unit/packaging/test_views.py @@ -9,6 +9,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime import pretend import pytest @@ -324,6 +325,15 @@ def test_long_singleline_license(self, db_request): "characters, it's really so lo..." ) + def test_created_with_published(self, db_request): + release = ReleaseFactory.create() + assert release.published > datetime(year=2008, month=1, day=1) + + def test_without_published_date(self, db_request): + release = ReleaseFactory.create(published=None) + db_request.db.flush() + assert release.published is None + class TestPEP740AttestationViewer: diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index a3018ecdb2bd..4ea70897fcdc 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -19,6 +19,7 @@ import zipfile from cgi import FieldStorage +from datetime import datetime import packaging.requirements import packaging.specifiers @@ -902,6 +903,7 @@ def file_upload(request): }, uploader=request.user if request.user else None, uploaded_via=request.user_agent, + published=datetime.now(), ) request.db.add(release) is_new_release = True diff --git a/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py b/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py index 7303a6f00062..9bdb059c1f4c 100644 --- a/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py +++ b/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py @@ -28,9 +28,7 @@ def upgrade(): op.add_column( "releases", - sa.Column( - "published", sa.DateTime(), server_default=sa.text("now()"), nullable=True - ), + sa.Column("published", sa.DateTime(), nullable=True), ) op.execute( diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 8f6fa0d63bd3..dfb8c56edce0 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -79,7 +79,7 @@ from warehouse.sitemap.models import SitemapMixin from warehouse.utils import dotted_navigator, wheel from warehouse.utils.attrs import make_repr -from warehouse.utils.db.types import bool_false, bool_true, datetime_now +from warehouse.utils.db.types import bool_false, datetime_now if typing.TYPE_CHECKING: from warehouse.oidc.models import OIDCPublisher @@ -166,7 +166,6 @@ def __contains__(self, project): class LifecycleStatus(enum.StrEnum): QuarantineEnter = "quarantine-enter" QuarantineExit = "quarantine-exit" - Archived = "archived" class Project(SitemapMixin, HasEvents, HasObservations, db.Model): @@ -328,36 +327,25 @@ def __acl__(self): (Allow, Authenticated, Permissions.SubmitMalwareObservation), ] - if self.lifecycle_status != LifecycleStatus.Archived: - # The project has zero or more OIDC publishers registered to it, - # each of which serves as an identity with the ability to upload releases - # (only if the project is not archived) - for publisher in self.oidc_publishers: - acls.append( - (Allow, f"oidc:{publisher.id}", [Permissions.ProjectsUpload]) - ) + # The project has zero or more OIDC publishers registered to it, + # each of which serves as an identity with the ability to upload releases. + for publisher in self.oidc_publishers: + acls.append((Allow, f"oidc:{publisher.id}", [Permissions.ProjectsUpload])) # Get all of the users for this project. - user_query = ( - session.query(Role) - .filter(Role.project == self) - .options(orm.lazyload(Role.project), orm.lazyload(Role.user)) - ) + query = session.query(Role).filter(Role.project == self) + query = query.options(orm.lazyload(Role.project)) + query = query.options(orm.lazyload(Role.user)) permissions = { (role.user_id, "Administer" if role.role_name == "Owner" else "Upload") - for role in user_query.all() + for role in query.all() } # Add all of the team members for this project. - team_query = ( - session.query(TeamProjectRole) - .filter(TeamProjectRole.project == self) - .options( - orm.lazyload(TeamProjectRole.project), - orm.lazyload(TeamProjectRole.team), - ) - ) - for role in team_query.all(): + query = session.query(TeamProjectRole).filter(TeamProjectRole.project == self) + query = query.options(orm.lazyload(TeamProjectRole.project)) + query = query.options(orm.lazyload(TeamProjectRole.team)) + for role in query.all(): permissions |= { (user.id, "Administer" if role.role_name.value == "Owner" else "Upload") for user in role.team.members @@ -365,41 +353,38 @@ def __acl__(self): # Add all organization owners for this project. if self.organization: - org_query = ( - session.query(OrganizationRole) - .filter( - OrganizationRole.organization == self.organization, - OrganizationRole.role_name == OrganizationRoleType.Owner, - ) - .options( - orm.lazyload(OrganizationRole.organization), - orm.lazyload(OrganizationRole.user), - ) + query = session.query(OrganizationRole).filter( + OrganizationRole.organization == self.organization, + OrganizationRole.role_name == OrganizationRoleType.Owner, ) - permissions |= {(role.user_id, "Administer") for role in org_query.all()} + query = query.options(orm.lazyload(OrganizationRole.organization)) + query = query.options(orm.lazyload(OrganizationRole.user)) + permissions |= {(role.user_id, "Administer") for role in query.all()} for user_id, permission_name in sorted(permissions, key=lambda x: (x[1], x[0])): # Disallow Write permissions for Projects in quarantine, allow Upload if self.lifecycle_status == LifecycleStatus.QuarantineEnter: - current_permissions = [ - Permissions.ProjectsRead, - Permissions.ProjectsUpload, - ] + acls.append( + ( + Allow, + f"user:{user_id}", + [Permissions.ProjectsRead, Permissions.ProjectsUpload], + ) + ) elif permission_name == "Administer": - current_permissions = [ - Permissions.ProjectsRead, - Permissions.ProjectsUpload, - Permissions.ProjectsWrite, - ] + acls.append( + ( + Allow, + f"user:{user_id}", + [ + Permissions.ProjectsRead, + Permissions.ProjectsUpload, + Permissions.ProjectsWrite, + ], + ) + ) else: - current_permissions = [Permissions.ProjectsUpload] - - if self.lifecycle_status == LifecycleStatus.Archived: - # Disallow upload permissions for archived projects - current_permissions.remove(Permissions.ProjectsUpload) - - if current_permissions: - acls.append((Allow, f"user:{user_id}", current_permissions)) + acls.append((Allow, f"user:{user_id}", [Permissions.ProjectsUpload])) return acls @property @@ -550,8 +535,8 @@ class ReleaseURL(db.Model): "Description", "Description-Content-Type", "Keywords", - "Home-Page", # Deprecated, but technically permitted by PEP 643 - "Download-Url", # Deprecated, but technically permitted by PEP 643 + "Home-Page", + "Download-Url", "Author", "Author-Email", "Maintainer", @@ -567,12 +552,6 @@ class ReleaseURL(db.Model): "Provides-Extra", "Provides-Dist", "Obsoletes-Dist", - # Although the following are deprecated fields, they are technically - # permitted as dynamic by PEP 643 - # https://github.com/pypa/setuptools/issues/4797#issuecomment-2589514950 - "Requires", - "Provides", - "Obsoletes", name="release_dynamic_fields", ) @@ -633,7 +612,7 @@ def __table_args__(cls): # noqa _pypi_ordering: Mapped[int | None] requires_python: Mapped[str | None] = mapped_column(Text) created: Mapped[datetime_now] = mapped_column() - published: Mapped[bool_true] = mapped_column() + published: Mapped[datetime_now | None] description_id: Mapped[UUID] = mapped_column( ForeignKey("release_descriptions.id", onupdate="CASCADE", ondelete="CASCADE"), @@ -781,7 +760,7 @@ def urls_by_verify_status(self, *, verified: bool): return _urls def verified_user_name_and_repo_name( - self, domains: set[str], reserved_names: typing.Collection[str] | None = None + self, domains: set[str], reserved_names: typing.Sequence[str] | None = None ): for _, url in self.urls_by_verify_status(verified=True).items(): try: @@ -1008,13 +987,6 @@ def __table_args__(cls): # noqa Index("journals_version_idx", "version"), Index("journals_submitted_by_idx", "submitted_by"), Index("journals_submitted_date_id_idx", cls.submitted_date, cls.id), - # Composite index for journals to be able to sort by - # `submitted_by`, and `submitted_date` in descending order. - Index( - "journals_submitted_by_and_reverse_date_idx", - cls._submitted_by, - cls.submitted_date.desc(), - ), ) id: Mapped[int] = mapped_column(primary_key=True) From cdf1acb394cdb80c6015bc6e18ab01a47d2c0b4b Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 10 Dec 2024 14:46:53 +0100 Subject: [PATCH 05/13] Change to a boolean field --- tests/common/db/packaging.py | 3 --- tests/unit/forklift/test_legacy.py | 1 + tests/unit/packaging/test_views.py | 8 +------- warehouse/forklift/legacy.py | 3 +-- ...py => bd2bf218e63f_add_published_field.py} | 19 +++++++------------ .../templates/manage/project/history.html | 8 ++++++++ 6 files changed, 18 insertions(+), 24 deletions(-) rename warehouse/migrations/versions/{3e7bf3217166_add_published_in_release.py => bd2bf218e63f_add_published_field.py} (73%) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 68f728f44a93..470f233a34c4 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -100,9 +100,6 @@ class Meta: uploader = factory.SubFactory(UserFactory) description = factory.SubFactory(DescriptionFactory) - published = factory.Faker( - "date_time_between_dates", datetime_start=datetime.datetime(2008, 1, 1) - ) class FileFactory(WarehouseFactory): diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 05959699419b..3042251d7f4a 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3613,6 +3613,7 @@ def test_upload_succeeds_creates_release( else None ), "uploaded_via_trusted_publisher": not test_with_user, + "published": True, } fileadd_event = { diff --git a/tests/unit/packaging/test_views.py b/tests/unit/packaging/test_views.py index fe070d95ef67..dc6ac4156dfd 100644 --- a/tests/unit/packaging/test_views.py +++ b/tests/unit/packaging/test_views.py @@ -9,7 +9,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime import pretend import pytest @@ -327,12 +326,7 @@ def test_long_singleline_license(self, db_request): def test_created_with_published(self, db_request): release = ReleaseFactory.create() - assert release.published > datetime(year=2008, month=1, day=1) - - def test_without_published_date(self, db_request): - release = ReleaseFactory.create(published=None) - db_request.db.flush() - assert release.published is None + assert release.published is True class TestPEP740AttestationViewer: diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 4ea70897fcdc..863076c87383 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -19,7 +19,6 @@ import zipfile from cgi import FieldStorage -from datetime import datetime import packaging.requirements import packaging.specifiers @@ -903,7 +902,6 @@ def file_upload(request): }, uploader=request.user if request.user else None, uploaded_via=request.user_agent, - published=datetime.now(), ) request.db.add(release) is_new_release = True @@ -934,6 +932,7 @@ def file_upload(request): else None ), "uploaded_via_trusted_publisher": bool(request.oidc_publisher), + "published": True, }, ) diff --git a/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py b/warehouse/migrations/versions/bd2bf218e63f_add_published_field.py similarity index 73% rename from warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py rename to warehouse/migrations/versions/bd2bf218e63f_add_published_field.py index 9bdb059c1f4c..313e65679ecd 100644 --- a/warehouse/migrations/versions/3e7bf3217166_add_published_in_release.py +++ b/warehouse/migrations/versions/bd2bf218e63f_add_published_field.py @@ -10,32 +10,27 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -add published in Release +add published field -Revision ID: 3e7bf3217166 +Revision ID: bd2bf218e63f Revises: f7720656a33c -Create Date: 2024-12-06 11:04:21.907167 +Create Date: 2024-12-10 10:40:19.588606 """ import sqlalchemy as sa from alembic import op -revision = "3e7bf3217166" +revision = "bd2bf218e63f" down_revision = "f7720656a33c" def upgrade(): op.add_column( "releases", - sa.Column("published", sa.DateTime(), nullable=True), - ) - - op.execute( - """ - UPDATE releases - SET published = created - """ + sa.Column( + "published", sa.Boolean(), server_default=sa.text("true"), nullable=False + ), ) diff --git a/warehouse/templates/manage/project/history.html b/warehouse/templates/manage/project/history.html index 6ca65478a265..002d80500033 100644 --- a/warehouse/templates/manage/project/history.html +++ b/warehouse/templates/manage/project/history.html @@ -53,6 +53,14 @@

{% trans %}Security history{% endtrans %}

{{ event.additional.publisher_url }} {% endif %} + + {% trans %}Published:{% endtrans %} + {% if event.additional.published is defined and event.additional.published is false %} + {% trans %}No{% endtrans %} + {% else %} + {% trans %}Yes{% endtrans %} + {% endif %} + {% elif event.tag == EventTag.Project.ReleaseRemove %} {# No link to removed release #} From d3f59b42e474baaa1aad80c824af0d003b44b011 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 10 Dec 2024 18:39:04 +0100 Subject: [PATCH 06/13] Filter out unpublished releases --- tests/unit/legacy/api/test_json.py | 16 +++++++ tests/unit/packaging/test_views.py | 77 ++++++++++++++++++++---------- warehouse/legacy/api/json.py | 8 +++- warehouse/locale/messages.pot | 2 +- warehouse/packaging/models.py | 6 ++- warehouse/packaging/utils.py | 4 +- warehouse/packaging/views.py | 5 +- 7 files changed, 88 insertions(+), 30 deletions(-) diff --git a/tests/unit/legacy/api/test_json.py b/tests/unit/legacy/api/test_json.py index 3a7b029e8c7e..5edea5594161 100644 --- a/tests/unit/legacy/api/test_json.py +++ b/tests/unit/legacy/api/test_json.py @@ -118,6 +118,13 @@ def test_all_non_prereleases_yanked(self, monkeypatch, db_request): db_request.matchdict = {"name": project.normalized_name} assert json.latest_release_factory(db_request) == release + def test_with_unpublished(self, db_request): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + ReleaseFactory.create(project=project, version="2.0", published=False) + db_request.matchdict = {"name": project.normalized_name} + assert json.latest_release_factory(db_request) == release + def test_project_quarantined(self, monkeypatch, db_request): project = ProjectFactory.create( lifecycle_status=LifecycleStatus.QuarantineEnter @@ -191,6 +198,15 @@ def test_renders(self, pyramid_config, db_request, db_session): ) ] + ReleaseFactory.create( + project=project, + version="3.1", + description=DescriptionFactory.create( + content_type=description_content_type + ), + published=False, + ) + for urlspec in project_urls: label, _, purl = urlspec.partition(",") db_session.add( diff --git a/tests/unit/packaging/test_views.py b/tests/unit/packaging/test_views.py index dc6ac4156dfd..a8ea5f6f805d 100644 --- a/tests/unit/packaging/test_views.py +++ b/tests/unit/packaging/test_views.py @@ -135,6 +135,19 @@ def test_only_yanked_release(self, monkeypatch, db_request): assert resp is response assert release_detail.calls == [pretend.call(release, db_request)] + def test_with_unpublished(self, monkeypatch, db_request): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + ReleaseFactory.create(project=project, version="1.1", published=False) + + response = pretend.stub() + release_detail = pretend.call_recorder(lambda ctx, request: response) + monkeypatch.setattr(views, "release_detail", release_detail) + + resp = views.project_detail(project, db_request) + assert resp is response + assert release_detail.calls == [pretend.call(release, db_request)] + class TestReleaseDetail: def test_normalizing_name_redirects(self, db_request): @@ -178,30 +191,45 @@ def test_normalizing_version_redirects(self, db_request): def test_detail_rendered(self, db_request): users = [UserFactory.create(), UserFactory.create(), UserFactory.create()] project = ProjectFactory.create() - releases = [ - ReleaseFactory.create( - project=project, - version=v, - description=DescriptionFactory.create( - raw="unrendered description", - html="rendered description", - content_type="text/html", - ), - ) - for v in ["1.0", "2.0", "3.0", "4.0.dev0"] - ] + [ - ReleaseFactory.create( - project=project, - version="5.0", - description=DescriptionFactory.create( - raw="plaintext description", - html="", - content_type="text/plain", - ), - yanked=True, - yanked_reason="plaintext yanked reason", - ) - ] + releases = ( + [ + ReleaseFactory.create( + project=project, + version=v, + description=DescriptionFactory.create( + raw="unrendered description", + html="rendered description", + content_type="text/html", + ), + ) + for v in ["1.0", "2.0", "3.0", "4.0.dev0"] + ] + + [ + ReleaseFactory.create( + project=project, + version="5.0", + description=DescriptionFactory.create( + raw="plaintext description", + html="", + content_type="text/plain", + ), + yanked=True, + yanked_reason="plaintext yanked reason", + ) + ] + + [ + ReleaseFactory.create( + project=project, + version="5.1", + description=DescriptionFactory.create( + raw="unrendered description", + html="rendered description", + content_type="text/html", + ), + published=False, + ) + ] + ) files = [ FileFactory.create( release=r, @@ -226,6 +254,7 @@ def test_detail_rendered(self, db_request): "bdists": [], "description": "rendered description", "latest_version": project.latest_version, + # Non published version are not listed here "all_versions": [ (r.version, r.created, r.is_prerelease, r.yanked, r.yanked_reason) for r in reversed(releases) diff --git a/warehouse/legacy/api/json.py b/warehouse/legacy/api/json.py index 513699cbd665..e7b8652527af 100644 --- a/warehouse/legacy/api/json.py +++ b/warehouse/legacy/api/json.py @@ -62,7 +62,10 @@ def _json_data(request, project, release, *, all_releases): ) ) .outerjoin(File) - .filter(Release.project == project) + .filter( + Release.project == project, + Release.published.is_(True), + ) ) # If we're not looking for all_releases, then we'll filter this further @@ -206,7 +209,8 @@ def latest_release_factory(request): .filter( Project.lifecycle_status.is_distinct_from( LifecycleStatus.QuarantineEnter - ) + ), + Release.published.is_(True), ) .order_by( Release.yanked.asc(), diff --git a/warehouse/locale/messages.pot b/warehouse/locale/messages.pot index fcf8eabcf41e..c732859b809c 100644 --- a/warehouse/locale/messages.pot +++ b/warehouse/locale/messages.pot @@ -784,7 +784,7 @@ msgstr "" msgid "Provide an Inspector link to specific lines of code." msgstr "" -#: warehouse/packaging/views.py:352 +#: warehouse/packaging/views.py:355 msgid "Your report has been recorded. Thank you for your help." msgstr "" diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index dfb8c56edce0..c2d608ae8f0a 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -454,7 +454,11 @@ def latest_version(self): return ( orm.object_session(self) .query(Release.version, Release.created, Release.is_prerelease) - .filter(Release.project == self, Release.yanked.is_(False)) + .filter( + Release.project == self, + Release.yanked.is_(False), + Release.published.is_(True), + ) .order_by(Release.is_prerelease.nullslast(), Release._pypi_ordering.desc()) .first() ) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 7397cf45a740..53bdc488a04a 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -53,9 +53,11 @@ def _simple_detail(project, request): .join(Release) .filter(Release.project == project) # Exclude projects that are in the `quarantine-enter` lifecycle status. + # And exclude un-published releases from the index .join(Project) .filter( - Project.lifecycle_status.is_distinct_from(LifecycleStatus.QuarantineEnter) + Project.lifecycle_status.is_distinct_from(LifecycleStatus.QuarantineEnter), + Release.published.is_(True), ) .all(), key=lambda f: (packaging_legacy.version.parse(f.release.version), f.filename), diff --git a/warehouse/packaging/views.py b/warehouse/packaging/views.py index 55f722c43c0b..d1c3998f293e 100644 --- a/warehouse/packaging/views.py +++ b/warehouse/packaging/views.py @@ -179,7 +179,10 @@ def project_detail(project, request): try: release = ( request.db.query(Release) - .filter(Release.project == project) + .filter( + Release.project == project, + Release.published.is_(True), + ) .order_by( Release.yanked, Release.is_prerelease.nullslast(), From 8fcb92e1e9966d2387eb0bfc87978cab7dabcf9f Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 3 Jan 2025 12:03:09 +0100 Subject: [PATCH 07/13] Use a SQLAlchemy event --- tests/unit/packaging/test_views.py | 76 +++++++++++++++--------------- warehouse/legacy/api/json.py | 2 - warehouse/locale/messages.pot | 2 +- warehouse/packaging/models.py | 22 +++++++-- warehouse/packaging/utils.py | 1 - warehouse/packaging/views.py | 5 +- 6 files changed, 58 insertions(+), 50 deletions(-) diff --git a/tests/unit/packaging/test_views.py b/tests/unit/packaging/test_views.py index a8ea5f6f805d..f2139dc97a36 100644 --- a/tests/unit/packaging/test_views.py +++ b/tests/unit/packaging/test_views.py @@ -191,45 +191,43 @@ def test_normalizing_version_redirects(self, db_request): def test_detail_rendered(self, db_request): users = [UserFactory.create(), UserFactory.create(), UserFactory.create()] project = ProjectFactory.create() - releases = ( - [ - ReleaseFactory.create( - project=project, - version=v, - description=DescriptionFactory.create( - raw="unrendered description", - html="rendered description", - content_type="text/html", - ), - ) - for v in ["1.0", "2.0", "3.0", "4.0.dev0"] - ] - + [ - ReleaseFactory.create( - project=project, - version="5.0", - description=DescriptionFactory.create( - raw="plaintext description", - html="", - content_type="text/plain", - ), - yanked=True, - yanked_reason="plaintext yanked reason", - ) - ] - + [ - ReleaseFactory.create( - project=project, - version="5.1", - description=DescriptionFactory.create( - raw="unrendered description", - html="rendered description", - content_type="text/html", - ), - published=False, - ) - ] + releases = [ + ReleaseFactory.create( + project=project, + version=v, + description=DescriptionFactory.create( + raw="unrendered description", + html="rendered description", + content_type="text/html", + ), + ) + for v in ["1.0", "2.0", "3.0", "4.0.dev0"] + ] + [ + ReleaseFactory.create( + project=project, + version="5.0", + description=DescriptionFactory.create( + raw="plaintext description", + html="", + content_type="text/plain", + ), + yanked=True, + yanked_reason="plaintext yanked reason", + ) + ] + + # Add an unpublished version + staged_release = ReleaseFactory.create( + project=project, + version="5.1", + description=DescriptionFactory.create( + raw="unrendered description", + html="rendered description", + content_type="text/html", + ), + published=False, ) + files = [ FileFactory.create( release=r, @@ -237,7 +235,7 @@ def test_detail_rendered(self, db_request): python_version="source", packagetype="sdist", ) - for r in releases + for r in releases + [staged_release] ] # Create a role for each user diff --git a/warehouse/legacy/api/json.py b/warehouse/legacy/api/json.py index e7b8652527af..8a95b4ccbb6d 100644 --- a/warehouse/legacy/api/json.py +++ b/warehouse/legacy/api/json.py @@ -64,7 +64,6 @@ def _json_data(request, project, release, *, all_releases): .outerjoin(File) .filter( Release.project == project, - Release.published.is_(True), ) ) @@ -210,7 +209,6 @@ def latest_release_factory(request): Project.lifecycle_status.is_distinct_from( LifecycleStatus.QuarantineEnter ), - Release.published.is_(True), ) .order_by( Release.yanked.asc(), diff --git a/warehouse/locale/messages.pot b/warehouse/locale/messages.pot index c732859b809c..fcf8eabcf41e 100644 --- a/warehouse/locale/messages.pot +++ b/warehouse/locale/messages.pot @@ -784,7 +784,7 @@ msgstr "" msgid "Provide an Inspector link to specific lines of code." msgstr "" -#: warehouse/packaging/views.py:355 +#: warehouse/packaging/views.py:352 msgid "Your report has been recorded. Thank you for your help." msgstr "" diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index c2d608ae8f0a..3b9fa99c745d 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -52,10 +52,12 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import ( Mapped, + ORMExecuteState, attribute_keyed_dict, declared_attr, mapped_column, validates, + with_loader_criteria, ) from urllib3.exceptions import LocationParseError from urllib3.util import parse_url @@ -79,7 +81,7 @@ from warehouse.sitemap.models import SitemapMixin from warehouse.utils import dotted_navigator, wheel from warehouse.utils.attrs import make_repr -from warehouse.utils.db.types import bool_false, datetime_now +from warehouse.utils.db.types import bool_false, bool_true, datetime_now if typing.TYPE_CHECKING: from warehouse.oidc.models import OIDCPublisher @@ -457,7 +459,6 @@ def latest_version(self): .filter( Release.project == self, Release.yanked.is_(False), - Release.published.is_(True), ) .order_by(Release.is_prerelease.nullslast(), Release._pypi_ordering.desc()) .first() @@ -616,7 +617,7 @@ def __table_args__(cls): # noqa _pypi_ordering: Mapped[int | None] requires_python: Mapped[str | None] = mapped_column(Text) created: Mapped[datetime_now] = mapped_column() - published: Mapped[datetime_now | None] + published: Mapped[bool_true] = mapped_column() description_id: Mapped[UUID] = mapped_column( ForeignKey("release_descriptions.id", onupdate="CASCADE", ondelete="CASCADE"), @@ -1034,6 +1035,21 @@ def ensure_monotonic_journals(config, session, flush_context, instances): return +@db.listens_for(db.Session, "do_orm_execute") +def filter_staged_release(config, state: ORMExecuteState): + if ( + state.is_select + and not state.is_column_load + and not state.is_relationship_load + and not state.statement.get_execution_options().get("include_staged", False) + ): + state.statement = state.statement.options( + with_loader_criteria( + Release, lambda cls: cls.published, include_aliases=True + ) + ) + + class ProhibitedProjectName(db.Model): __tablename__ = "prohibited_project_names" __table_args__ = ( diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 53bdc488a04a..b58e62610871 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -57,7 +57,6 @@ def _simple_detail(project, request): .join(Project) .filter( Project.lifecycle_status.is_distinct_from(LifecycleStatus.QuarantineEnter), - Release.published.is_(True), ) .all(), key=lambda f: (packaging_legacy.version.parse(f.release.version), f.filename), diff --git a/warehouse/packaging/views.py b/warehouse/packaging/views.py index d1c3998f293e..55f722c43c0b 100644 --- a/warehouse/packaging/views.py +++ b/warehouse/packaging/views.py @@ -179,10 +179,7 @@ def project_detail(project, request): try: release = ( request.db.query(Release) - .filter( - Release.project == project, - Release.published.is_(True), - ) + .filter(Release.project == project) .order_by( Release.yanked, Release.is_prerelease.nullslast(), From cf2abc801bd9966b2eefb5ecb009c163f79fd9af Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 22 Jan 2025 10:20:40 +0100 Subject: [PATCH 08/13] Remove field from PR --- .../bd2bf218e63f_add_published_field.py | 38 ------------------- warehouse/packaging/models.py | 1 - 2 files changed, 39 deletions(-) delete mode 100644 warehouse/migrations/versions/bd2bf218e63f_add_published_field.py diff --git a/warehouse/migrations/versions/bd2bf218e63f_add_published_field.py b/warehouse/migrations/versions/bd2bf218e63f_add_published_field.py deleted file mode 100644 index 313e65679ecd..000000000000 --- a/warehouse/migrations/versions/bd2bf218e63f_add_published_field.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -add published field - -Revision ID: bd2bf218e63f -Revises: f7720656a33c -Create Date: 2024-12-10 10:40:19.588606 -""" - -import sqlalchemy as sa - -from alembic import op - -revision = "bd2bf218e63f" -down_revision = "f7720656a33c" - - -def upgrade(): - op.add_column( - "releases", - sa.Column( - "published", sa.Boolean(), server_default=sa.text("true"), nullable=False - ), - ) - - -def downgrade(): - op.drop_column("releases", "published") diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 3b9fa99c745d..0b5dcc5f8856 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -617,7 +617,6 @@ def __table_args__(cls): # noqa _pypi_ordering: Mapped[int | None] requires_python: Mapped[str | None] = mapped_column(Text) created: Mapped[datetime_now] = mapped_column() - published: Mapped[bool_true] = mapped_column() description_id: Mapped[UUID] = mapped_column( ForeignKey("release_descriptions.id", onupdate="CASCADE", ondelete="CASCADE"), From eb025b68576712073e0ec6621429330468f5ffd9 Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 22 Jan 2025 10:46:58 +0100 Subject: [PATCH 09/13] Resolve merge conflict --- warehouse/packaging/models.py | 114 +++++++++++++++++++++------------- 1 file changed, 70 insertions(+), 44 deletions(-) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 0b5dcc5f8856..751657373a62 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -168,6 +168,7 @@ def __contains__(self, project): class LifecycleStatus(enum.StrEnum): QuarantineEnter = "quarantine-enter" QuarantineExit = "quarantine-exit" + Archived = "archived" class Project(SitemapMixin, HasEvents, HasObservations, db.Model): @@ -329,25 +330,36 @@ def __acl__(self): (Allow, Authenticated, Permissions.SubmitMalwareObservation), ] - # The project has zero or more OIDC publishers registered to it, - # each of which serves as an identity with the ability to upload releases. - for publisher in self.oidc_publishers: - acls.append((Allow, f"oidc:{publisher.id}", [Permissions.ProjectsUpload])) + if self.lifecycle_status != LifecycleStatus.Archived: + # The project has zero or more OIDC publishers registered to it, + # each of which serves as an identity with the ability to upload releases + # (only if the project is not archived) + for publisher in self.oidc_publishers: + acls.append( + (Allow, f"oidc:{publisher.id}", [Permissions.ProjectsUpload]) + ) # Get all of the users for this project. - query = session.query(Role).filter(Role.project == self) - query = query.options(orm.lazyload(Role.project)) - query = query.options(orm.lazyload(Role.user)) + user_query = ( + session.query(Role) + .filter(Role.project == self) + .options(orm.lazyload(Role.project), orm.lazyload(Role.user)) + ) permissions = { (role.user_id, "Administer" if role.role_name == "Owner" else "Upload") - for role in query.all() + for role in user_query.all() } # Add all of the team members for this project. - query = session.query(TeamProjectRole).filter(TeamProjectRole.project == self) - query = query.options(orm.lazyload(TeamProjectRole.project)) - query = query.options(orm.lazyload(TeamProjectRole.team)) - for role in query.all(): + team_query = ( + session.query(TeamProjectRole) + .filter(TeamProjectRole.project == self) + .options( + orm.lazyload(TeamProjectRole.project), + orm.lazyload(TeamProjectRole.team), + ) + ) + for role in team_query.all(): permissions |= { (user.id, "Administer" if role.role_name.value == "Owner" else "Upload") for user in role.team.members @@ -355,38 +367,41 @@ def __acl__(self): # Add all organization owners for this project. if self.organization: - query = session.query(OrganizationRole).filter( - OrganizationRole.organization == self.organization, - OrganizationRole.role_name == OrganizationRoleType.Owner, + org_query = ( + session.query(OrganizationRole) + .filter( + OrganizationRole.organization == self.organization, + OrganizationRole.role_name == OrganizationRoleType.Owner, + ) + .options( + orm.lazyload(OrganizationRole.organization), + orm.lazyload(OrganizationRole.user), + ) ) - query = query.options(orm.lazyload(OrganizationRole.organization)) - query = query.options(orm.lazyload(OrganizationRole.user)) - permissions |= {(role.user_id, "Administer") for role in query.all()} + permissions |= {(role.user_id, "Administer") for role in org_query.all()} for user_id, permission_name in sorted(permissions, key=lambda x: (x[1], x[0])): # Disallow Write permissions for Projects in quarantine, allow Upload if self.lifecycle_status == LifecycleStatus.QuarantineEnter: - acls.append( - ( - Allow, - f"user:{user_id}", - [Permissions.ProjectsRead, Permissions.ProjectsUpload], - ) - ) + current_permissions = [ + Permissions.ProjectsRead, + Permissions.ProjectsUpload, + ] elif permission_name == "Administer": - acls.append( - ( - Allow, - f"user:{user_id}", - [ - Permissions.ProjectsRead, - Permissions.ProjectsUpload, - Permissions.ProjectsWrite, - ], - ) - ) + current_permissions = [ + Permissions.ProjectsRead, + Permissions.ProjectsUpload, + Permissions.ProjectsWrite, + ] else: - acls.append((Allow, f"user:{user_id}", [Permissions.ProjectsUpload])) + current_permissions = [Permissions.ProjectsUpload] + + if self.lifecycle_status == LifecycleStatus.Archived: + # Disallow upload permissions for archived projects + current_permissions.remove(Permissions.ProjectsUpload) + + if current_permissions: + acls.append((Allow, f"user:{user_id}", current_permissions)) return acls @property @@ -456,10 +471,7 @@ def latest_version(self): return ( orm.object_session(self) .query(Release.version, Release.created, Release.is_prerelease) - .filter( - Release.project == self, - Release.yanked.is_(False), - ) + .filter(Release.project == self, Release.yanked.is_(False)) .order_by(Release.is_prerelease.nullslast(), Release._pypi_ordering.desc()) .first() ) @@ -540,8 +552,8 @@ class ReleaseURL(db.Model): "Description", "Description-Content-Type", "Keywords", - "Home-Page", - "Download-Url", + "Home-Page", # Deprecated, but technically permitted by PEP 643 + "Download-Url", # Deprecated, but technically permitted by PEP 643 "Author", "Author-Email", "Maintainer", @@ -557,6 +569,12 @@ class ReleaseURL(db.Model): "Provides-Extra", "Provides-Dist", "Obsoletes-Dist", + # Although the following are deprecated fields, they are technically + # permitted as dynamic by PEP 643 + # https://github.com/pypa/setuptools/issues/4797#issuecomment-2589514950 + "Requires", + "Provides", + "Obsoletes", name="release_dynamic_fields", ) @@ -617,6 +635,7 @@ def __table_args__(cls): # noqa _pypi_ordering: Mapped[int | None] requires_python: Mapped[str | None] = mapped_column(Text) created: Mapped[datetime_now] = mapped_column() + published: Mapped[bool_true] = mapped_column() description_id: Mapped[UUID] = mapped_column( ForeignKey("release_descriptions.id", onupdate="CASCADE", ondelete="CASCADE"), @@ -764,7 +783,7 @@ def urls_by_verify_status(self, *, verified: bool): return _urls def verified_user_name_and_repo_name( - self, domains: set[str], reserved_names: typing.Sequence[str] | None = None + self, domains: set[str], reserved_names: typing.Collection[str] | None = None ): for _, url in self.urls_by_verify_status(verified=True).items(): try: @@ -991,6 +1010,13 @@ def __table_args__(cls): # noqa Index("journals_version_idx", "version"), Index("journals_submitted_by_idx", "submitted_by"), Index("journals_submitted_date_id_idx", cls.submitted_date, cls.id), + # Composite index for journals to be able to sort by + # `submitted_by`, and `submitted_date` in descending order. + Index( + "journals_submitted_by_and_reverse_date_idx", + cls._submitted_by, + cls.submitted_date.desc(), + ), ) id: Mapped[int] = mapped_column(primary_key=True) From dec62389b7df61b5b6089dbdd31d0ed752e45144 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 28 Jan 2025 10:33:03 +0100 Subject: [PATCH 10/13] Revert un-needed changes --- warehouse/legacy/api/json.py | 6 ++---- warehouse/packaging/utils.py | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/warehouse/legacy/api/json.py b/warehouse/legacy/api/json.py index 8a95b4ccbb6d..513699cbd665 100644 --- a/warehouse/legacy/api/json.py +++ b/warehouse/legacy/api/json.py @@ -62,9 +62,7 @@ def _json_data(request, project, release, *, all_releases): ) ) .outerjoin(File) - .filter( - Release.project == project, - ) + .filter(Release.project == project) ) # If we're not looking for all_releases, then we'll filter this further @@ -208,7 +206,7 @@ def latest_release_factory(request): .filter( Project.lifecycle_status.is_distinct_from( LifecycleStatus.QuarantineEnter - ), + ) ) .order_by( Release.yanked.asc(), diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index b58e62610871..7397cf45a740 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -53,10 +53,9 @@ def _simple_detail(project, request): .join(Release) .filter(Release.project == project) # Exclude projects that are in the `quarantine-enter` lifecycle status. - # And exclude un-published releases from the index .join(Project) .filter( - Project.lifecycle_status.is_distinct_from(LifecycleStatus.QuarantineEnter), + Project.lifecycle_status.is_distinct_from(LifecycleStatus.QuarantineEnter) ) .all(), key=lambda f: (packaging_legacy.version.parse(f.release.version), f.filename), From 56feeb07250e461a71cc318c02b8eb843e261f33 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 28 Jan 2025 11:06:59 +0100 Subject: [PATCH 11/13] Remove event handling --- warehouse/templates/manage/project/history.html | 8 -------- 1 file changed, 8 deletions(-) diff --git a/warehouse/templates/manage/project/history.html b/warehouse/templates/manage/project/history.html index 002d80500033..6ca65478a265 100644 --- a/warehouse/templates/manage/project/history.html +++ b/warehouse/templates/manage/project/history.html @@ -53,14 +53,6 @@

{% trans %}Security history{% endtrans %}

{{ event.additional.publisher_url }} {% endif %} - - {% trans %}Published:{% endtrans %} - {% if event.additional.published is defined and event.additional.published is false %} - {% trans %}No{% endtrans %} - {% else %} - {% trans %}Yes{% endtrans %} - {% endif %} - {% elif event.tag == EventTag.Project.ReleaseRemove %} {# No link to removed release #} From 69060967e90ad264199c2e21b830af3f9c51ce96 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 28 Jan 2025 14:42:06 +0100 Subject: [PATCH 12/13] Add more tests --- tests/unit/forklift/test_legacy.py | 1 - tests/unit/legacy/api/test_json.py | 78 +++++++++++++++++++++++++++++ tests/unit/packaging/test_models.py | 28 +++++++++++ warehouse/forklift/legacy.py | 1 - warehouse/packaging/models.py | 2 +- 5 files changed, 107 insertions(+), 3 deletions(-) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 8177010fba1a..3ba814a87bef 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3586,7 +3586,6 @@ def test_upload_succeeds_creates_release( else None ), "uploaded_via_trusted_publisher": not test_with_user, - "published": True, } fileadd_event = { diff --git a/tests/unit/legacy/api/test_json.py b/tests/unit/legacy/api/test_json.py index 5edea5594161..1878fbe8cdf2 100644 --- a/tests/unit/legacy/api/test_json.py +++ b/tests/unit/legacy/api/test_json.py @@ -125,6 +125,84 @@ def test_with_unpublished(self, db_request): db_request.matchdict = {"name": project.normalized_name} assert json.latest_release_factory(db_request) == release + def test_only_unpublished(self, db_request): + project = ProjectFactory.create() + ReleaseFactory.create(project=project, version="1.0", published=False) + db_request.matchdict = {"name": project.normalized_name} + resp = json.latest_release_factory(db_request) + + assert isinstance(resp, HTTPNotFound) + _assert_has_cors_headers(resp.headers) + + @pytest.mark.parametrize( + ("release0_state", "release1_state", "release2_state", "latest_release"), + [ + ("published", "published", "published", 2), + ("published", "published", "unpublished", 1), + ("published", "published", "yanked", 1), + ("published", "unpublished", "published", 2), + ("published", "unpublished", "unpublished", 0), + ("published", "unpublished", "yanked", 0), + ("published", "yanked", "published", 2), + ("published", "yanked", "unpublished", 0), + ("published", "yanked", "yanked", 0), + ("unpublished", "published", "published", 2), + ("unpublished", "published", "unpublished", 1), + ("unpublished", "published", "yanked", 1), + ("unpublished", "unpublished", "published", 2), + ("unpublished", "unpublished", "unpublished", -1), + ("unpublished", "unpublished", "yanked", 2), # Same endpoint as none yanked + ("unpublished", "yanked", "published", 2), + ("unpublished", "yanked", "unpublished", 1), + ("unpublished", "yanked", "yanked", 2), + ("yanked", "published", "published", 2), + ("yanked", "published", "unpublished", 1), + ("yanked", "published", "yanked", 1), + ("yanked", "unpublished", "published", 2), + ("yanked", "unpublished", "unpublished", 0), + ("yanked", "unpublished", "yanked", 2), + ("yanked", "yanked", "published", 2), + ("yanked", "yanked", "unpublished", 1), + ("yanked", "yanked", "yanked", 2), + ], + ) + def test_with_mixed_states( + self, db_request, release0_state, release1_state, release2_state, latest_release + ): + project = ProjectFactory.create() + + releases = [] + for version, state in [ + ("1.0", release0_state), + ("1.1", release1_state), + ("2.0", release2_state), + ]: + if state == "published": + releases.append( + ReleaseFactory.create( + project=project, version=version, published=True + ) + ) + elif state == "unpublished": + releases.append( + ReleaseFactory.create( + project=project, version=version, published=False + ) + ) + else: + releases.append( + ReleaseFactory.create(project=project, version=version, yanked=True) + ) + + db_request.matchdict = {"name": project.normalized_name} + + resp = json.latest_release_factory(db_request) + if latest_release >= 0: + assert resp == releases[latest_release] + else: + assert isinstance(resp, HTTPNotFound) + _assert_has_cors_headers(resp.headers) + def test_project_quarantined(self, monkeypatch, db_request): project = ProjectFactory.create( lifecycle_status=LifecycleStatus.QuarantineEnter diff --git a/tests/unit/packaging/test_models.py b/tests/unit/packaging/test_models.py index 8a80d58129e2..6904a0c38a91 100644 --- a/tests/unit/packaging/test_models.py +++ b/tests/unit/packaging/test_models.py @@ -28,6 +28,7 @@ Project, ProjectFactory, ProjectMacaroonWarningAssociation, + Release, ReleaseURL, ) @@ -1215,6 +1216,33 @@ def test_description_relationship(self, db_request): assert release in db_request.db.deleted assert description in db_request.db.deleted + def test_published(self, db_request): + release = DBReleaseFactory.create() + assert release.published + + def test_unpublished(self, db_request): + release = DBReleaseFactory.create(published=False) + assert not release.published + + +@pytest.mark.parametrize( + "published", + [ + True, + False, + ], +) +def test_filter_staged_releases(db_request, published): + DBReleaseFactory.create(published=published) + assert db_request.db.query(Release).count() == (1 if published else 0) + + +def test_filter_staged_releases_unpublished(db_request): + DBReleaseFactory.create(published=False) + assert ( + db_request.db.query(Release).execution_options(include_staged=True).count() == 1 + ) + class TestFile: def test_requires_python(self, db_session): diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 4e92034a86cd..1dbddefd09c5 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -970,7 +970,6 @@ def file_upload(request): else None ), "uploaded_via_trusted_publisher": bool(request.oidc_publisher), - "published": True, }, ) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 751657373a62..0aea9d6241f2 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -1061,7 +1061,7 @@ def ensure_monotonic_journals(config, session, flush_context, instances): @db.listens_for(db.Session, "do_orm_execute") -def filter_staged_release(config, state: ORMExecuteState): +def filter_staged_release(_, state: ORMExecuteState): if ( state.is_select and not state.is_column_load From 9397ef202420686d873bb28ab14bea18b1e54992 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 28 Jan 2025 14:48:39 +0100 Subject: [PATCH 13/13] Rename from unpublished to staged --- tests/unit/legacy/api/test_json.py | 44 ++++++++++++++--------------- tests/unit/packaging/test_models.py | 10 +------ tests/unit/packaging/test_views.py | 4 +-- 3 files changed, 25 insertions(+), 33 deletions(-) diff --git a/tests/unit/legacy/api/test_json.py b/tests/unit/legacy/api/test_json.py index 1878fbe8cdf2..23973fa4bb73 100644 --- a/tests/unit/legacy/api/test_json.py +++ b/tests/unit/legacy/api/test_json.py @@ -118,14 +118,14 @@ def test_all_non_prereleases_yanked(self, monkeypatch, db_request): db_request.matchdict = {"name": project.normalized_name} assert json.latest_release_factory(db_request) == release - def test_with_unpublished(self, db_request): + def test_with_staged(self, db_request): project = ProjectFactory.create() release = ReleaseFactory.create(project=project, version="1.0") ReleaseFactory.create(project=project, version="2.0", published=False) db_request.matchdict = {"name": project.normalized_name} assert json.latest_release_factory(db_request) == release - def test_only_unpublished(self, db_request): + def test_only_staged(self, db_request): project = ProjectFactory.create() ReleaseFactory.create(project=project, version="1.0", published=False) db_request.matchdict = {"name": project.normalized_name} @@ -138,31 +138,31 @@ def test_only_unpublished(self, db_request): ("release0_state", "release1_state", "release2_state", "latest_release"), [ ("published", "published", "published", 2), - ("published", "published", "unpublished", 1), + ("published", "published", "staged", 1), ("published", "published", "yanked", 1), - ("published", "unpublished", "published", 2), - ("published", "unpublished", "unpublished", 0), - ("published", "unpublished", "yanked", 0), + ("published", "staged", "published", 2), + ("published", "staged", "staged", 0), + ("published", "staged", "yanked", 0), ("published", "yanked", "published", 2), - ("published", "yanked", "unpublished", 0), + ("published", "yanked", "staged", 0), ("published", "yanked", "yanked", 0), - ("unpublished", "published", "published", 2), - ("unpublished", "published", "unpublished", 1), - ("unpublished", "published", "yanked", 1), - ("unpublished", "unpublished", "published", 2), - ("unpublished", "unpublished", "unpublished", -1), - ("unpublished", "unpublished", "yanked", 2), # Same endpoint as none yanked - ("unpublished", "yanked", "published", 2), - ("unpublished", "yanked", "unpublished", 1), - ("unpublished", "yanked", "yanked", 2), + ("staged", "published", "published", 2), + ("staged", "published", "staged", 1), + ("staged", "published", "yanked", 1), + ("staged", "staged", "published", 2), + ("staged", "staged", "staged", -1), + ("staged", "staged", "yanked", 2), # Same endpoint as none yanked + ("staged", "yanked", "published", 2), + ("staged", "yanked", "staged", 1), + ("staged", "yanked", "yanked", 2), ("yanked", "published", "published", 2), - ("yanked", "published", "unpublished", 1), + ("yanked", "published", "staged", 1), ("yanked", "published", "yanked", 1), - ("yanked", "unpublished", "published", 2), - ("yanked", "unpublished", "unpublished", 0), - ("yanked", "unpublished", "yanked", 2), + ("yanked", "staged", "published", 2), + ("yanked", "staged", "staged", 0), + ("yanked", "staged", "yanked", 2), ("yanked", "yanked", "published", 2), - ("yanked", "yanked", "unpublished", 1), + ("yanked", "yanked", "staged", 1), ("yanked", "yanked", "yanked", 2), ], ) @@ -183,7 +183,7 @@ def test_with_mixed_states( project=project, version=version, published=True ) ) - elif state == "unpublished": + elif state == "staged": releases.append( ReleaseFactory.create( project=project, version=version, published=False diff --git a/tests/unit/packaging/test_models.py b/tests/unit/packaging/test_models.py index 6904a0c38a91..46f5138b9a99 100644 --- a/tests/unit/packaging/test_models.py +++ b/tests/unit/packaging/test_models.py @@ -1216,14 +1216,6 @@ def test_description_relationship(self, db_request): assert release in db_request.db.deleted assert description in db_request.db.deleted - def test_published(self, db_request): - release = DBReleaseFactory.create() - assert release.published - - def test_unpublished(self, db_request): - release = DBReleaseFactory.create(published=False) - assert not release.published - @pytest.mark.parametrize( "published", @@ -1237,7 +1229,7 @@ def test_filter_staged_releases(db_request, published): assert db_request.db.query(Release).count() == (1 if published else 0) -def test_filter_staged_releases_unpublished(db_request): +def test_filter_staged_releases_with_staged(db_request): DBReleaseFactory.create(published=False) assert ( db_request.db.query(Release).execution_options(include_staged=True).count() == 1 diff --git a/tests/unit/packaging/test_views.py b/tests/unit/packaging/test_views.py index f2139dc97a36..8eba115cf851 100644 --- a/tests/unit/packaging/test_views.py +++ b/tests/unit/packaging/test_views.py @@ -135,7 +135,7 @@ def test_only_yanked_release(self, monkeypatch, db_request): assert resp is response assert release_detail.calls == [pretend.call(release, db_request)] - def test_with_unpublished(self, monkeypatch, db_request): + def test_with_staged(self, monkeypatch, db_request): project = ProjectFactory.create() release = ReleaseFactory.create(project=project, version="1.0") ReleaseFactory.create(project=project, version="1.1", published=False) @@ -216,7 +216,7 @@ def test_detail_rendered(self, db_request): ) ] - # Add an unpublished version + # Add a staged version staged_release = ReleaseFactory.create( project=project, version="5.1",