diff --git a/.gitignore b/.gitignore index 25ff9bd7..c2b206d1 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,4 @@ dmypy.json [._]s[a-rt-v][a-z] [._]ss[a-gi-z] [._]sw[a-p] +*.db diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..dd504c34 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,20 @@ +# Sample Seeding Script for Daily Billable Usage + +To run the script to populate the test DB use: + +``` +python manage.py shell < /mnt/MOC/coldfront-plugin-cloud/scripts/seed_sample_daily_billable_usage_shell.py +``` + +Configure the `seed_sample_daily_billable_usage_shell.py` file before running the script. + +``` +# --- configure before running --- +ALLOCATION_ID = 4 + +# Pick one date mode (set the others to None / False): +SINGLE_DATE = None # YYYY-MM-DD, or None +MONTH = None # YYYY-MM, or None +CURRENT_MONTH = True # seed every day in the current calendar month +THROUGH_TODAY = True # with CURRENT_MONTH, skip future days +``` diff --git a/scripts/seed_sample_daily_billable_usage_shell.py b/scripts/seed_sample_daily_billable_usage_shell.py new file mode 100644 index 00000000..3e122d0c --- /dev/null +++ b/scripts/seed_sample_daily_billable_usage_shell.py @@ -0,0 +1,97 @@ +"""Seed AllocationDailyBillableUsage rows for local UI development. + +Run from a ColdFront checkout with NERC local_settings: + + python manage.py shell < path/to/seed_daily_billable_usage_shell.py + +Or from django shell: + + exec(open("path/to/seed_daily_billable_usage_shell.py").read()) +""" + +import calendar +from datetime import date as date_type +from decimal import Decimal + +from coldfront.core.allocation.models import Allocation +from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, +) + +# --- configure before running --- +ALLOCATION_ID = 4 + +# Pick one date mode (set the others to None / False): +SINGLE_DATE = None # YYYY-MM-DD, or None +MONTH = None # YYYY-MM, or None +CURRENT_MONTH = True # seed every day in the current calendar month +THROUGH_TODAY = True # with CURRENT_MONTH, skip future days + +RAMP = True # increase each SU type slightly per day (chart-friendly) + +DEFAULT_USAGE = { + "OpenStack CPU": Decimal("100.00"), + "OpenStack V100 GPU": Decimal("50.00"), + "Storage": Decimal("30.12"), +} + +_RAMP_STEP = { + "OpenStack CPU": Decimal("5"), + "OpenStack V100 GPU": Decimal("2"), + "Storage": Decimal("0.5"), +} + + +def dates_in_month(month: str) -> list[str]: + year, mon = map(int, month.split("-", 1)) + _, last_day = calendar.monthrange(year, mon) + return [f"{year}-{mon:02d}-{day:02d}" for day in range(1, last_day + 1)] + + +def usage_for_day( + base: dict[str, Decimal], day_index: int, ramp: bool +) -> dict[str, Decimal]: + if not ramp: + return dict(base) + return { + su_type: amount + _RAMP_STEP.get(su_type, Decimal("0")) * day_index + for su_type, amount in base.items() + } + + +def resolve_dates() -> list[str]: + if SINGLE_DATE: + return [SINGLE_DATE] + if CURRENT_MONTH: + today = date_type.today() + month_str = f"{today.year}-{today.month:02d}" + dates = dates_in_month(month_str) + if THROUGH_TODAY: + today_str = today.isoformat() + dates = [d for d in dates if d <= today_str] + return dates + if MONTH: + return dates_in_month(MONTH) + raise ValueError("Set SINGLE_DATE, MONTH, or CURRENT_MONTH=True") + + +allocation = Allocation.objects.get(pk=ALLOCATION_ID) +dates = resolve_dates() +total_rows = 0 + +for day_index, day in enumerate(dates): + for su_type, value in usage_for_day(DEFAULT_USAGE, day_index, RAMP).items(): + AllocationDailyBillableUsage.objects.update_or_create( + allocation=allocation, + date=day, + su_type=su_type, + defaults={"value": value}, + ) + total_rows += 1 + +print( + f"Seeded {len(DEFAULT_USAGE)} SU type(s) × {len(dates)} day(s) " + f"= {total_rows} row(s) for allocation {allocation.id}" +) +if len(dates) > 1: + print(f" dates: {dates[0]} … {dates[-1]}") diff --git a/setup.cfg b/setup.cfg index 26c21a95..140c07d1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ install_requires = kubernetes>=36.0.0 openshift coldfront >= 1.1.0 + django-model-utils python-cinderclient python-keystoneclient python-novaclient diff --git a/src/coldfront_plugin_cloud/billable_usage.py b/src/coldfront_plugin_cloud/billable_usage.py new file mode 100644 index 00000000..882c627b --- /dev/null +++ b/src/coldfront_plugin_cloud/billable_usage.py @@ -0,0 +1,95 @@ +from collections.abc import Iterable + +from coldfront.core.allocation.models import Allocation +from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, +) +from coldfront_plugin_cloud.models.usage_models import UsageInfo, validate_date_str + + +def _rows_to_usage_info(rows: Iterable[AllocationDailyBillableUsage]) -> UsageInfo: + """Build a UsageInfo from ORM rows (one row per SU type). + + Args: + rows: AllocationDailyBillableUsage instances for a single allocation + and date (or any set of rows to collapse into one dict). + + Returns: + UsageInfo mapping SU type names to charge values. Empty when rows + is empty. + + Raises: + ValueError: If any row has an empty su_type. + + Example: + >>> rows = [ + ... AllocationDailyBillableUsage(su_type="OpenStack CPU", value=100), + ... AllocationDailyBillableUsage(su_type="Storage", value=30.12), + ... ] + >>> info = _rows_to_usage_info(rows) + >>> info.root["OpenStack CPU"] + Decimal('100') + >>> info.total_charges + Decimal('130.12') + """ + usage_info = UsageInfo({}) + for row in rows: + if not row.su_type: + raise ValueError(f"usage row id={row.pk} has empty su_type") + usage_info.root[row.su_type] = row.value + return usage_info + + +def get_daily_billable_usage_by_date( + allocation: Allocation, start_date: str, end_date: str +) -> dict[str, UsageInfo]: + """Load billable usage grouped by day across an inclusive date range. + + Args: + allocation: ColdFront allocation to read. + start_date: First day (inclusive), ``YYYY-MM-DD``. + end_date: Last day (inclusive), ``YYYY-MM-DD``. + + Returns: + Mapping of ``YYYY-MM-DD`` date strings to UsageInfo. Dates with no + usage rows are omitted. + + Raises: + ValueError: If allocation is unsaved, a date is invalid, or + start_date is after end_date. + + Example: + >>> usage_by_date = get_daily_billable_usage_by_date( + ... allocation, "2025-11-01", "2025-11-30" + ... ) + >>> usage_by_date["2025-11-15"].root + {'OpenStack CPU': Decimal('100.00'), 'Storage': Decimal('30.12')} + """ + if allocation.pk is None: + raise ValueError("allocation must be saved (have a primary key)") + if not isinstance(start_date, str) or not start_date.strip(): + raise ValueError("start_date must be a non-empty YYYY-MM-DD string") + if not isinstance(end_date, str) or not end_date.strip(): + raise ValueError("end_date must be a non-empty YYYY-MM-DD string") + start_date = validate_date_str(start_date) + end_date = validate_date_str(end_date) + if start_date > end_date: + raise ValueError( + f"start_date {start_date} must be on or before end_date {end_date}" + ) + + rows = AllocationDailyBillableUsage.objects.filter( + allocation=allocation, + date__gte=start_date, + date__lte=end_date, + ).order_by("date", "su_type") + + usage_by_date: dict[str, UsageInfo] = {} + for row in rows: + day = row.date.isoformat() if hasattr(row.date, "isoformat") else str(row.date) + usage_by_date.setdefault(day, UsageInfo({})) + if not row.su_type: + raise ValueError(f"usage row id={row.pk} has empty su_type") + usage_by_date[day].root[row.su_type] = row.value + + return usage_by_date diff --git a/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py b/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py index eac3e19a..b7d48b77 100644 --- a/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py +++ b/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py @@ -12,6 +12,9 @@ from coldfront_plugin_cloud.models import usage_models from coldfront_plugin_cloud.models.usage_models import UsageInfo, validate_date_str from coldfront_plugin_cloud import utils +from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, +) import boto3 from django.core.management.base import BaseCommand @@ -85,10 +88,20 @@ def add_arguments(self, parser): parser.add_argument( "--date", type=str, default=self.previous_day_string, help="Date." ) + parser.add_argument( + "--remove", + action="store_true", + help="Remove usage entries for the specified date instead of fetching.", + ) def handle(self, *args, **options): date = options["date"] validate_date_str(date) + remove = options.get("remove", False) + + if remove: + self.handle_remove(date) + return allocations = self.get_allocations_for_daily_billing() @@ -122,6 +135,8 @@ def handle(self, *args, **options): ) continue + self.store_usage_in_database(allocation, date, new_usage) + # Only update the latest value if the processed date is newer or same date. if not previous_total or date >= previous_total.date: new_total = TotalByDate(date, new_usage.total_charges) @@ -302,3 +317,32 @@ def send_alert_email(cls, allocation: Allocation, resource: Resource, alert_valu if x != allocation.project.pi.email ], ) + + @staticmethod + def store_usage_in_database(allocation: Allocation, date: str, usage_info): + """Store usage information in the database for each SU type. + + Args: + allocation: The allocation to store usage for + date: The date string in YYYY-MM-DD format + usage_info: UsageInfo pydantic model instance with SU type charges + """ + for su_type, value in usage_info.root.items(): + AllocationDailyBillableUsage.objects.update_or_create( + allocation=allocation, + date=date, + su_type=su_type, + defaults={"value": value}, + ) + + @staticmethod + def handle_remove(date: str): + """Remove all usage entries for the specified date. + + Args: + date: The date string in YYYY-MM-DD format for which to remove entries + """ + deleted_count, _ = AllocationDailyBillableUsage.objects.filter( + date=date + ).delete() + logger.info(f"Removed {deleted_count} usage entries for date {date}") diff --git a/src/coldfront_plugin_cloud/migrations/0001_initial.py b/src/coldfront_plugin_cloud/migrations/0001_initial.py new file mode 100644 index 00000000..abdd5d21 --- /dev/null +++ b/src/coldfront_plugin_cloud/migrations/0001_initial.py @@ -0,0 +1,95 @@ +# Generated manually for coldfront_plugin_cloud + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("allocation", "__first__"), + ] + + operations = [ + migrations.CreateModel( + name="AllocationDailyBillableUsage", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "date", + models.DateField( + help_text="The date for which this usage was recorded" + ), + ), + ( + "su_type", + models.CharField( + help_text="The type of Service Unit (e.g., OpenStack CPU, OpenStack V100 GPU)", + max_length=255, + ), + ), + ( + "value", + models.DecimalField( + decimal_places=2, + help_text="The usage value/cost for this SU type on this date", + max_digits=12, + ), + ), + ( + "allocation", + models.ForeignKey( + help_text="The allocation this usage belongs to", + on_delete=django.db.models.deletion.CASCADE, + related_name="daily_usage_records", + to="allocation.allocation", + ), + ), + ], + options={ + "db_table": "coldfront_plugin_cloud_allocationdailybillableusage", + "ordering": ["-date", "allocation", "su_type"], + }, + ), + migrations.AddIndex( + model_name="allocationdailybillableusage", + index=models.Index( + fields=["allocation", "date"], name="coldfront_p_allocat_5c8e3d_idx" + ), + ), + migrations.AddIndex( + model_name="allocationdailybillableusage", + index=models.Index(fields=["date"], name="coldfront_p_date_3e8a9e_idx"), + ), + migrations.AlterUniqueTogether( + name="allocationdailybillableusage", + unique_together={("allocation", "date", "su_type")}, + ), + ] diff --git a/src/coldfront_plugin_cloud/migrations/__init__.py b/src/coldfront_plugin_cloud/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/coldfront_plugin_cloud/models/__init__.py b/src/coldfront_plugin_cloud/models/__init__.py index e69de29b..088ee2e0 100644 --- a/src/coldfront_plugin_cloud/models/__init__.py +++ b/src/coldfront_plugin_cloud/models/__init__.py @@ -0,0 +1 @@ +import coldfront_plugin_cloud.models.daily_billable_usage # noqa: F401 diff --git a/src/coldfront_plugin_cloud/models/daily_billable_usage.py b/src/coldfront_plugin_cloud/models/daily_billable_usage.py new file mode 100644 index 00000000..b71c7b47 --- /dev/null +++ b/src/coldfront_plugin_cloud/models/daily_billable_usage.py @@ -0,0 +1,36 @@ +from django.db import models +from model_utils.models import TimeStampedModel +from coldfront.core.allocation.models import Allocation + + +class AllocationDailyBillableUsage(TimeStampedModel): + """Stores daily billable usage for allocations by SU type.""" + + allocation = models.ForeignKey( + Allocation, + on_delete=models.CASCADE, + related_name="daily_usage_records", + help_text="The allocation this usage belongs to", + ) + date = models.DateField(help_text="The date for which this usage was recorded") + su_type = models.CharField( + max_length=255, + help_text="The type of Service Unit (e.g., OpenStack CPU, OpenStack V100 GPU)", + ) + value = models.DecimalField( + max_digits=12, + decimal_places=2, + help_text="The usage value/cost for this SU type on this date", + ) + + class Meta: + db_table = "coldfront_plugin_cloud_allocationdailybillableusage" + unique_together = [["allocation", "date", "su_type"]] + indexes = [ + models.Index(fields=["allocation", "date"]), + models.Index(fields=["date"]), + ] + ordering = ["-date", "allocation", "su_type"] + + def __str__(self): + return f"{self.allocation.id} - {self.date} - {self.su_type}: {self.value}" diff --git a/src/coldfront_plugin_cloud/tests/unit/test_daily_billable_usage.py b/src/coldfront_plugin_cloud/tests/unit/test_daily_billable_usage.py new file mode 100644 index 00000000..c1527630 --- /dev/null +++ b/src/coldfront_plugin_cloud/tests/unit/test_daily_billable_usage.py @@ -0,0 +1,318 @@ +from datetime import timedelta +from decimal import Decimal +from unittest import mock + +from django.core.exceptions import ValidationError +from django.db import IntegrityError +from django.utils import timezone + +from coldfront.core.allocation.models import Allocation +from coldfront_plugin_cloud.billable_usage import ( + _rows_to_usage_info, + get_daily_billable_usage_by_date, +) +from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, +) +from coldfront_plugin_cloud.models.usage_models import UsageInfo +from coldfront_plugin_cloud.tests import base + + +class TestRowsToUsageInfo(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def test_empty_iterable(self): + usage = _rows_to_usage_info([]) + self.assertEqual(usage, UsageInfo({})) + + def test_multiple_rows(self): + rows = [ + AllocationDailyBillableUsage(su_type="OpenStack CPU", value=Decimal("100")), + AllocationDailyBillableUsage(su_type="Storage", value=Decimal("30.12")), + ] + expected = UsageInfo( + {"OpenStack CPU": Decimal("100"), "Storage": Decimal("30.12")} + ) + self.assertEqual(_rows_to_usage_info(rows), expected) + + def test_rows_is_none(self): + with self.assertRaises(TypeError): + _rows_to_usage_info(None) + + def test_non_model_row(self): + with self.assertRaises(AttributeError): + _rows_to_usage_info(["not-a-row"]) + + def test_empty_su_type(self): + # Enforce that every row has a usable key; empty SU type would corrupt the usage dict. + allocation = self._new_allocation() + row = AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date="2025-11-15", + su_type="", + value=Decimal("1.00"), + ) + with self.assertRaises(ValueError) as ctx: + _rows_to_usage_info([row]) + self.assertIn(f"id={row.pk}", str(ctx.exception)) + + +class TestGetDailyBillableUsage(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def _create_usage_row(self, allocation, date, su_type, value): + return AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date=date, + su_type=su_type, + value=Decimal(value), + ) + + def _get_usage_for_date(self, allocation, date): + return get_daily_billable_usage_by_date(allocation, date, date).get( + date, UsageInfo({}) + ) + + def test_happy_path(self): + allocation = self._new_allocation() + date = "2025-11-15" + self._create_usage_row(allocation, date, "OpenStack CPU", "100.00") + self._create_usage_row(allocation, date, "OpenStack V100 GPU", "50.00") + self._create_usage_row(allocation, date, "Storage", "30.12") + + expected = UsageInfo( + { + "OpenStack CPU": Decimal("100.00"), + "OpenStack V100 GPU": Decimal("50.00"), + "Storage": Decimal("30.12"), + } + ) + self.assertEqual(self._get_usage_for_date(allocation, date), expected) + + def test_no_rows_returns_empty_usage_info(self): + allocation = self._new_allocation() + date = "2025-11-15" + self.assertEqual(self._get_usage_for_date(allocation, date), UsageInfo({})) + + def test_wrong_allocation_type(self): + date = "2025-11-15" + with self.assertRaises(AttributeError): + get_daily_billable_usage_by_date("not-an-allocation", date, date) + + def test_unsaved_allocation(self): + allocation = Allocation() + date = "2025-11-15" + with self.assertRaises(ValueError) as ctx: + get_daily_billable_usage_by_date(allocation, date, date) + self.assertIn("primary key", str(ctx.exception)) + + def test_empty_date(self): + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_by_date(allocation, "", "") + + def test_whitespace_date(self): + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_by_date(allocation, " ", " ") + + def test_invalid_date(self): + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_by_date(allocation, "2025-13-01", "2025-13-01") + + def test_excludes_other_allocation_and_date(self): + allocation = self._new_allocation() + other_allocation = self._new_allocation() + date = "2025-11-15" + self._create_usage_row(allocation, date, "OpenStack CPU", "100.00") + self._create_usage_row(allocation, "2025-11-16", "OpenStack CPU", "200.00") + self._create_usage_row(other_allocation, date, "OpenStack CPU", "999.00") + + expected = UsageInfo({"OpenStack CPU": Decimal("100.00")}) + self.assertEqual(self._get_usage_for_date(allocation, date), expected) + + +class TestGetDailyBillableUsageByDate(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def _create_usage_row(self, allocation, date, su_type, value): + return AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date=date, + su_type=su_type, + value=Decimal(value), + ) + + def test_groups_usage_by_date(self): + allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-01", "OpenStack CPU", "10.00") + self._create_usage_row(allocation, "2025-11-01", "Storage", "5.50") + self._create_usage_row(allocation, "2025-11-02", "OpenStack CPU", "22.50") + self._create_usage_row(allocation, "2025-10-31", "OpenStack CPU", "99.00") + + expected = { + "2025-11-01": UsageInfo( + {"OpenStack CPU": Decimal("10.00"), "Storage": Decimal("5.50")} + ), + "2025-11-02": UsageInfo({"OpenStack CPU": Decimal("22.50")}), + } + self.assertEqual( + get_daily_billable_usage_by_date(allocation, "2025-11-01", "2025-11-30"), + expected, + ) + + def test_empty_when_no_matching_rows(self): + allocation = self._new_allocation() + usage_by_date = get_daily_billable_usage_by_date( + allocation, "2025-11-01", "2025-11-30" + ) + self.assertEqual(usage_by_date, {}) + + def test_start_date_after_end_date(self): + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_by_date(allocation, "2025-11-30", "2025-11-01") + + def test_unsaved_allocation(self): + allocation = Allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_by_date(allocation, "2025-11-01", "2025-11-30") + + +class TestAllocationDailyBillableUsageModel(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def _create_usage_row(self, allocation, date, su_type, value): + return AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date=date, + su_type=su_type, + value=Decimal(value), + ) + + def test_str(self): + # A stable __str__ helps debugging/logging/admin views when inspecting usage rows. + allocation = self._new_allocation() + row = self._create_usage_row( + allocation, "2025-11-15", "OpenStack CPU", "100.00" + ) + self.assertEqual( + str(row), + f"{allocation.id} - 2025-11-15 - OpenStack CPU: 100.00", + ) + + def test_unique_together_raises_on_duplicate(self): + # Enforce one row per (allocation, date, su_type) so upserts don't create duplicates. + allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "100.00") + with self.assertRaises(IntegrityError): + AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date="2025-11-15", + su_type="OpenStack CPU", + value=Decimal("200.00"), + ) + + def test_allocation_delete_cascades_to_usage_rows(self): + # Foreign key cascade prevents orphaned usage rows when an allocation is deleted. + allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "100.00") + self._create_usage_row(allocation, "2025-11-16", "Storage", "30.00") + self.assertEqual(AllocationDailyBillableUsage.objects.count(), 2) + + allocation.delete() + + self.assertEqual(AllocationDailyBillableUsage.objects.count(), 0) + + def test_meta_ordering(self): + # Ordering affects deterministic reads/merges; this locks the contract to the model definition. + self.assertEqual( + AllocationDailyBillableUsage._meta.ordering, + ["-date", "allocation", "su_type"], + ) + allocation_a = self._new_allocation() + allocation_b = self._new_allocation() + self._create_usage_row(allocation_a, "2025-11-01", "Storage", "10.00") + self._create_usage_row(allocation_a, "2025-11-15", "OpenStack CPU", "20.00") + self._create_usage_row(allocation_b, "2025-11-15", "OpenStack CPU", "30.00") + + rows = list(AllocationDailyBillableUsage.objects.all()) + self.assertEqual(len(rows), 3) + self.assertEqual(rows[0].date.isoformat(), "2025-11-15") + self.assertEqual(rows[1].date.isoformat(), "2025-11-15") + self.assertEqual(rows[2].date.isoformat(), "2025-11-01") + # Same date: ordering uses Allocation model default ordering via FK. + nov_15_allocation_ids = {rows[0].allocation_id, rows[1].allocation_id} + self.assertEqual(nov_15_allocation_ids, {allocation_a.id, allocation_b.id}) + + def test_value_stores_two_decimal_places(self): + # Value is money-like and must round-trip without float precision loss. + allocation = self._new_allocation() + row = self._create_usage_row(allocation, "2025-11-15", "Storage", "30.12") + row.refresh_from_db() + self.assertEqual(row.value, Decimal("30.12")) + + def test_value_rejects_overflow_beyond_max_digits(self): + # Schema protection: reject invoice values that exceed the declared DecimalField size. + allocation = self._new_allocation() + row = AllocationDailyBillableUsage( + allocation=allocation, + date="2025-11-15", + su_type="OpenStack CPU", + value=Decimal("10000000000.00"), + ) + with self.assertRaises(ValidationError): + row.full_clean() + + def test_timestamps_set_on_create_and_modified_on_update(self): + # TimeStampedModel should set created/modified and advance modified on updates. + allocation = self._new_allocation() + row = self._create_usage_row( + allocation, "2025-11-15", "OpenStack CPU", "100.00" + ) + created_at = row.created + modified_at = row.modified + self.assertIsNotNone(created_at) + self.assertIsNotNone(modified_at) + + later = timezone.now() + timedelta(seconds=1) + with mock.patch("django.utils.timezone.now", return_value=later): + row.value = Decimal("150.00") + row.save() + row.refresh_from_db() + + self.assertEqual(row.created, created_at) + self.assertGreater(row.modified, modified_at) + + def test_related_name_daily_usage_records(self): + # Reverse relation is the primary way callers will traverse allocation -> daily usage rows. + allocation = self._new_allocation() + other_allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "100.00") + self._create_usage_row(allocation, "2025-11-16", "Storage", "30.00") + self._create_usage_row( + other_allocation, "2025-11-15", "OpenStack CPU", "999.00" + ) + + self.assertEqual(allocation.daily_usage_records.count(), 2) + self.assertEqual( + allocation.daily_usage_records.filter(date="2025-11-15").count(), + 1, + ) + self.assertEqual( + allocation.daily_usage_records.get(date="2025-11-15").value, + Decimal("100.00"), + ) diff --git a/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py b/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py index 477801ff..04c73f8d 100644 --- a/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py +++ b/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py @@ -1,4 +1,5 @@ import io +from decimal import Decimal from unittest import mock from unittest.mock import Mock, patch @@ -240,3 +241,197 @@ def test_send_alert_email(self): receiver_list=[allocation_1.project.pi.email], cc=[manager.email], ) + + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.RESOURCES_DAILY_ENABLED", + ["FakeProd"], + ) + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.Command.get_allocation_usage" + ) + def test_database_insertion_and_removal(self, mock_get_allocation_usage): + """Test database insertion, updates, and removal of usage entries.""" + from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, + ) + + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo( + {"OpenStack CPU": "100.00", "OpenStack GPU": "50.00"} + ), + usage_models.UsageInfo({"Storage": "30.12"}), + ] + + fakeprod = self.new_openstack_resource( + name="FakeProd", internal_name="FakeProd" + ) + prod_project = self.new_project() + allocation_1 = self.new_allocation( + project=prod_project, resource=fakeprod, quantity=1, status="Active" + ) + utils.set_attribute_on_allocation( + allocation_1, attributes.ALLOCATION_PROJECT_ID, "test-allocation-1" + ) + + # Verify no entries before running command + self.assertEqual(AllocationDailyBillableUsage.objects.count(), 0) + + # Test initial insertion + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Verify database entries were created + usage_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + self.assertEqual(usage_entries.count(), 3) + + # Check individual SU types + cpu_usage = usage_entries.get(su_type="OpenStack CPU") + self.assertEqual(cpu_usage.value, Decimal("100.00")) + + gpu_usage = usage_entries.get(su_type="OpenStack GPU") + self.assertEqual(gpu_usage.value, Decimal("50.00")) + + storage_usage = usage_entries.get(su_type="Storage") + self.assertEqual(storage_usage.value, Decimal("30.12")) + + # Test update_or_create by running again with different values for same date + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo( + {"OpenStack CPU": "110.00", "OpenStack GPU": "55.00"} + ), + usage_models.UsageInfo({"Storage": "35.00"}), + ] + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Should still have 3 entries (not duplicates) + usage_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + self.assertEqual(usage_entries.count(), 3) + + # Check updated values + cpu_usage = usage_entries.get(su_type="OpenStack CPU") + self.assertEqual(cpu_usage.value, Decimal("110.00")) + + # Add data for another date to test selective removal + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo({"OpenStack CPU": "120.00"}), + usage_models.UsageInfo({"Storage": "40.00"}), + ] + call_command("fetch_daily_billable_usage", date="2025-11-16") + + # Verify data exists for both dates + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-15").count(), 3 + ) + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-16").count(), 2 + ) + + # Test removal - remove data for 2025-11-15 + call_command("fetch_daily_billable_usage", date="2025-11-15", remove=True) + + # Verify data for 2025-11-15 is deleted + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-15").count(), 0 + ) + + # Verify data for 2025-11-16 still exists + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-16").count(), 2 + ) + + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.RESOURCES_DAILY_ENABLED", + ["FakeProd"], + ) + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.Command.get_allocation_usage" + ) + def test_multiple_allocations_same_date(self, mock_get_allocation_usage): + """Test that multiple allocations can store usage for the same date.""" + from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, + ) + + fakeprod = self.new_openstack_resource( + name="FakeProd", internal_name="FakeProd" + ) + prod_project1 = self.new_project() + prod_project2 = self.new_project() + + allocation_1 = self.new_allocation( + project=prod_project1, resource=fakeprod, quantity=1, status="Active" + ) + allocation_2 = self.new_allocation( + project=prod_project2, resource=fakeprod, quantity=1, status="Active" + ) + + utils.set_attribute_on_allocation( + allocation_1, attributes.ALLOCATION_PROJECT_ID, "test-allocation-1" + ) + utils.set_attribute_on_allocation( + allocation_2, attributes.ALLOCATION_PROJECT_ID, "test-allocation-2" + ) + + # Mock returns for both allocations + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo({"OpenStack CPU": "100.00"}), + usage_models.UsageInfo({"Storage": "30.00"}), + usage_models.UsageInfo({"OpenStack CPU": "200.00"}), + usage_models.UsageInfo({"Storage": "50.00"}), + ] + + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Verify both allocations have data + alloc1_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + alloc2_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_2, date="2025-11-15" + ) + + self.assertEqual(alloc1_entries.count(), 2) + self.assertEqual(alloc2_entries.count(), 2) + + # Verify values are correct for each allocation + self.assertEqual( + alloc1_entries.get(su_type="OpenStack CPU").value, Decimal("100.00") + ) + self.assertEqual( + alloc2_entries.get(su_type="OpenStack CPU").value, Decimal("200.00") + ) + + # Test updating with same date but different values - should update, not error + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo({"OpenStack CPU": "150.00"}), + usage_models.UsageInfo({"Storage": "35.00"}), + usage_models.UsageInfo({"OpenStack CPU": "250.00"}), + usage_models.UsageInfo({"Storage": "55.00"}), + ] + + # This should not raise any errors and should update existing values + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Verify counts remain the same (no duplicates) + alloc1_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + alloc2_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_2, date="2025-11-15" + ) + + self.assertEqual(alloc1_entries.count(), 2) + self.assertEqual(alloc2_entries.count(), 2) + + # Verify values were updated + self.assertEqual( + alloc1_entries.get(su_type="OpenStack CPU").value, Decimal("150.00") + ) + self.assertEqual(alloc1_entries.get(su_type="Storage").value, Decimal("35.00")) + self.assertEqual( + alloc2_entries.get(su_type="OpenStack CPU").value, Decimal("250.00") + ) + self.assertEqual(alloc2_entries.get(su_type="Storage").value, Decimal("55.00"))