Merge pull request #1582 from GSA/main

03/05/2025 Production Deploy

Author: ccostino

.ds.baseline

@@ -209,7 +209,7 @@
         "filename": "tests/app/aws/test_s3.py",
         "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747",
         "is_verified": false,
-        "line_number": 40,
+        "line_number": 42,
         "is_secret": false
       }
     ],
@@ -384,5 +384,5 @@
       }
     ]
   },
-  "generated_at": "2025-02-10T16:57:15Z"
+  "generated_at": "2025-02-27T21:09:45Z"
 }

Makefile

@@ -148,3 +148,8 @@ clean:
 # 	cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api
 # 	cf stop notify-api-failwhale
 # 	@echo "Failwhale is disabled"
+
+.PHONY: test-single
+test-single: export NEW_RELIC_ENVIRONMENT=test
+test-single: ## Run a single test file
+	poetry run pytest -s $(TEST_FILE)

app/clients/cloudwatch/aws_cloudwatch.py

@@ -107,13 +107,20 @@ def event_to_db_format(self, event):
         provider_response = self._aws_value_or_default(
             event, "delivery", "providerResponse"
         )
+        message_cost = self._aws_value_or_default(event, "delivery", "priceInUSD")
+        if message_cost is None or message_cost == "":
+            message_cost = 0.0
+        else:
+            message_cost = float(message_cost)
+
         my_timestamp = self._aws_value_or_default(event, "notification", "timestamp")
         return {
             "notification.messageId": event["notification"]["messageId"],
             "status": event["status"],
             "delivery.phoneCarrier": phone_carrier,
             "delivery.providerResponse": provider_response,
             "@timestamp": my_timestamp,
+            "delivery.priceInUSD": message_cost,
         }
 
     # Here is an example of how to get the events with log insights

app/dao/date_util.py

@@ -93,3 +93,25 @@ def generate_date_range(start_date, end_date=None, days=0):
             current_date += timedelta(days=1)
     else:
         return "An end_date or number of days must be specified"
+
+
+def generate_hourly_range(start_date, end_date=None, hours=0):
+    if end_date:
+        current_time = start_date
+        while current_time <= end_date:
+            try:
+                yield current_time
+            except ValueError:
+                pass
+            current_time += timedelta(hours=1)
+    elif hours > 0:
+        end_time = start_date + timedelta(hours=hours)
+        current_time = start_date
+        while current_time < end_time:
+            try:
+                yield current_time
+            except ValueError:
+                pass
+            current_time += timedelta(hours=1)
+    else:
+        return "An end_date or number of hours must be specified"

app/dao/notifications_dao.py

@@ -267,6 +267,29 @@ def get_notifications_for_job(
     return pagination
 
 
+def get_recent_notifications_for_job(
+    service_id, job_id, filter_dict=None, page=1, page_size=None
+):
+    if page_size is None:
+        page_size = current_app.config["PAGE_SIZE"]
+
+    stmt = select(Notification).where(
+        Notification.service_id == service_id,
+        Notification.job_id == job_id,
+    )
+
+    stmt = _filter_query(stmt, filter_dict)
+    stmt = stmt.order_by(desc(Notification.job_row_number))
+    results = db.session.execute(stmt).scalars().all()
+
+    page_size = current_app.config["PAGE_SIZE"]
+    offset = (page - 1) * page_size
+    paginated_results = results[offset : offset + page_size]
+
+    pagination = Pagination(paginated_results, page, page_size, len(results))
+    return pagination
+
+
 def dao_get_notification_count_for_job_id(*, job_id):
     stmt = select(func.count(Notification.id)).where(Notification.job_id == job_id)
     return db.session.execute(stmt).scalar()
@@ -279,6 +302,44 @@ def dao_get_notification_count_for_service(*, service_id):
     return db.session.execute(stmt).scalar()
 
 
+def dao_get_notification_count_for_service_message_ratio(service_id, current_year):
+    start_date = datetime(current_year, 1, 1)
+    end_date = datetime(current_year + 1, 1, 1)
+    stmt1 = (
+        select(func.count())
+        .select_from(Notification)
+        .where(
+            Notification.service_id == service_id,
+            Notification.status
+            not in [
+                NotificationStatus.CANCELLED,
+                NotificationStatus.CREATED,
+                NotificationStatus.SENDING,
+            ],
+            Notification.created_at >= start_date,
+            Notification.created_at < end_date,
+        )
+    )
+    stmt2 = (
+        select(func.count())
+        .select_from(NotificationHistory)
+        .where(
+            NotificationHistory.service_id == service_id,
+            NotificationHistory.status
+            not in [
+                NotificationStatus.CANCELLED,
+                NotificationStatus.CREATED,
+                NotificationStatus.SENDING,
+            ],
+            NotificationHistory.created_at >= start_date,
+            NotificationHistory.created_at < end_date,
+        )
+    )
+    recent_count = db.session.execute(stmt1).scalar_one()
+    old_count = db.session.execute(stmt2).scalar_one()
+    return recent_count + old_count
+
+
 def dao_get_failed_notification_count():
     stmt = select(func.count(Notification.id)).where(
         Notification.status == NotificationStatus.FAILED
@@ -446,7 +507,7 @@ def insert_notification_history_delete_notifications(
          SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
              key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
              client_reference, international, phone_prefix, rate_multiplier, notification_status,
-              created_by_id, document_download_count
+              created_by_id, document_download_count, message_cost
           FROM notifications
         WHERE service_id = :service_id
           AND notification_type = :notification_type
@@ -781,7 +842,6 @@ def dao_update_delivery_receipts(receipts, delivered):
         new_receipts.append(r)
 
     receipts = new_receipts
-
     id_to_carrier = {
         r["notification.messageId"]: r["delivery.phoneCarrier"] for r in receipts
     }
@@ -790,9 +850,13 @@ def dao_update_delivery_receipts(receipts, delivered):
     }
     id_to_timestamp = {r["notification.messageId"]: r["@timestamp"] for r in receipts}
 
+    id_to_message_cost = {
+        r["notification.messageId"]: r["delivery.priceInUSD"] for r in receipts
+    }
     status_to_update_with = NotificationStatus.DELIVERED
     if not delivered:
         status_to_update_with = NotificationStatus.FAILED
+
     stmt = (
         update(Notification)
         .where(Notification.message_id.in_(id_to_carrier.keys()))
@@ -816,6 +880,12 @@ def dao_update_delivery_receipts(receipts, delivered):
                     for key, value in id_to_provider_response.items()
                 ]
             ),
+            message_cost=case(
+                *[
+                    (Notification.message_id == key, value)
+                    for key, value in id_to_message_cost.items()
+                ]
+            ),
         )
     )
     db.session.execute(stmt)
@@ -847,7 +917,6 @@ def dao_close_out_delivery_receipts():
 
 
 def dao_batch_insert_notifications(batch):
-
     db.session.bulk_save_objects(batch)
     db.session.commit()
     current_app.logger.info(f"Batch inserted notifications: {len(batch)}")

app/dao/services_dao.py

@@ -8,7 +8,11 @@
 
 from app import db
 from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.date_util import generate_date_range, get_current_calendar_year
+from app.dao.date_util import (
+    generate_date_range,
+    generate_hourly_range,
+    get_current_calendar_year,
+)
 from app.dao.organization_dao import dao_get_organization_by_email_address
 from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
@@ -523,6 +527,75 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
     return total_notifications, data
 
 
+def dao_fetch_stats_for_service_from_hours(service_id, start_date, end_date):
+    start_date = get_midnight_in_utc(start_date)
+    end_date = get_midnight_in_utc(end_date + timedelta(days=1))
+
+    # Update to group by HOUR instead of DAY
+    total_substmt = (
+        select(
+            func.date_trunc("hour", NotificationAllTimeView.created_at).label(
+                "hour"
+            ),  # UPDATED
+            Job.notification_count.label("notification_count"),
+        )
+        .join(Job, NotificationAllTimeView.job_id == Job.id)
+        .where(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+        )
+        .group_by(
+            Job.id,
+            Job.notification_count,
+            func.date_trunc("hour", NotificationAllTimeView.created_at),  # UPDATED
+        )
+        .subquery()
+    )
+
+    # Also update this to group by hour
+    total_stmt = select(
+        total_substmt.c.hour,  # UPDATED
+        func.sum(total_substmt.c.notification_count).label("total_notifications"),
+    ).group_by(
+        total_substmt.c.hour
+    )  # UPDATED
+
+    # Ensure we're using hourly timestamps in the response
+    total_notifications = {
+        row.hour: row.total_notifications
+        for row in db.session.execute(total_stmt).all()
+    }
+
+    # Update the second query to also use "hour"
+    stmt = (
+        select(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("hour", NotificationAllTimeView.created_at).label(
+                "hour"
+            ),  # UPDATED
+            func.count(NotificationAllTimeView.id).label("count"),
+        )
+        .where(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+        )
+        .group_by(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("hour", NotificationAllTimeView.created_at),  # UPDATED
+        )
+    )
+
+    data = db.session.execute(stmt).all()
+
+    return total_notifications, data
+
+
 def dao_fetch_stats_for_service_from_days_for_user(
     service_id, start_date, end_date, user_id
 ):
@@ -827,3 +900,38 @@ def get_specific_days_stats(
         for day, rows in grouped_data.items()
     }
     return stats
+
+
+def get_specific_hours_stats(
+    data, start_date, hours=None, end_date=None, total_notifications=None
+):
+    if hours is not None and end_date is not None:
+        raise ValueError("Only set hours OR set end_date, not both.")
+    elif hours is not None:
+        gen_range = [start_date + timedelta(hours=i) for i in range(hours)]
+    elif end_date is not None:
+        gen_range = generate_hourly_range(start_date, end_date=end_date)
+    else:
+        raise ValueError("Either hours or end_date must be set.")
+
+    # Ensure all hours exist in the output (even if empty)
+    grouped_data = {hour: [] for hour in gen_range}
+
+    # Group notifications based on full hour timestamps
+    for row in data:
+        notification_type, status, timestamp, count = row
+
+        row_hour = timestamp.replace(minute=0, second=0, microsecond=0)
+        if row_hour in grouped_data:
+            grouped_data[row_hour].append(row)
+
+    # Format statistics, returning only hours with results
+    stats = {
+        hour.strftime("%Y-%m-%dT%H:00:00Z"): statistics.format_statistics(
+            rows, total_notifications.get(hour, 0) if total_notifications else None
+        )
+        for hour, rows in grouped_data.items()
+        if rows
+    }
+
+    return stats

app/delivery/send_to_providers.py

@@ -118,7 +118,13 @@ def send_sms_to_provider(notification):
                     "international": notification.international,
                 }
                 db.session.close()  # no commit needed as no changes to objects have been made above
-
+                real_sender_number = notification.reply_to_text
+                # interleave spaces to bypass PII scrubbing since sender number is not PII
+                arr = list(real_sender_number)
+                real_sender_number = " ".join(arr)
+                current_app.logger.info(
+                    f"#notify-debug-api-1701 real sender number going to AWS is {real_sender_number}"
+                )
                 message_id = provider.send_sms(**send_sms_kwargs)
 
                 update_notification_message_id(notification.id, message_id)