diff --git a/per/dashboard_data.py b/per/dashboard_data.py new file mode 100644 index 000000000..f5fef144a --- /dev/null +++ b/per/dashboard_data.py @@ -0,0 +1,444 @@ +from collections import defaultdict +from typing import Any + +from django.db.models import Prefetch + +from .dashboard_utils import AREA_NAMES, contains_affirmative +from .models import ( + AreaResponse, + FormComponentResponse, + FormPrioritization, + FormPrioritizationComponent, + Overview, + PerAssessment, +) + + +def _phase_display(phase: int | None) -> str | None: + try: + display = Overview.Phase(phase).label + except (TypeError, ValueError): + return None + + if display == "Action And Accountability": + return "Action & accountability" + if display == "WorkPlan": + return "Workplan" + return display + + +def _area_name(area: Any) -> str | None: + if area is None: + return None + + area_number = getattr(area, "area_num", None) + if isinstance(area_number, int): + return AREA_NAMES.get(area_number, getattr(area, "title", None)) + return getattr(area, "title", None) or getattr(area, "name", None) + + +def _coordinate(country: Any, coordinate: str) -> float | None: + if country is None: + return None + + centroid = getattr(country, "centroid", None) + if centroid is not None: + value = getattr(centroid, coordinate, None) + else: + fallback_coordinate = {"x": "longitude", "y": "latitude"}[coordinate] + value = getattr(country, fallback_coordinate, None) + + return round(value, 5) if value is not None else None + + +def _date_sort_key(value: Any) -> int: + return value.toordinal() if value is not None else -1 + + +def _datetime_sort_key(value: Any) -> float: + if value is None: + return 0.0 + + try: + return value.timestamp() + except (AttributeError, OSError, OverflowError, ValueError): + return 0.0 + + +def _overview_sort_key(overview: Overview) -> tuple[int, int, float, int]: + return ( + overview.assessment_number if overview.assessment_number is not None else -1, + _date_sort_key(overview.date_of_assessment), + _datetime_sort_key(overview.updated_at), + overview.id, + ) + + +def _assessment_sort_key(assessment: PerAssessment) -> tuple[int, int]: + overview = assessment.overview + return (overview.id if overview is not None else -1, assessment.id) + + +def _load_overviews() -> list[Overview]: + return list( + Overview.objects.select_related( + "country", + "country__region", + "type_of_assessment", + ).order_by("id") + ) + + +def _load_related_data( + overview_ids: list[int], + *, + latest_assessment_only: bool = False, + include_prioritization: bool = True, + include_component_details: bool = True, +) -> tuple[dict[int, list[PerAssessment]], dict[int, FormPrioritization]]: + if not overview_ids: + return {}, {} + + component_response_queryset = FormComponentResponse.objects.select_related( + "component", + "component__area", + "rating", + ) + if not include_component_details: + component_response_queryset = component_response_queryset.defer( + "urban_considerations", + "epi_considerations", + "climate_environmental_considerations", + "migration_considerations", + "notes", + ) + + assessment_queryset = PerAssessment.objects.filter(overview_id__in=overview_ids).select_related( + "overview", "overview__country", "overview__country__region", "overview__type_of_assessment" + ) + if latest_assessment_only: + assessment_queryset = assessment_queryset.order_by("overview_id", "-id").distinct("overview_id") + else: + assessment_queryset = assessment_queryset.order_by("overview_id", "-id") + assessment_queryset = assessment_queryset.prefetch_related( + Prefetch( + "area_responses", + queryset=AreaResponse.objects.select_related("area").prefetch_related( + Prefetch("component_response", queryset=component_response_queryset) + ), + ) + ) + assessments_by_overview: dict[int, list[PerAssessment]] = defaultdict(list) + for assessment in assessment_queryset: + if assessment.overview_id is not None: + assessments_by_overview[assessment.overview_id].append(assessment) + + if not include_prioritization: + return dict(assessments_by_overview), {} + + prioritization_queryset = ( + FormPrioritization.objects.filter(overview_id__in=overview_ids) + .order_by("overview_id", "-id") + .prefetch_related( + Prefetch( + "prioritized_action_responses", + queryset=FormPrioritizationComponent.objects.select_related("component", "component__area"), + ) + ) + ) + prioritization_by_overview: dict[int, FormPrioritization] = {} + for prioritization in prioritization_queryset: + prioritization_by_overview.setdefault(prioritization.overview_id, prioritization) + + return dict(assessments_by_overview), prioritization_by_overview + + +def _serialize_component_response(response: FormComponentResponse) -> dict[str, Any] | None: + component = response.component + if component is None: + return None + + area = component.area + rating = response.rating + return { + "response_id": response.id, + "component_id": component.id, + "component_name": component.title or component.description, + "component_num": component.component_num, + "area_id": area.id if area is not None else None, + "area_name": _area_name(area), + "rating_id": rating.id if rating is not None else None, + "rating_value": rating.value if rating is not None else None, + "rating_title": rating.title if rating is not None else None, + "urban_considerations": response.urban_considerations, + "epi_considerations": response.epi_considerations, + "climate_environmental_considerations": response.climate_environmental_considerations, + "migration_considerations": response.migration_considerations, + "notes": response.notes, + } + + +def _serialize_assessment_components(assessment: PerAssessment | None) -> list[dict[str, Any]]: + if assessment is None: + return [] + + components: list[dict[str, Any]] = [] + for area_response in assessment.area_responses.all(): + for component_response in area_response.component_response.all(): + serialized = _serialize_component_response(component_response) + if serialized is not None: + components.append(serialized) + return components + + +def _serialize_prioritized_components(prioritization: FormPrioritization | None) -> list[dict[str, Any]]: + if prioritization is None: + return [] + + components: list[dict[str, Any]] = [] + for prioritized_component in prioritization.prioritized_action_responses.all(): + component = prioritized_component.component + if ( + component is None + or component.id == 14 + # Historic prioritization rows use NULL to mean that the component + # is selected by its membership in this relation. Only an explicit + # False value means that it must not appear in the public summary. + or prioritized_component.is_prioritized is False + ): + continue + + components.append( + { + "componentId": component.id, + "componentTitle": component.title or component.description, + "areaTitle": _area_name(component.area), + "description": component.description, + } + ) + return components + + +def _base_process_data(overview: Overview) -> dict[str, Any]: + country = overview.country + region = country.region if country is not None else None + type_of_assessment = overview.type_of_assessment + latitude = _coordinate(country, "y") + longitude = _coordinate(country, "x") + + return { + "id": overview.id, + "assessment_number": overview.assessment_number, + "date_of_assessment": overview.date_of_assessment, + "assessment_date": overview.date_of_assessment, + "created_at": overview.created_at, + "updated_at": overview.updated_at, + "country_id": overview.country_id, + "country_name": country.name if country is not None else None, + "country_iso3": country.iso3 if country is not None else None, + "region_id": country.region_id if country is not None else None, + "region_name": region.label if region is not None else None, + "latitude": latitude, + "longitude": longitude, + "lat": latitude, + "lon": longitude, + "phase": overview.phase, + "phase_display": _phase_display(overview.phase), + "type_of_assessment": overview.type_of_assessment_id, + "type_of_assessment_name": type_of_assessment.name if type_of_assessment is not None else None, + "assessment_method": overview.assessment_method, + } + + +def _serialize_process( + overview: Overview, + assessments: list[PerAssessment], + prioritization: FormPrioritization | None, +) -> dict[str, Any]: + latest_assessment = assessments[0] if assessments else None + component_responses = _serialize_assessment_components(latest_assessment) + derived_considerations = { + "epi_considerations": any(contains_affirmative(item["epi_considerations"]) for item in component_responses), + "climate_environmental_considerations": any( + contains_affirmative(item["climate_environmental_considerations"]) for item in component_responses + ), + "urban_considerations": any(contains_affirmative(item["urban_considerations"]) for item in component_responses), + "migration_considerations": any(contains_affirmative(item["migration_considerations"]) for item in component_responses), + } + + return { + **_base_process_data(overview), + "prioritized_components": _serialize_prioritized_components(prioritization), + "epi_considerations": overview.assess_preparedness_of_country, + "climate_environmental_considerations": overview.assess_climate_environment_of_country, + "urban_considerations": overview.assess_urban_aspect_of_country, + "migration_considerations": overview.assess_migration_aspect_of_country, + "epi_considerations_from_assessment": derived_considerations["epi_considerations"], + "climate_environmental_considerations_from_assessment": derived_considerations["climate_environmental_considerations"], + "urban_considerations_from_assessment": derived_considerations["urban_considerations"], + "migration_considerations_from_assessment": derived_considerations["migration_considerations"], + "components": component_responses, + } + + +def get_per_map_data() -> dict[str, list[dict[str, Any]]]: + overviews = _load_overviews() + assessments_by_overview, prioritization_by_overview = _load_related_data( + [overview.id for overview in overviews], + latest_assessment_only=True, + ) + processes = [ + _serialize_process( + overview, + assessments_by_overview.get(overview.id, []), + prioritization_by_overview.get(overview.id), + ) + for overview in overviews + ] + + latest_overview_by_country: dict[int | None, Overview] = {} + latest_process_by_country: dict[int | None, dict[str, Any]] = {} + for overview, process in zip(overviews, processes): + current = latest_overview_by_country.get(overview.country_id) + if current is None or _overview_sort_key(overview) > _overview_sort_key(current): + latest_overview_by_country[overview.country_id] = overview + latest_process_by_country[overview.country_id] = process + + results = sorted( + latest_process_by_country.values(), + key=lambda process: ( + process["country_id"] is None, + process["country_id"] if process["country_id"] is not None else 0, + ), + ) + + return { + "results": results, + "processes": processes, + } + + +def _component_assessment_metadata(assessment: PerAssessment) -> dict[str, Any]: + overview = assessment.overview + country = overview.country if overview is not None else None + region = country.region if country is not None else None + type_of_assessment = overview.type_of_assessment if overview is not None else None + return { + "assessment_id": assessment.id, + "process_id": overview.id if overview is not None else None, + "assessment_number": overview.assessment_number if overview is not None else None, + "country_id": overview.country_id if overview is not None else None, + "country_name": country.name if country is not None else None, + "country_iso3": country.iso3 if country is not None else None, + "region_id": country.region_id if country is not None else None, + "region_name": region.label if region is not None else None, + "date_of_assessment": overview.date_of_assessment if overview is not None else None, + "type_of_assessment": overview.type_of_assessment_id if overview is not None else None, + "type_of_assessment_name": type_of_assessment.name if type_of_assessment is not None else None, + "assessment_method": overview.assessment_method if overview is not None else None, + "updated_at": overview.updated_at if overview is not None else None, + } + + +def _country_assessment_entry(assessment: PerAssessment) -> dict[str, Any]: + overview = assessment.overview + metadata = _component_assessment_metadata(assessment) + return { + **metadata, + "date": metadata["date_of_assessment"], + "phase": overview.phase if overview is not None else None, + "phase_display": _phase_display(overview.phase) if overview is not None else None, + } + + +def _serialize_performance_component_response( + response: FormComponentResponse, +) -> dict[str, Any] | None: + component = response.component + if component is None: + return None + + area = component.area + rating = response.rating + return { + "component_id": component.id, + "component_name": component.title or component.description, + "component_num": component.component_num, + "area_id": area.id if area is not None else None, + "area_name": _area_name(area), + "rating_value": rating.value if rating is not None else None, + "rating_title": rating.title if rating is not None else None, + } + + +def _serialize_performance_assessment_components( + assessment: PerAssessment, +) -> list[dict[str, Any]]: + components: list[dict[str, Any]] = [] + for area_response in assessment.area_responses.all(): + for component_response in area_response.component_response.all(): + serialized = _serialize_performance_component_response(component_response) + if serialized is not None: + components.append(serialized) + return components + + +def get_per_dashboard_data() -> dict[str, Any]: + overviews = _load_overviews() + assessments_by_overview, _ = _load_related_data( + [overview.id for overview in overviews], + include_prioritization=False, + include_component_details=False, + ) + component_map: dict[int, dict[str, Any]] = {} + country_assessments: dict[str, list[dict[str, Any]]] = defaultdict(list) + + assessments = sorted( + [assessment for values in assessments_by_overview.values() for assessment in values], + key=_assessment_sort_key, + ) + for assessment in assessments: + metadata = _component_assessment_metadata(assessment) + components = _serialize_performance_assessment_components(assessment) + for component in components: + component_id = component["component_id"] + component_map.setdefault( + component_id, + { + "component_id": component_id, + "component_num": component["component_num"], + "component_name": component["component_name"], + "area_id": component["area_id"], + "area_name": component["area_name"], + "assessments": [], + }, + )["assessments"].append( + { + **metadata, + "rating_value": component["rating_value"], + "rating_title": component["rating_title"], + } + ) + + country_name = metadata["country_name"] + if country_name: + country_assessments[country_name].append(_country_assessment_entry(assessment)) + + items = sorted( + component_map.values(), + key=lambda item: (item["area_id"] or 0, item["component_num"] or 0, item["component_id"] or 0), + ) + for item in items: + item["assessments"].sort( + key=lambda assessment: ( + assessment["country_id"] if assessment["country_id"] is not None else -1, + assessment["assessment_number"] if assessment["assessment_number"] is not None else -1, + _date_sort_key(assessment["date_of_assessment"]), + assessment["assessment_id"], + ) + ) + + return { + "assessments": items, + "countryAssessments": dict(country_assessments), + } diff --git a/per/dashboard_utils.py b/per/dashboard_utils.py new file mode 100644 index 000000000..2ad233c58 --- /dev/null +++ b/per/dashboard_utils.py @@ -0,0 +1,37 @@ +from unicodedata import category, normalize + +AREA_NAMES = { + 1: "Policy Strategy and Standards", + 2: "Analysis and planning", + 3: "Operational capacity", + 4: "Coordination", + 5: "Operations support", +} + +AFFIRMATIVE_WORDS = { + "yes", + "si", + "sí", + "oui", + "da", + "ja", + "sim", + "aye", + "yep", + "igen", + "hai", + "evet", + "是", + "はい", + "예", + "نعم", +} + + +def contains_affirmative(value: object) -> bool: + if not isinstance(value, str) or not value: + return False + + normalized = normalize("NFD", value.casefold()) + normalized = "".join(character for character in normalized if category(character) != "Mn") + return any(word in normalized for word in AFFIRMATIVE_WORDS) diff --git a/per/drf_views.py b/per/drf_views.py index ef7b87395..0d531acde 100644 --- a/per/drf_views.py +++ b/per/drf_views.py @@ -2,6 +2,7 @@ import pytz from django.conf import settings +from django.core.cache import cache from django.db import transaction from django.db.models import Count, F, Prefetch, Q from django.http import HttpResponse @@ -42,6 +43,8 @@ from .admin_classes import RegionRestrictedAdmin from .custom_renderers import NarrowCSVRenderer +from .dashboard_data import get_per_dashboard_data, get_per_map_data +from .dashboard_utils import contains_affirmative from .models import ( AreaResponse, AssessmentType, @@ -101,51 +104,6 @@ UserPerCountrySerializer, ) -# Helpers for transformed "-2" endpoints -AREA_NAMES = { - 1: "Policy Strategy and Standards", - 2: "Analysis and planning", - 3: "Operational capacity", - 4: "Coordination", - 5: "Operations support", -} - -AFFIRMATIVE_WORDS = {"yes", "si", "sí", "oui", "da", "ja", "sim", "aye", "yep", "igen", "hai", "evet", "是", "はい", "예", "نعم"} - - -def _contains_affirmative(text: str) -> bool: - if not text or not isinstance(text, str): - return False - try: - import unicodedata - - normalized = unicodedata.normalize("NFD", text.lower()) - normalized = "".join(ch for ch in normalized if unicodedata.category(ch) != "Mn") - except Exception: - normalized = text.lower() - return any(word in normalized for word in AFFIRMATIVE_WORDS) - - -def _phase_display_from_int(phase: int | None, existing_display: str | None = None) -> str | None: - """Return normalized phase display using Overview.Phase IntegerChoices. - - Uses the IntegerChoices label, then normalizes: - - "WorkPlan" -> "Workplan" - - "Action And Accountability" -> "Action & accountability" - """ - label = None - try: - if isinstance(phase, int): - label = Overview.Phase(phase).label # from IntegerChoices - except Exception: - label = None - disp = label or existing_display - if disp == "Action And Accountability": - return "Action & accountability" - if disp == "WorkPlan": - return "Workplan" - return disp - class PERDocsFilter(filters.FilterSet): id = filters.NumberFilter(field_name="id", lookup_expr="exact") @@ -732,6 +690,12 @@ def get_queryset(self): # Consolidated public endpoints (map-data, assessments-processed, dashboard-data) +PER_MAP_DATA_CACHE_KEY = "per-dashboard:map-data:v2" +PER_MAP_DATA_CACHE_TIMEOUT_SECONDS = 60 +PER_DASHBOARD_DATA_CACHE_KEY = "per-dashboard:dashboard-data:v1" +PER_DASHBOARD_DATA_CACHE_TIMEOUT_SECONDS = 60 + + class PerMapDataView(views.APIView): """Public consolidated PER map data. @@ -739,142 +703,21 @@ class PerMapDataView(views.APIView): """ def get(self, request): - latest_overviews = ( - Overview.objects.order_by("country_id", "-assessment_number", "-date_of_assessment") - .distinct("country_id") - .select_related("country", "type_of_assessment", "country__region") - ) - items = [] - for ov in latest_overviews: - # Compute normalized phase display from int value or existing string - normalized_phase_display = _phase_display_from_int(getattr(ov, "phase", None), getattr(ov, "phase_display", None)) - - # Attach components from latest assessment tied to the overview - components = [] - epi_considerations = False - climate_considerations = False - urban_considerations = False - migration_considerations = False - latest_assessment = ( - PerAssessment.objects.filter(overview_id=getattr(ov, "id", None)) - .prefetch_related( - Prefetch( - "area_responses", - queryset=AreaResponse.objects.prefetch_related( - Prefetch( - "component_response", - queryset=FormComponentResponse.objects.select_related("component", "component__area", "rating"), - ) - ), - ) - ) - .first() + if settings.DJANGO_READ_ONLY: + cached_data = cache.get(PER_MAP_DATA_CACHE_KEY) + if cached_data is not None: + return Response(cached_data) + + data = get_per_map_data() + + if settings.DJANGO_READ_ONLY: + cache.set( + PER_MAP_DATA_CACHE_KEY, + data, + timeout=PER_MAP_DATA_CACHE_TIMEOUT_SECONDS, ) - if latest_assessment: - for ar in latest_assessment.area_responses.all(): - for cr in ar.component_response.all(): - # Flags - epi = _contains_affirmative(getattr(cr, "epi_considerations", "")) - urb = _contains_affirmative(getattr(cr, "urban_considerations", "")) - clim = _contains_affirmative(getattr(cr, "climate_environmental_considerations", "")) - mig = _contains_affirmative(getattr(cr, "migration_considerations", "")) - epi_considerations = epi_considerations or epi - urban_considerations = urban_considerations or urb - climate_considerations = climate_considerations or clim - migration_considerations = migration_considerations or mig - - comp = getattr(cr, "component", None) - area = getattr(comp, "area", None) if comp else None - rating = getattr(cr, "rating", None) - # Resolve area name via AREA_NAMES when area_num is an int - area_num_val = getattr(area, "area_num", None) - components.append( - { - "component_id": getattr(comp, "id", None) or getattr(cr, "component_id", None), - "component_name": getattr(comp, "title", None) - or getattr(comp, "description_en", None) - or getattr(comp, "description", None), - "component_num": getattr(comp, "component_num", None), - "area_id": getattr(area, "id", None), - "area_name": ( - AREA_NAMES.get(area_num_val) if isinstance(area_num_val, int) else getattr(area, "name", None) - ), - "rating_value": getattr(rating, "value", None), - "rating_title": getattr(rating, "title", None), - } - ) - - # Prioritized components (workplan/prioritization) - prioritized_components = [] - fp = FormPrioritization.objects.filter(overview_id=getattr(ov, "id", None)).first() - if fp: - for pac in fp.prioritized_action_responses.exclude(component_id=14).select_related( - "component", "component__area" - ): - pc_comp = pac.component - pc_area = pc_comp.area if pc_comp else None - area_num_val2 = getattr(pc_area, "area_num", None) - prioritized_components.append( - { - "componentId": getattr(pc_comp, "id", None), - "componentTitle": getattr(pc_comp, "title", None) - or getattr(pc_comp, "description_en", None) - or getattr(pc_comp, "description", None), - "areaTitle": ( - AREA_NAMES.get(area_num_val2) - if isinstance(area_num_val2, int) - else getattr(pc_area, "name", None) - ), - "description": getattr(pc_comp, "description", None) or getattr(pc_comp, "description_en", None), - } - ) - items.append( - { - "id": getattr(ov, "id", None), - "assessment_number": ov.assessment_number, - "date_of_assessment": ov.date_of_assessment, - "country_id": getattr(ov, "country_id", None), - "country_name": ov.country.name if ov.country else None, - "phase": getattr(ov, "phase", None), - "phase_display": normalized_phase_display, - "type_of_assessment": getattr(ov.type_of_assessment, "id", None), - "type_of_assessment_name": getattr(ov.type_of_assessment, "name", None), - "country_iso3": getattr(ov.country, "iso3", None), - "region_id": getattr(getattr(ov.country, "region", None), "id", None), - "region_name": getattr(getattr(ov.country, "region", None), "label", None), - "latitude": ( - round(ov.country.centroid.y, 5) - if getattr(ov.country, "centroid", None) - else ( - round(getattr(ov.country, "latitude", None), 5) - if getattr(ov.country, "latitude", None) is not None - else None - ) - ), - "longitude": ( - round(ov.country.centroid.x, 5) - if getattr(ov.country, "centroid", None) - else ( - round(getattr(ov.country, "longitude", None), 5) - if getattr(ov.country, "longitude", None) is not None - else None - ) - ), - "updated_at": getattr(ov, "updated_at", None), - "prioritized_components": prioritized_components, - "epi_considerations": getattr(ov, "assess_preparedness_of_country", None), - "climate_environmental_considerations": getattr(ov, "assess_climate_environment_of_country", None), - "urban_considerations": getattr(ov, "assess_urban_aspect_of_country", None), - "migration_considerations": getattr(ov, "assess_migration_aspect_of_country", None), - "epi_considerations_from_assessment": epi_considerations, - "climate_environmental_considerations_from_assessment": climate_considerations, - "urban_considerations_from_assessment": urban_considerations, - "migration_considerations_from_assessment": migration_considerations, - "components": components, - } - ) - return Response({"results": items}) + return Response(data) class PerAssessmentsProcessedView(views.APIView): @@ -945,12 +788,12 @@ def get(self, request): "epi_considerations": getattr(cr, "epi_considerations", None), "climate_environmental_considerations": getattr(cr, "climate_environmental_considerations", None), "migration_considerations": getattr(cr, "migration_considerations", None), - "urban_considerations_simplified": _contains_affirmative(getattr(cr, "urban_considerations", "")), - "epi_considerations_simplified": _contains_affirmative(getattr(cr, "epi_considerations", "")), - "climate_environmental_considerations_simplified": _contains_affirmative( + "urban_considerations_simplified": contains_affirmative(getattr(cr, "urban_considerations", "")), + "epi_considerations_simplified": contains_affirmative(getattr(cr, "epi_considerations", "")), + "climate_environmental_considerations_simplified": contains_affirmative( getattr(cr, "climate_environmental_considerations", "") ), - "migration_considerations_simplified": _contains_affirmative( + "migration_considerations_simplified": contains_affirmative( getattr(cr, "migration_considerations", "") ), "notes": getattr(cr, "notes", None), @@ -992,103 +835,21 @@ class PerDashboardDataView(views.APIView): """ def get(self, request): - # Build aggregation by component across all assessments - component_map = {} - country_assessments: dict[str, list] = {} - # Prefetch for performance - assessments = PerAssessment.objects.select_related("overview", "overview__country").prefetch_related( - Prefetch( - "area_responses", - queryset=AreaResponse.objects.prefetch_related( - Prefetch( - "component_response", - queryset=FormComponentResponse.objects.select_related("component", "component__area", "rating"), - ) - ), + if settings.DJANGO_READ_ONLY: + cached_data = cache.get(PER_DASHBOARD_DATA_CACHE_KEY) + if cached_data is not None: + return Response(cached_data) + + data = get_per_dashboard_data() + + if settings.DJANGO_READ_ONLY: + cache.set( + PER_DASHBOARD_DATA_CACHE_KEY, + data, + timeout=PER_DASHBOARD_DATA_CACHE_TIMEOUT_SECONDS, ) - ) - - for a in assessments: - assessment_entry = { - "assessment_id": getattr(a, "id", None), - "assessment_number": getattr(a.overview, "assessment_number", None), - "date_of_assessment": getattr(a.overview, "date_of_assessment", None), - "country_id": getattr(a.overview, "country_id", None), - "country_name": getattr(getattr(a.overview, "country", None), "name", None), - "country_iso3": getattr(getattr(a.overview, "country", None), "iso3", None), - } - # Also prepare detailed assessment for countryAssessments with ratings - ca_components = [] - - for ar in a.area_responses.all(): - for cr in ar.component_response.all(): - comp = getattr(cr, "component", None) - if comp is None: - continue - area = getattr(comp, "area", None) - comp_id = getattr(comp, "id", None) - if comp_id is None: - continue - # Component key aggregation - if comp_id not in component_map: - component_map[comp_id] = { - "component_id": comp_id, - "component_num": getattr(comp, "component_num", None), - "component_name": getattr(comp, "title", None) - or getattr(comp, "description_en", None) - or getattr(comp, "description", None), - "area_id": getattr(area, "id", None), - "area_name": ( - AREA_NAMES.get(int(getattr(area, "area_num", 0))) - if isinstance(getattr(area, "area_num", None), int) - else getattr(area, "name", None) - ), - "assessments": [], - } - - component_map[comp_id]["assessments"].append(assessment_entry) - - # Build component entry with rating for countryAssessments - rating = getattr(cr, "rating", None) - ca_components.append( - { - "component_id": comp_id, - "component_name": getattr(comp, "title", None) - or getattr(comp, "description_en", None) - or getattr(comp, "description", None), - "component_num": getattr(comp, "component_num", None), - "area_id": getattr(area, "id", None), - "area_name": ( - AREA_NAMES.get(int(getattr(area, "area_num", 0))) - if isinstance(getattr(area, "area_num", None), int) - else getattr(area, "name", None) - ), - "rating_value": getattr(rating, "value", None), - "rating_title": getattr(rating, "title", None) or "", - } - ) - - # Append to countryAssessments mapping - country_name = assessment_entry["country_name"] - if country_name: - phase_display = _phase_display_from_int( - getattr(a.overview, "phase", None), getattr(a.overview, "phase_display", None) - ) - country_assessments.setdefault(country_name, []).append( - { - "assessment_number": assessment_entry["assessment_number"], - "date": assessment_entry["date_of_assessment"], - "components": ca_components, - "phase": getattr(a.overview, "phase", None), - "phase_display": phase_display, - } - ) - # Convert to list - items = list(component_map.values()) - # Optional: sort by area then component_num for stable output - items.sort(key=lambda x: ((x["area_id"] or 0), (x["component_num"] or 0))) - return Response({"assessments": items, "countryAssessments": country_assessments}) + return Response(data) class PerFileViewSet(mixins.ListModelMixin, mixins.CreateModelMixin, viewsets.GenericViewSet): diff --git a/per/test_views.py b/per/test_views.py index e2debbe9b..3d0120de2 100644 --- a/per/test_views.py +++ b/per/test_views.py @@ -1,7 +1,12 @@ import json +from datetime import date from unittest import mock from django.core import management +from django.core.cache import cache +from django.db import connection +from django.test import override_settings +from django.test.utils import CaptureQueriesContext from api.factories.country import CountryFactory from api.factories.region import RegionFactory @@ -19,7 +24,22 @@ SectorTagFactory, ) -from .models import WorkPlanStatus +from .dashboard_utils import contains_affirmative +from .models import ( + AreaResponse, + FormComponentResponse, + FormPrioritizationComponent, + PerAssessment, + PerComponentRating, + WorkPlanStatus, +) + +TEST_LOC_MEM_CACHE = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "per-dashboard-tests", + } +} class PerTestCase(APITestCase): @@ -426,3 +446,234 @@ def test_ops_learning_coverage_aggregates_duplicate_appeal(self): self.assertEqual(result_by_appeal[self.appeal1.code]["counts"], 2) self.assertEqual(result_by_appeal[self.appeal1.code]["tagging_status"], "in_progress") + + +class PerDashboardDataTestCase(APITestCase): + def setUp(self): + super().setUp() + self.region = RegionFactory.create(name=0, label="Africa") + + def create_country(self, name: str, iso: str, iso3: str): + return CountryFactory.create(name=name, iso=iso, iso3=iso3, region=self.region) + + def create_component_assessment(self, overview, *, component_id=None, duplicate=False, priority_value=True): + area = FormAreaFactory.create(title="Analysis and planning", area_num=2) + component = FormComponentFactory.create( + area=area, + component_num=3, + title="Component three", + **({"id": component_id} if component_id is not None else {}), + ) + rating = PerComponentRating.objects.create(title="High", value=4) + component_response = FormComponentResponse.objects.create( + component=component, + rating=rating, + epi_considerations="yes", + migration_considerations=None, + ) + area_response = AreaResponse.objects.create(area=area) + area_response.component_response.add(component_response) + + assessment = PerAssessment.objects.create(overview=overview) + assessment.area_responses.add(area_response) + + if duplicate: + duplicate_area_response = AreaResponse.objects.create(area=area) + duplicate_area_response.component_response.add(component_response) + assessment.area_responses.add(duplicate_area_response) + + prioritization = FormPrioritizationFactory.create(overview=overview) + prioritized_component = FormPrioritizationComponent.objects.create( + component=component, + is_prioritized=priority_value, + ) + prioritization.prioritized_action_responses.add(prioritized_component) + return assessment, component + + def test_dashboard_helpers_normalize_affirmative_responses(self): + self.assertTrue(contains_affirmative("YES")) + self.assertTrue(contains_affirmative("Si\u0301")) + self.assertFalse(contains_affirmative("no")) + self.assertFalse(contains_affirmative(None)) + + def test_map_data_returns_complete_history_and_deterministic_latest_processes(self): + country_one = self.create_country("Country One", "C1", "C01") + country_two = self.create_country("Country Two", "C2", "C02") + older_dated = OverviewFactory.create( + country=country_one, + assessment_number=1, + date_of_assessment=None, + ) + latest_dated = OverviewFactory.create( + country=country_one, + assessment_number=1, + date_of_assessment=date(2024, 1, 1), + ) + latest_number = OverviewFactory.create( + country=country_two, + assessment_number=2, + date_of_assessment=None, + ) + + response = self.client.get("/api/v2/per-map-data") + + self.assert_200(response) + self.assertEqual( + {item["id"] for item in response.data["processes"]}, + { + older_dated.id, + latest_dated.id, + latest_number.id, + }, + ) + self.assertEqual(len(response.data["results"]), 2) + latest_by_country = {item["country_id"]: item for item in response.data["results"]} + self.assertEqual(latest_by_country[country_one.id]["id"], latest_dated.id) + self.assertEqual(latest_by_country[country_two.id]["id"], latest_number.id) + + def test_map_data_results_are_ordered_by_country(self): + first_country = self.create_country("First Country", "F1", "F01") + second_country = self.create_country("Second Country", "S1", "S01") + OverviewFactory.create(country=second_country) + OverviewFactory.create(country=first_country) + + response = self.client.get("/api/v2/per-map-data") + + self.assert_200(response) + self.assertEqual( + [item["country_id"] for item in response.data["results"]], + [first_country.id, second_country.id], + ) + + def test_map_data_preserves_authoritative_considerations_and_priorities(self): + country = self.create_country("Country With Considerations", "C3", "C03") + overview = OverviewFactory.create( + country=country, + assess_preparedness_of_country=True, + assess_climate_environment_of_country=False, + assess_urban_aspect_of_country=None, + assess_migration_aspect_of_country=True, + ) + self.create_component_assessment(overview) + + response = self.client.get("/api/v2/per-map-data") + + self.assert_200(response) + process = response.data["results"][0] + self.assertIs(process["epi_considerations"], True) + self.assertIs(process["climate_environmental_considerations"], False) + self.assertIsNone(process["urban_considerations"]) + self.assertIs(process["migration_considerations"], True) + self.assertEqual(process["migration_considerations_from_assessment"], False) + self.assertEqual(process["prioritized_components"][0]["componentId"], process["components"][0]["component_id"]) + + def test_map_data_keeps_legacy_null_priority_components(self): + country = self.create_country("Country With Legacy Priorities", "C6", "C06") + overview = OverviewFactory.create(country=country) + _, component = self.create_component_assessment(overview, component_id=100_000, priority_value=None) + + response = self.client.get("/api/v2/per-map-data") + + self.assert_200(response) + process = response.data["results"][0] + self.assertEqual( + [item["componentId"] for item in process["prioritized_components"]], + [component.id], + ) + + def test_dashboard_data_enriches_component_assessments_and_retains_empty_country_assessments(self): + country_with_components = self.create_country("Country With Components", "C4", "C04") + country_without_components = self.create_country("Country Without Components", "C5", "C05") + overview_with_components = OverviewFactory.create(country=country_with_components) + assessment, component = self.create_component_assessment(overview_with_components, duplicate=True) + overview_without_components = OverviewFactory.create(country=country_without_components) + empty_assessment = PerAssessment.objects.create(overview=overview_without_components) + + response = self.client.get("/api/v2/per-dashboard-data") + + self.assert_200(response) + component_item = next(item for item in response.data["assessments"] if item["component_id"] == component.id) + component_assessment = next(item for item in component_item["assessments"] if item["assessment_id"] == assessment.id) + self.assertEqual(component_assessment["country_id"], country_with_components.id) + self.assertEqual(component_assessment["country_name"], country_with_components.name) + self.assertEqual(component_assessment["region_id"], self.region.id) + self.assertEqual(component_assessment["region_name"], "Africa") + self.assertIn("date_of_assessment", component_assessment) + self.assertEqual(component_assessment["rating_value"], 4) + self.assertEqual(component_assessment["rating_title"], "High") + + country_entry = response.data["countryAssessments"][country_with_components.name][0] + self.assertEqual(country_entry["assessment_id"], assessment.id) + self.assertEqual(country_entry["country_iso3"], "C04") + self.assertEqual(country_entry["phase_display"], "Orientation") + self.assertNotIn("components", country_entry) + + empty_country_entry = response.data["countryAssessments"][country_without_components.name][0] + self.assertEqual(empty_country_entry["assessment_id"], empty_assessment.id) + self.assertNotIn("components", empty_country_entry) + + @override_settings(CACHES=TEST_LOC_MEM_CACHE) + def test_map_data_uses_cache_in_read_only_mode(self): + country = self.create_country("Cached Map Country", "M1", "M01") + OverviewFactory.create(country=country) + + with override_settings(DJANGO_READ_ONLY=True): + cache.clear() + try: + first_response = self.client.get("/api/v2/per-map-data") + + self.assert_200(first_response) + with mock.patch("per.drf_views.get_per_map_data") as get_per_map_data: + second_response = self.client.get("/api/v2/per-map-data") + + self.assert_200(second_response) + get_per_map_data.assert_not_called() + self.assertEqual(second_response.data, first_response.data) + finally: + cache.clear() + + @override_settings(CACHES=TEST_LOC_MEM_CACHE) + def test_dashboard_data_uses_cache_in_read_only_mode(self): + country = self.create_country("Cached Dashboard Country", "D1", "D01") + overview = OverviewFactory.create(country=country) + self.create_component_assessment(overview) + + with override_settings(DJANGO_READ_ONLY=True): + cache.clear() + try: + first_response = self.client.get("/api/v2/per-dashboard-data") + + self.assert_200(first_response) + with mock.patch("per.drf_views.get_per_dashboard_data") as get_per_dashboard_data: + second_response = self.client.get("/api/v2/per-dashboard-data") + + self.assert_200(second_response) + get_per_dashboard_data.assert_not_called() + self.assertEqual(second_response.data, first_response.data) + finally: + cache.clear() + + def test_map_data_query_count_is_bounded_for_many_overviews(self): + for index in range(12): + country = self.create_country(f"Query Country {index}", f"Q{chr(65 + index)}", f"Q{index:02d}") + overview = OverviewFactory.create(country=country, date_of_assessment=None) + self.create_component_assessment(overview) + + with CaptureQueriesContext(connection) as queries: + response = self.client.get("/api/v2/per-map-data") + + self.assert_200(response) + self.assertLessEqual(len(queries), 8) + + def test_dashboard_data_query_count_is_bounded_for_many_assessments(self): + for index in range(12): + country = self.create_country(f"Assessment Country {index}", f"R{chr(65 + index)}", f"R{index:02d}") + overview = OverviewFactory.create(country=country, date_of_assessment=None) + self.create_component_assessment(overview) + + with CaptureQueriesContext(connection) as queries: + response = self.client.get("/api/v2/per-dashboard-data") + + self.assert_200(response) + self.assertEqual(len(response.data["countryAssessments"]), 12) + self.assertLessEqual(len(queries), 8)