diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 6239e702e..0897a49ec 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -108,7 +108,6 @@ paths: 201: description: > The feed was successfully created. No content is returned. - content: application/json: schema: @@ -116,19 +115,15 @@ paths: 400: description: > The request was invalid. - 401: description: > The request was not authenticated or has invalid authentication credentials. - 409: description: > A feed with the producer_url already exists. - 500: description: > An internal server error occurred. - put: description: Update the specified GTFS feed in the Mobility Database. tags: @@ -147,23 +142,18 @@ paths: 200: description: > The feed was successfully updated. No content is returned. - 204: description: > The feed update request was successfully received, but the update process was skipped as the request matches with the source feed. - 400: description: > The request was invalid. - 401: description: > The request was not authenticated or has invalid authentication credentials. - 500: description: > An internal server error occurred. - /v1/operations/gtfs_feeds/{id}: parameters: - $ref: "#/components/parameters/feed_id_path_param" @@ -178,7 +168,6 @@ paths: 200: description: > Successful pull of the GTFS feeds common info for the provided ID. - content: application/json: schema: @@ -189,7 +178,6 @@ paths: get: description: > Returns historical availability checks for a GTFS feed, ordered by checked_at from newest to oldest. Availability is based on scheduled lightweight HTTP checks (HEAD or ranged GET requests) and does not download or validate the full GTFS dataset. - tags: - "operations" operationId: getGtfsFeedAvailability @@ -228,7 +216,6 @@ paths: 200: description: > Successful pull of the GTFS-RT feeds common info for the provided ID. - content: application/json: schema: @@ -252,7 +239,6 @@ paths: 201: description: > The feed was successfully created. No content is returned. - content: application/json: schema: @@ -260,15 +246,12 @@ paths: 400: description: > The request was invalid. - 401: description: > The request was not authenticated or has invalid authentication credentials. - 409: description: > A feed with the producer_url already exists. - 500: description: "An internal server error occurred. \n" put: @@ -289,23 +272,18 @@ paths: 200: description: > The feed was successfully updated. No content is returned. - 204: description: > The feed update request was successfully received, but the update process was skipped as the request matches with the source feed. - 400: description: > The request was invalid. - 401: description: > The request was not authenticated or has invalid authentication credentials. - 500: description: > An internal server error occurred. - /v1/operations/licenses: get: description: Get the list of all licenses in the DB. @@ -373,7 +351,6 @@ paths: post: description: > Propagate a license ID to all feeds sharing the same normalized license URL. Use dry_run=true (the default) to preview which feeds would be updated without persisting any changes. - tags: - "licenses" operationId: propagateMatchLicense @@ -413,7 +390,6 @@ paths: 200: description: > Successful pull of the license info for the provided ID. - content: application/json: schema: @@ -624,7 +600,6 @@ paths: summary: Delete a feature flag description: > Deletes a feature flag definition. All user assignments for this flag are removed as well (cascading delete). - operationId: deleteFeatureFlag tags: - "users" @@ -694,7 +669,6 @@ paths: summary: Create an early access program description: > Creates a program and, atomically, the feature flags it grants. A program with no `feature_flags` entries is valid but grants nothing — the caller is expected to include at least one. - operationId: createEarlyAccessProgram tags: - "early-access" @@ -718,7 +692,6 @@ paths: "422": description: > Validation error — e.g. an unknown `feature_flag_id`, or a `value` that does not match that flag's `value_type`. Nothing is written. - /v1/operations/early-access-programs/{id}: parameters: - name: id @@ -750,7 +723,6 @@ paths: summary: Update an early access program description: > Updates the program's own fields (name, description, window, disabled, invite_retention_days) and replaces its feature flag grants. The program's own fields are a partial update - omit a field to leave it unchanged - but `feature_flags` is a full replace: omitting it, or sending an empty array, removes every grant this program makes going forward. This never touches `user_feature_flag` rows already granted to existing enrollees. - operationId: updateEarlyAccessProgram tags: - "early-access" @@ -776,12 +748,10 @@ paths: "422": description: > Validation error - an unknown `feature_flag_id`, or a `value` that does not match that flag's `value_type`. Nothing is written. - delete: summary: Delete an early access program description: > Deletes the program and cascades to its feature flag grants, enrollments, and any outstanding invited emails. Does not touch `user_feature_flag` rows already granted — deleting a program never revokes access already given out. - operationId: deleteEarlyAccessProgram tags: - "early-access" @@ -802,7 +772,6 @@ paths: This variant takes the addresses already extracted, as JSON. To hand over a CSV file instead, post it to `.../invited-emails:upload`, which parses it server-side and returns the same categorisation. Malformed addresses are never silently skipped by either variant - they come back in `invalid` with a reason, for the operator to fix and resubmit. - operationId: importEarlyAccessInvitedEmails tags: - "early-access" @@ -838,7 +807,6 @@ paths: summary: Bulk-remove invited emails description: > Removes outstanding (unclaimed) invited-email rows for this program. The addresses go in the request body rather than the path, so they stay out of URLs and therefore out of proxy and access logs. Removing an invite does not touch any flag already granted from an earlier claim. - operationId: removeEarlyAccessInvitedEmails tags: - "early-access" @@ -875,7 +843,6 @@ paths: Same categorisation and same `dry_run` semantics as posting JSON to `.../invited-emails`, but takes the CSV itself so an operator does not have to extract the addresses first. Send the CSV as the raw request body with `Content-Type: text/csv`, not as a multipart upload. Addresses are read from the column headed `email`, matched case-insensitively and at any position; every other column is ignored, so a CRM export can be uploaded unedited. Quoted fields containing commas are handled, as are CRLF line endings. A file with no `email` column is a 422 rather than an empty import, so a wrong file is never mistaken for a program with no invitees. The file's contents are never logged. - operationId: uploadEarlyAccessInvitedEmailsCsv tags: - "early-access" @@ -893,7 +860,6 @@ paths: required: false description: > When true (default), categorise without persisting anything. - schema: type: boolean default: true @@ -918,13 +884,11 @@ paths: "422": description: > The file has no `email` column, is not valid UTF-8, or its `email` column holds more than 5000 addresses. - /v1/operations/early-access-programs/{id}/report: get: summary: Combined enrollment and invite report for a program description: > One row per person across both halves of the program: everyone already enrolled and every address still holding an outstanding invite, distinguished by `status`. Includes a summary count block. Note that an enrolled row's `email` is the account's email as it is *now*, which is not necessarily the address the invite was sent to - the invite row is deleted on claim, so the two cannot be correlated afterwards. - operationId: getEarlyAccessProgramReport tags: - "early-access" @@ -957,7 +921,6 @@ paths: required: false description: > Restrict `rows` to one half of the program: `enrolled` for people already in it, `invited` for addresses still holding an unclaimed invite. Omit for both. The `summary` block always covers the whole program regardless of this filter. - schema: type: string enum: [enrolled, invited] @@ -966,7 +929,6 @@ paths: "200": description: > The program's combined report, as JSON or as CSV depending on `format`. In CSV the summary block is omitted and only the rows are emitted. - content: application/json: schema: @@ -1050,7 +1012,6 @@ components: status: description: > Describes status of the Feed. Should be one of - * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -1067,27 +1028,23 @@ components: official: description: > A boolean value indicating if the feed is official or not. Official feeds are provided by the transit agency or a trusted source. - type: boolean example: true official_updated_at: description: > The date and time the official status was last updated, in ISO 8601 date-time format. - type: string example: 2023-07-10T22:06:00Z format: date-time seasonal: description: > Indicates whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Seasonal feeds are excluded from the rolling 7-day service coverage checks. Defaults to false when the feed has not been marked as seasonal. - type: boolean default: false example: true feed_name: description: > An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. - type: string example: Bus note: @@ -1096,7 +1053,6 @@ components: related_links: description: > A list of related links for the feed. - type: array items: $ref: "#/components/schemas/FeedRelatedLink" @@ -1106,25 +1062,21 @@ components: code: description: > A short code to identify the type of link. - type: string example: next_1 description: description: > A description of the link. - type: string example: The URL for a future feed version with an upcoming service period. url: description: > The URL of the related link. - type: string format: url created_at: description: > The date and time the related link was created, in ISO 8601 date-time format. - type: string example: 2023-07-10T22:06:00Z format: date-time @@ -1150,7 +1102,6 @@ components: visualization_dataset_id: description: > The dataset ID of the dataset used to compute the visualization files. - type: string example: mdb-1210-202402121801 reliability_seal: @@ -1173,20 +1124,17 @@ components: system_id: description: > The system ID of the feed. This is a unique identifier for the system that the feed belongs to. - type: string example: system-1234 provider_url: description: > The URL of the provider's website. This is the website of the organization that operates the system that the feed belongs to. - type: string format: url example: https://www.citybikenyc.com/ versions: description: > A list of GBFS versions that the feed supports. Each version is represented by its version number and a list of endpoints. - type: array items: $ref: "#/components/schemas/GbfsVersion" @@ -1203,27 +1151,23 @@ components: version: description: > The version of the GBFS specification that the feed is using. This is a string that follows the semantic versioning format. - type: string example: 2.3 created_at: description: > The date when the GBFS version was saved to the database. - type: string format: date-time example: 2023-07-10T22:06:00Z last_updated_at: description: > The date when the GBFS version was last updated in the database. - type: string format: date-time example: 2023-07-10T22:06:00Z source: description: > Indicates the origin of the version information. Possible values are: - * `autodiscovery`: Retrieved directly from the main GBFS autodiscovery URL. * `gbfs_versions`: Retrieved from the `gbfs_versions` endpoint. type: string @@ -1233,7 +1177,6 @@ components: endpoints: description: > A list of endpoints that are available in the version. - type: array items: $ref: "#/components/schemas/GbfsEndpoint" @@ -1243,12 +1186,10 @@ components: type: object description: > A validation report of the GBFS feed. - properties: validated_at: description: > The date and time the GBFS feed was validated, in ISO 8601 date-time format. - type: string example: 2023-07-10T22:06:00Z format: date-time @@ -1259,14 +1200,12 @@ components: report_summary_url: description: > The URL of the JSON report of the validation summary. - type: string format: url example: https://storage.googleapis.com/mobilitydata-datasets-prod/validation-reports/gbfs-1234-202402121801.json validator_version: description: > The version of the validator used to validate the GBFS feed. - type: string example: 1.0.13 GbfsEndpoint: @@ -1275,26 +1214,22 @@ components: name: description: > The name of the endpoint. This is a human-readable name for the endpoint. - type: string example: system_information url: description: > The URL of the endpoint. This is the URL where the endpoint can be accessed. - type: string format: url example: https://gbfs.citibikenyc.com/gbfs/system_information.json language: description: > The language of the endpoint. This is the language that the endpoint is available in for versions 2.3 and prior. - type: string example: en is_feature: description: > A boolean value indicating if the endpoint is a feature. A feature is defined as an optionnal endpoint. - type: boolean example: false GbfsFeeds: @@ -1325,7 +1260,6 @@ components: example: vp description: > The type of realtime entry: - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -1366,7 +1300,6 @@ components: status: description: > Describes status of the Feed. Should be one of - * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -1390,13 +1323,11 @@ components: official: description: > A boolean value indicating if the feed is official or not. Official feeds are provided by the transit agency or a trusted source. - type: boolean example: true seasonal: description: > Indicates whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Seasonal feeds are excluded from the rolling 7-day service coverage checks. Defaults to false when the feed has not been marked as seasonal. - type: boolean default: false example: true @@ -1418,7 +1349,6 @@ components: feed_name: description: > An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. - type: string example: Bus note: @@ -1449,7 +1379,6 @@ components: example: vp description: > The type of realtime entry: - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -1484,7 +1413,6 @@ components: FeedReliabilitySummary: description: > A summary of the feed's Seal of Reliability. `null` when the feed has never been evaluated. Use `GET /v1/gtfs_feeds/{id}/reliability` for the per-criterion breakdown. - type: object nullable: true required: @@ -1494,7 +1422,6 @@ components: has_seal: description: > Whether the feed currently holds the Seal of Reliability. - type: boolean example: true earned_at: @@ -1518,13 +1445,11 @@ components: on_probation: description: > Whether at least one criterion is serving probation - the six clean months a criterion must go through, with no failure, after a confirmed failure before it can count towards the seal again. While this is true the feed cannot hold the seal even if every criterion currently passes. - type: boolean example: true probation_ends_at: description: > The earliest date the feed could regain the seal, in ISO 8601 date-time format: the end of the longest-running probation across its criteria. `null` when no criterion is on probation, and also when the stored probation has already elapsed without the nightly job clearing it - a stale countdown is not served. - type: string format: date-time nullable: true @@ -1532,7 +1457,6 @@ components: FeedReliabilityReport: description: > The full Seal of Reliability breakdown for a GTFS feed: the same summary as the embedded `reliability_seal` object, plus one entry per criterion. All six criteria are always returned, in a stable order, so a client can render them unconditionally. - type: object required: - feed_id @@ -1551,7 +1475,6 @@ components: seal_status: description: > Descriptive status of the feed's seal. `has_seal` is true only when this is `granted`. `not_granted`: at least one criterion is failing. `unknown`: none is failing, but not every criterion has been evaluated yet. `never_evaluated`: none of the criteria has been evaluated yet. - type: string enum: - granted @@ -1596,8 +1519,7 @@ components: description: > One criterion's contribution to the Seal of Reliability. - `status` is the criterion's own check at the last evaluation, undebounced, so a criterion can read `fail` while the feed still holds the seal - that is the at-risk state, and `in_grace_period` distinguishes it from a confirmed failure. Conversely a criterion can read `pass` while `on_probation` is true, in which case it still does not count towards the seal. The three states a client renders are therefore: healthy (`pass`), at risk (`fail` with `in_grace_period`), and failing (`fail` without it) - with `on_probation` as an independent flag on top. - + `status` is the criterion's debounced verdict - the one the seal is decided on, so a client can always explain the `has_seal` beside it. A criterion failing its daily check but still inside its grace period reads `pass` with `in_grace_period` true: grace is not a failing state, it is the warning before one. Conversely a criterion can read `pass` while `on_probation` is true, in which case it still does not count towards the seal. The three states a client renders are therefore: healthy (`pass`), at risk (`pass` with `in_grace_period`), and failing (`fail`) - with `on_probation` as an independent flag on top. type: object required: - criterion @@ -1608,7 +1530,6 @@ components: criterion: description: > Which criterion this entry describes. - * `official` - the feed is provided by the agency or a trusted source. * `stable` - the feed has a stable producer URL and a long enough track record. * `available` - the feed URL responds to scheduled availability checks. @@ -1626,15 +1547,14 @@ components: example: compliant status: description: > - The criterion's verdict at the last evaluation, with no grace period applied. - - * `pass` - the check passed. - * `fail` - the check failed. The seal is only withdrawn once the failure outlasts - the criterion's grace period, so check `in_grace_period` before presenting this - as a loss. - * `unknown` - the criterion was evaluated but its inputs were missing, so no verdict - could be reached this time. It is skipped when deciding the seal rather than counted - as a failure. + The criterion's debounced verdict: what it contributes to the seal, grace period already applied. + * `pass` - the criterion is not counting against the seal. Either its check passed, + or the check failed and the failure is still inside the criterion's grace period, + which `in_grace_period` tells apart. + * `fail` - the failure is confirmed and the criterion is withholding the seal. + * `unknown` - not produced. A run whose inputs were missing reaches no verdict and + leaves this value untouched, so the last verdict stands. Listed only because the + underlying column can hold it. * `not_applicable` - the criterion does not apply to this feed (for example a coverage criterion on a seasonal feed) and is withdrawn from the seal entirely. * `never_evaluated` - the criterion has produced no verdict for this feed yet. It is @@ -1649,14 +1569,12 @@ components: example: fail in_grace_period: description: > - Whether a failing check is still inside the criterion's grace period, and so is not yet counting against the seal. Can only be true while `status` is `fail`, and is always false while `on_probation` is true, since a failure during probation restarts probation outright rather than being absorbed. - + Whether the criterion's daily check is currently failing but the failure is still inside its grace period, and so is not yet counting against the seal. This is the at-risk state, and the only thing in the response that reports the raw daily check. Can only be true while `status` is `pass`, and is always false while `on_probation` is true, since a failure during probation restarts probation outright rather than being absorbed. type: boolean example: true grace_period_ends_at: description: > When the grace period expires and the failure starts counting against the seal, in ISO 8601 date-time format. `null` unless `in_grace_period` is true, and also when the window has already elapsed without the nightly job acting on it. - type: string format: date-time nullable: true @@ -1664,13 +1582,11 @@ components: on_probation: description: > Whether this criterion is serving the six clean months required after a confirmed failure. While true, the criterion does not count towards the seal whatever its `status`. Never true for `official` or `stable`, which are point-in-time state checks with no track record to rebuild. - type: boolean example: false probation_ends_at: description: > When this criterion finishes probation, in ISO 8601 date-time format. `null` when it is not on probation, and also when the window has already elapsed without the nightly job clearing it. - type: string format: date-time nullable: true @@ -1684,7 +1600,6 @@ components: first_failure_at: description: > Start of the current run of failing checks, in ISO 8601 date-time format. `null` once the criterion passes again. This is what the grace period is measured from. - type: string format: date-time nullable: true @@ -1692,7 +1607,6 @@ components: last_failure_at: description: > The most recent failing check, in ISO 8601 date-time format. Kept as history and never cleared, so it can be set on a criterion that currently passes. - type: string format: date-time nullable: true @@ -1770,7 +1684,6 @@ components: type: object description: > `latest_state` is the feed's latest dataset measured against the one before it; `latest_failure` is the same measurement at the criterion's last observed failure. Both have the structure of an `items[]` entry, and either can be null. Together they name at most four datasets, shared when the latest state is itself the failure. - required: - feed_id - items @@ -1802,7 +1715,6 @@ components: type: array description: > One entry per dataset, ordered by downloaded_at from newest to oldest. The first entry of the unpaged list is the feed's current coverage; it is marked with `is_latest`. - items: $ref: "#/components/schemas/GtfsFeedContinuousCoverage" GtfsFeedContinuousCoverage: @@ -1812,7 +1724,6 @@ components: Three windows are reported. `service_window` is the service dates the validator derived from `calendar.txt` and `calendar_dates.txt`; `feed_info_window` is what the dataset's `feed_info.txt` declares; `coverage_window` is the one the calculation actually used, with `coverage_window_source` naming which of the two it came from. Any of them may be absent when the dataset did not supply the underlying files. - required: - dataset_id - is_latest @@ -1826,7 +1737,6 @@ components: type: boolean description: > Whether this is the feed's latest dataset. Exactly one entry in the unpaged list has this set, so a client can identify the headline entry without assuming it is on the current page. - example: true downloaded_at: type: string @@ -1843,10 +1753,8 @@ components: Which input `coverage_window` was taken from. * `service_dates` - the service dates derived by the validator from `calendar.txt` and - `calendar_dates.txt`. * `feed_info` - the dates declared in `feed_info.txt`, used only when the service dates - are missing. enum: - service_dates @@ -1857,7 +1765,6 @@ components: nullable: true description: > Whether `coverage_window` stays inside the maximum coverage window the seal allows (two years). Null when there is no coverage window to measure. - example: true service_window: $ref: "#/components/schemas/ServiceDateWindow" @@ -1868,34 +1775,29 @@ components: nullable: true description: > Whether `feed_info_window` agrees with `service_window` on both bounds. Null when either window is missing, which is not the same as a mismatch. - example: true previous_dataset_id: type: string nullable: true description: > Stable identifier of the dataset downloaded immediately before this one. Null for the oldest dataset of the feed. Populated even when that dataset falls outside the requested page or date range, so overlap is never reported as absent merely because of paging. - example: mdb-123-202604290029 overlap_days: type: integer nullable: true description: > Days of overlap between this dataset's coverage window and that of the dataset immediately older than it. Zero means the windows meet exactly; a gap is reported as `gap_days` instead. Null when either window is missing or there is no older dataset. - example: 15 gap_days: type: integer nullable: true description: > Days of uncovered service between the end of the older dataset's window and the start of this one. Null when the windows overlap or meet, which is the passing case. - example: 3 files: type: array description: > The files the calculation reads, and whether each was present in this dataset. Always returned in the same order with one entry per file, so a client can render a fixed row. - items: $ref: "#/components/schemas/GtfsFeedContinuousCoverageFile" GtfsFeedContinuousCoverageFile: @@ -1944,7 +1846,6 @@ components: hosted_url: description: > As a convenience, the URL of the latest uploaded dataset hosted by MobilityData. It should be the same URL as the one found in the latest dataset id dataset. An alternative way to find this is to use the latest dataset id to obtain the dataset and then use its hosted_url. - type: string format: url example: https://storage.googleapis.com/mobilitydata-datasets-prod/mdb-1210/mdb-1210-202402121801/mdb-1210-202402121801.zip @@ -2074,14 +1975,12 @@ components: producer_url: description: > URL where the producer is providing the dataset. Refer to the authentication information to know how to access this URL. - type: string format: url example: https://ladotbus.com/gtfs is_producer_url_unstable: description: > Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time. This may be because the URL contains a date/time, or because the transit provider has communicated that it is not permanent (e.g. it is updated monthly). - * true - The producer URL is unstable and changes over time. * false - The producer URL is stable and unchanging over time. * null (default) - There is not enough information to determine the stability of the producer URL. @@ -2091,12 +1990,10 @@ components: authentication_type: description: > Defines the type of authentication required to access the `producer_url`. Valid values for this field are: - * 0 or (empty) - No authentication required. * 1 - The authentication requires an API key, which should be passed as value of the parameter api_key_parameter_name in the URL. Please visit URL in authentication_info_url for more information. * 2 - The authentication requires an HTTP header, which should be passed as the value of the header api_key_parameter_name in the HTTP request. When not provided, the authentication type is assumed to be 0. - type: integer enum: - 0 @@ -2106,7 +2003,6 @@ components: authentication_info_url: description: > Contains a URL to a human-readable page describing how the authentication should be performed and how credentials can be created. This field is required for `authentication_type=1` and `authentication_type=2`. - type: string format: url example: https://apidevelopers.ladottransit.com @@ -2114,7 +2010,6 @@ components: type: string description: > Defines the name of the parameter to pass in the URL to provide the API key. This field is required for `authentication_type=1` and `authentication_type=2`. - example: Ocp-Apim-Subscription-Key license_url: description: A URL where to find the license for the feed. @@ -2151,7 +2046,6 @@ components: country_code: description: > ISO 3166-1 alpha-2 code designating the country where the system is located. For a list of valid codes [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). - type: string example: US country: @@ -2161,7 +2055,6 @@ components: subdivision_name: description: > ISO 3166-2 english subdivision name designating the subdivision (e.g province, state, region) where the system is located. For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). - type: string example: California municipality: @@ -2205,7 +2098,6 @@ components: type: string description: > The type of location: `country` (has an ISO 3166-1 code), `subdivision` (has an ISO 3166-2 code) or `municipality` (a locality below the subdivision level). - enum: - country - subdivision @@ -2509,7 +2401,6 @@ components: match_type: description: > Type of match performed. One of: - - 'exact': Direct match found in database - 'heuristic': Matched via pattern-based rules (CC resolver, common patterns) - 'fuzzy': Similarity-based match against same-host licenses @@ -2518,7 +2409,6 @@ components: confidence: description: > Match confidence score (0.0-1.0), examples: - - 1.0: Exact match - 0.99: Creative Commons resolved - 0.95: Pattern heuristic match @@ -2540,7 +2430,6 @@ components: matched_source: description: > Source of the match. Examples: - - 'db.license': Exact match from database - 'cc-resolver': Creative Commons license resolver - 'pattern-heuristics': Generic pattern matching @@ -2553,7 +2442,6 @@ components: regional_id: description: > Regional/jurisdictional variant identifier for ported licenses - (e.g., 'CC-BY-2.1-jp' for Japan-ported Creative Commons) type: string example: CC-BY-4.0-nl @@ -2613,13 +2501,11 @@ components: dry_run: description: > When true (default), compute and return the affected feeds without persisting any changes. Set to false to apply the changes. - type: boolean default: true override: description: > When false (default), only feeds whose license_id is currently unset (null) are updated. When true, feeds that already have a license_id are also updated. - type: boolean default: false PropagateLicenseResponse: @@ -2671,7 +2557,6 @@ components: feed_name: description: > An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. - type: string example: Bus note: @@ -2704,13 +2589,11 @@ components: default: false description: > Whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Defaults to false. - locations: $ref: "#/components/schemas/Locations" related_links: description: > A list of related links for the feed. - type: array items: $ref: '#/components/schemas/FeedRelatedLink' @@ -2719,7 +2602,6 @@ components: default: false description: > When true, after the feed is created, propagate its license_id to all other feeds sharing the same normalized license_url where license_id is currently unset. - required: - source_info - operational_status @@ -2738,7 +2620,6 @@ components: feed_name: description: > An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. - type: string example: Bus note: @@ -2771,7 +2652,6 @@ components: default: false description: > Whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Defaults to false. - entity_types: type: array minItems: 1 @@ -2784,11 +2664,6 @@ components: example: vp description: > The type of realtime entry: - - - - - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2803,7 +2678,6 @@ components: related_links: description: > A list of related links for the feed. - type: array items: $ref: '#/components/schemas/FeedRelatedLink' @@ -2812,7 +2686,6 @@ components: default: false description: > When true, after the feed is created, propagate its license_id to all other feeds sharing the same normalized license_url where license_id is currently unset. - required: - source_info - operational_status @@ -2886,7 +2759,6 @@ components: feed_name: description: > An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. - type: string example: Bus note: @@ -2913,11 +2785,6 @@ components: example: vp description: > The type of realtime entry: - - - - - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2927,28 +2794,23 @@ components: items: type: string example: "mdb-20" - operational_status_action: + operational_status: type: string - enum: - - no_change - - wip - - published - - unpublished + enum: [wip, published, unpublished] + description: > + Current operational status of the feed. Omit this field to leave the stored value unchanged; send a value to set it explicitly. Mirrors the `operational_status` returned by the feed GET endpoints, so a response can be sent straight back as an update with no change detected. official: type: boolean description: Whether this is an official feed. seasonal: type: boolean - default: false description: > - Whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Defaults to false. - + Whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Omit this field to leave the stored value unchanged; send true or false to set it explicitly. propagate_license: type: boolean default: false description: > When true, after the feed is updated, propagate its license_id to all other feeds sharing the same normalized license_url where license_id is currently unset. - required: - id - status @@ -2972,7 +2834,6 @@ components: feed_name: description: > An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. - type: string example: Bus note: @@ -2988,28 +2849,23 @@ components: type: array items: $ref: "#/components/schemas/Redirect" - operational_status_action: + operational_status: type: string - enum: - - no_change - - wip - - published - - unpublished + enum: [wip, published, unpublished] + description: > + Current operational status of the feed. Omit this field to leave the stored value unchanged; send a value to set it explicitly. Mirrors the `operational_status` returned by the feed GET endpoints, so a response can be sent straight back as an update with no change detected. official: type: boolean description: Whether this is an official feed. seasonal: type: boolean - default: false description: > - Whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Defaults to false. - + Whether the feed is seasonal, i.e. it only provides service during recurring periods of the year (for example a summer-only or winter-only service). Omit this field to leave the stored value unchanged; send true or false to set it explicitly. propagate_license: type: boolean default: false description: > When true, after the feed is updated, propagate its license_id to all other feeds sharing the same normalized license_url where license_id is currently unset. - required: - id - status @@ -3017,11 +2873,6 @@ components: x-operation: true description: > Describes status of the Feed. Should be one of - - - - - * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -3039,11 +2890,6 @@ components: x-operation: true description: > Describes data type of a feed. Should be one of - - - - - * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. @@ -3057,7 +2903,6 @@ components: x-operation: true description: > The value of a feature flag. The type is determined by the flag's value_type field. Possible shapes: boolean (true/false), string, number, array, or object. - FeatureFlagAssignment: x-operation: true type: object @@ -3135,7 +2980,6 @@ components: type: boolean description: > When true, the flag is hidden from the user-facing API. It remains visible and manageable through the Operations API. - default: false CreateFeatureFlagRequest: x-operation: true @@ -3168,7 +3012,6 @@ components: type: boolean description: > When true, the flag is hidden from the user-facing API. It remains visible and manageable through the Operations API. - default: false UpdateFeatureFlagRequest: x-operation: true @@ -3189,7 +3032,6 @@ components: type: boolean description: > When true, the flag is hidden from the user-facing API. It remains visible and manageable through the Operations API. - EarlyAccessProgramFeatureFlagGrant: x-operation: true type: object @@ -3230,7 +3072,6 @@ components: type: integer description: > Days an unclaimed invited email is kept before the purge task deletes it. - default: 90 created_at: type: string @@ -3266,7 +3107,6 @@ components: type: array description: > The feature flags this program grants, written atomically with the program row. A program created with no entries here is valid but grants nothing. - items: $ref: "#/components/schemas/EarlyAccessProgramFeatureFlagGrant" default: [] @@ -3275,7 +3115,6 @@ components: type: object description: > The program's own fields are a partial update - omit one to leave it unchanged. `feature_flags` is the exception: it is a full replace, so omitting it or sending an empty array removes every grant this program makes going forward. - properties: name: type: string @@ -3290,7 +3129,6 @@ components: type: array description: > The program's complete set of feature flag grants after this call - a "set" operation, not an incremental add/remove. Omitting this field, or sending an empty array, removes all of this program's grants going forward without touching flags already granted to existing enrollees. - items: $ref: "#/components/schemas/EarlyAccessProgramFeatureFlagGrant" default: [] @@ -3304,7 +3142,6 @@ components: type: array description: > Addresses to invite. Lower-cased and de-duplicated before matching. Capped at 5000 — larger submissions are rejected with 422 rather than truncated. - items: type: string format: email @@ -3313,14 +3150,12 @@ components: type: boolean description: > When true (default), compute and return the categorisation without persisting any changes. Set to false to apply — grant matched accounts immediately and create invite rows for the rest. - default: true EarlyAccessImportCategorySummary: x-operation: true type: object description: > A count plus a short sample of the addresses in this category. Only the `invalid` category is returned in full — see ImportEarlyAccessInvitedEmailsResponse. - required: - count properties: @@ -3369,7 +3204,6 @@ components: - $ref: "#/components/schemas/EarlyAccessImportCategorySummary" description: > Addresses with no matching account. An invite row is created on apply, claimed automatically the next time that email signs in. - already_invited: allOf: - $ref: "#/components/schemas/EarlyAccessImportCategorySummary" @@ -3382,7 +3216,6 @@ components: type: array description: > Addresses that were rejected, returned in full (not just a sample) so the operator can fix and resubmit them. - items: $ref: "#/components/schemas/EarlyAccessInvalidEmailEntry" RemoveEarlyAccessInvitedEmailsRequest: @@ -3410,13 +3243,11 @@ components: type: integer description: > Number of submitted addresses that had no outstanding invite for this program (already claimed, never invited, or already removed). - EarlyAccessReportRow: x-operation: true type: object description: > One person in a program's report. `status` says which half of the program they are in, and which of the remaining fields are populated. - required: - email - status @@ -3426,7 +3257,6 @@ components: format: email description: > For an enrolled row, the account's email as it is now; for an invited row, the address the invite was created for. - status: type: string enum: [enrolled, invited] @@ -3528,7 +3358,6 @@ components: type: array description: > All feature flags with their resolved state for this user. Every flag is returned (including those the user has no explicit override for); user_value holds the per-user override when set, otherwise the flag's default_value applies. - items: $ref: "#/components/schemas/UserFeatureFlagState" default: [] @@ -3555,7 +3384,6 @@ components: in: query description: > Response format. `json` (the default) returns the paginated object; `csv` returns the same rows as `text/csv` with a header row. - required: False schema: type: string diff --git a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py index 89e01900f..c4520249d 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py @@ -95,6 +95,20 @@ # comparing it would report a phantom change whenever the request omits it. _DERIVED_SOURCE_INFO_FIELDS: Final[tuple[str, ...]] = ("license_is_spdx",) +# Tri-state fields: an omitted value means "preserve the stored one", and `to_orm` skips +# them, so the diff must not report a phantom change. The inverse case of +# _DERIVED_SOURCE_INFO_FIELDS: those are readable but not settable, these are settable but +# not clearable by omission. Without this, a request that never mentions one of them would +# force the write branch of `_update_feed` -- a 200 instead of a 204, plus a needless +# materialized view refresh and web revalidation task. +# +# This is what lets a feed GET response be sent straight back as an update with no change +# detected: absence means "unchanged" for every field that has no other way to say it. +_PRESERVE_WHEN_OMITTED_FIELDS: Final[tuple[str, ...]] = ( + "seasonal", + "operational_status", +) + def _normalize_for_diff(value): """Recursively coerce "absent" representations to None so change detection mirrors @@ -311,15 +325,15 @@ def detect_changes( ) -> DeepDiff: """Detect changes between the feed and the update request.""" copy_feed = impl_class.from_orm(feed) - copy_feed.operational_status_action = ( - update_request_feed.operational_status_action - ) current_values = _strip_derived_fields( _normalize_for_diff(copy_feed.model_dump()) ) requested_values = _strip_derived_fields( _normalize_for_diff(update_request_feed.model_dump()) ) + for field in _PRESERVE_WHEN_OMITTED_FIELDS: + if requested_values.get(field) is None: + requested_values[field] = current_values.get(field) diff = DeepDiff( current_values, requested_values, @@ -393,10 +407,9 @@ def _update_feed( else UpdateRequestGtfsRtFeedImpl ) diff = self.detect_changes(feed_from_db, update_request_feed, impl_class) - if len(diff.affected_paths) > 0 or ( - update_request_feed.operational_status_action is not None - and update_request_feed.operational_status_action != "no_change" - ): + # Every settable field, `operational_status` included, is visible to the diff, so + # this is the single gate: write exactly when something actually changed. + if len(diff.affected_paths) > 0: # Capture pre-mutation state for notification events (before to_orm mutates the object). old_producer_url = getattr(feed_from_db, "producer_url", None) old_redirect_target_ids = { @@ -499,15 +512,6 @@ def _update_feed( @staticmethod def _populate_feed_values(feed, impl_class, session, update_request_feed): impl_class.to_orm(update_request_feed, feed, session) - action = update_request_feed.operational_status_action - # This is a temporary solution as the operational_status is not visible in the diff - if action is not None and not action.lower() == "no_change": - if action.lower() == "wip": - feed.operational_status = "wip" - elif action.lower() == "published": - feed.operational_status = "published" - elif action.lower() == "unpublished": - feed.operational_status = "unpublished" session.add(feed) @staticmethod diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py index 4c05bd22f..9ad660d5b 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py @@ -75,6 +75,7 @@ def from_orm(cls, obj: Gtfsfeed | None) -> UpdateRequestGtfsFeed | None: ), official=obj.official, seasonal=obj.seasonal, + operational_status=obj.operational_status, ) @classmethod @@ -90,7 +91,15 @@ def to_orm( entity.note = update_request.note entity.feed_contact_email = update_request.feed_contact_email entity.official = update_request.official - entity.seasonal = update_request.seasonal + # Tri-state, matching the catalog CSV's empty cell (populate_db_gtfs.py): an omitted + # `seasonal` means "leave the stored value alone", so only an explicit true/false + # writes. A feed marked seasonal by hand must survive an update request from a client + # whose spec predates the field. + if update_request.seasonal is not None: + entity.seasonal = update_request.seasonal + # Tri-state as well: omitted means "leave the operational status alone". + if update_request.operational_status is not None: + entity.operational_status = update_request.operational_status entity.producer_url = ( None if ( diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py index ebf7a5873..7a38d5a3e 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py @@ -87,6 +87,7 @@ def from_orm(cls, obj: Gtfsrealtimefeed | None) -> UpdateRequestGtfsRtFeed | Non feed_references=sorted([item.stable_id for item in obj.gtfs_feeds]), official=obj.official, seasonal=obj.seasonal, + operational_status=obj.operational_status, ) @classmethod @@ -102,7 +103,15 @@ def to_orm( entity.note = update_request.note entity.feed_contact_email = update_request.feed_contact_email entity.official = update_request.official - entity.seasonal = update_request.seasonal + # Tri-state, matching the catalog CSV's empty cell (populate_db_gtfs.py): an omitted + # `seasonal` means "leave the stored value alone", so only an explicit true/false + # writes. A feed marked seasonal by hand must survive an update request from a client + # whose spec predates the field. + if update_request.seasonal is not None: + entity.seasonal = update_request.seasonal + # Tri-state as well: omitted means "leave the operational status alone". + if update_request.operational_status is not None: + entity.operational_status = update_request.operational_status entity.producer_url = ( None if ( diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py index ebc59e883..f4e27087e 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py @@ -123,7 +123,10 @@ def test_to_orm_invalid_source_info(): result = UpdateRequestGtfsFeedImpl.to_orm(update_request, entity, session) assert result.producer_url is None assert result.is_producer_url_unstable is None - assert result.seasonal is False + # The request omits `seasonal`, so to_orm leaves the attribute alone. This entity was + # never persisted, so the NOT NULL server default has not applied yet -- None here is + # the preserve path, not a stored value. + assert result.seasonal is None assert result.authentication_type is None assert result.authentication_info_url is None assert result.api_key_parameter_name is None diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py index f275912a7..c79fdc39a 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py @@ -35,6 +35,7 @@ def test_from_orm(): license_url="http://license.url", redirectingids=[redirecting_id], externalids=[external_id], + seasonal=True, ) result = UpdateRequestGtfsRtFeedImpl.from_orm(gtfs_feed) @@ -48,6 +49,7 @@ def test_from_orm(): assert result.source_info.authentication_type == 1 assert result.source_info.authentication_info_url == "http://auth.info.url" assert result.source_info.api_key_parameter_name == "api_key" + assert result.seasonal is True assert result.source_info.license_url == "http://license.url" assert len(result.redirects) == 1 assert result.redirects[0].target_id == "target_stable_id" @@ -79,6 +81,7 @@ def test_to_orm(): external_ids=[ExternalIdImpl(external_id="external_id")], entity_types=["vp"], feed_references=["feed_reference"], + seasonal=True, ) entity = Gtfsrealtimefeed(id="1", stable_id="stable_id", data_type="gtfs") target_feed = Gtfsfeed(id=2, stable_id="target_stable_id") @@ -98,6 +101,7 @@ def test_to_orm(): assert result.note == "note" assert result.feed_contact_email == "email@example.com" assert result.producer_url == "http://producer.url" + assert result.seasonal is True assert result.authentication_type == "1" assert result.authentication_info_url == "http://auth.info.url" assert result.api_key_parameter_name == "api_key" diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py b/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py index 42179e85d..ef9a072a5 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py @@ -45,7 +45,6 @@ def update_request_gtfs_feed(): license_url=feed_mdb_40.license_url, ), redirects=[], - operational_status_action="no_change", official=True, ) diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py b/functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py index 14e0d7626..fcb9e8a77 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py @@ -32,7 +32,6 @@ def _make_request(source_info: SourceInfo, **overrides) -> UpdateRequestGtfsFeed source_info=source_info, redirects=[], external_ids=[], - operational_status_action="no_change", official=True, ) payload.update(overrides) @@ -110,6 +109,65 @@ def test_detect_changes_detects_cleared_list(): assert diff.affected_paths +def test_detect_changes_ignores_omitted_seasonal(): + """An omitted `seasonal` means preserve, so it is not a change. + + `to_orm` skips a None `seasonal`, so reporting it here would push `_update_feed` down + the write branch for nothing: a 200 instead of a 204, plus a materialized view refresh + and a web revalidation task on an update that changes nothing. + """ + source_info = SourceInfo(producer_url="https://example.com/feed") + current = _make_request(source_info, seasonal=True) + requested = _make_request(source_info) # client's spec predates the field + + assert requested.seasonal is None + diff = _detect(current, requested) + + assert not diff.affected_paths + + +def test_detect_changes_detects_explicitly_cleared_seasonal(): + """An explicit false is a real edit and must still be reported, unlike an omission.""" + source_info = SourceInfo(producer_url="https://example.com/feed") + current = _make_request(source_info, seasonal=True) + requested = _make_request(source_info, seasonal=False) + + diff = _detect(current, requested) + + assert diff.affected_paths + + +def test_detect_changes_ignores_omitted_operational_status(): + """Omitting `operational_status` means preserve, so it is not a change. + + Replaces the old `operational_status_action="no_change"` sentinel, which sat outside the + diff entirely. + """ + source_info = SourceInfo(producer_url="https://example.com/feed") + current = _make_request(source_info, operational_status="published") + requested = _make_request(source_info) + + assert requested.operational_status is None + diff = _detect(current, requested) + + assert not diff.affected_paths + + +def test_detect_changes_detects_operational_status_change(): + """A real status change is reported by the diff rather than bypassing it. + + It used to be invisible to change detection and applied by a special case in + `_populate_feed_values`; now the diff is the single gate on whether a write happens. + """ + source_info = SourceInfo(producer_url="https://example.com/feed") + current = _make_request(source_info, operational_status="wip") + requested = _make_request(source_info, operational_status="published") + + diff = _detect(current, requested) + + assert diff.affected_paths + + def test_normalize_for_diff_coerces_empty_values(): normalized = _normalize_for_diff( { diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py index 03d51c21a..374fe65e2 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py @@ -35,7 +35,6 @@ def update_request_gtfs_feed(): license_is_spdx=True, ), redirects=[], - operational_status_action="no_change", official=True, ) @@ -87,7 +86,7 @@ async def test_update_gtfs_feed_field_change( async def test_update_gtfs_feed_set_wip( mock_revalidation, update_request_gtfs_feed, db_session ): - update_request_gtfs_feed.operational_status_action = "wip" + update_request_gtfs_feed.operational_status = "wip" api = OperationsApiImpl() response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 @@ -102,8 +101,15 @@ async def test_update_gtfs_feed_set_wip( @pytest.mark.asyncio @pytest.mark.usefixtures("update_request_gtfs_feed", "db_session") -async def test_update_gtfs_feed_set_wip_nochange(update_request_gtfs_feed, db_session): - update_request_gtfs_feed.operational_status_action = "no_change" +async def test_update_gtfs_feed_omitted_operational_status_is_preserved( + update_request_gtfs_feed, db_session +): + """Omitting `operational_status` leaves the stored value alone and reports no change. + + This replaces the old `operational_status_action="no_change"` sentinel: absence now + carries that meaning, the same tri-state contract `seasonal` uses. + """ + assert update_request_gtfs_feed.operational_status is None api = OperationsApiImpl() response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 204 @@ -122,7 +128,7 @@ async def test_update_gtfs_feed_set_wip_nochange(update_request_gtfs_feed, db_se async def test_update_gtfs_feed_set_published( mock_revalidation, update_request_gtfs_feed, db_session ): - update_request_gtfs_feed.operational_status_action = "published" + update_request_gtfs_feed.operational_status = "published" api = OperationsApiImpl() response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 @@ -141,7 +147,7 @@ async def test_update_gtfs_feed_set_published( async def test_update_gtfs_feed_set_unpublished( mock_revalidation, update_request_gtfs_feed, db_session ): - update_request_gtfs_feed.operational_status_action = "unpublished" + update_request_gtfs_feed.operational_status = "unpublished" api = OperationsApiImpl() response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 @@ -178,3 +184,77 @@ async def test_update_gtfs_feed_official_field(update_request_gtfs_feed, db_sess .one() ) assert db_feed.official is True + + +@pytest.mark.asyncio +@patch("feeds_operations.impl.feeds_operations_impl.create_web_revalidation_task") +async def test_update_gtfs_feed_seasonal_field( + mock_revalidation, update_request_gtfs_feed, db_session +): + """An explicit `seasonal` in the request is persisted.""" + # Establish a known pre-state so toggling `seasonal` to True is a genuine change + # regardless of test ordering (the row is shared across this module). + seeded_feed = ( + db_session.query(Gtfsfeed) + .filter(Gtfsfeed.stable_id == feed_mdb_40.stable_id) + .one() + ) + seeded_feed.seasonal = False + db_session.commit() + + update_request_gtfs_feed.seasonal = True + api = OperationsApiImpl() + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) + assert response.status_code == 200 + + db_session.expire_all() + db_feed = ( + db_session.query(Gtfsfeed) + .filter(Gtfsfeed.stable_id == feed_mdb_40.stable_id) + .one() + ) + assert db_feed.seasonal is True + + +@pytest.mark.asyncio +@patch("feeds_operations.impl.feeds_operations_impl.create_web_revalidation_task") +async def test_update_gtfs_feed_omitted_seasonal_is_preserved( + mock_revalidation, update_request_gtfs_feed, db_session +): + """A request that never mentions `seasonal` must not clear it. + + Clients generated from a spec predating the field send no `seasonal` at all. While the + property carried `default: false`, that omission reset an operator-set flag on the next + edit of any other field -- which would silently un-mark the TDG/ODPT/JBDA feeds this + issue exists to mark. The no-phantom-change half of the fix is pinned in + test_detect_changes.py, which does not depend on this shared row's state. + """ + seeded_feed = ( + db_session.query(Gtfsfeed) + .filter(Gtfsfeed.stable_id == feed_mdb_40.stable_id) + .one() + ) + seeded_feed.seasonal = True + db_session.commit() + + # Drive an unrelated edit by making the STORED note stale, rather than by changing the + # request. The write then restores `note` to its fixture value, so this test leaves the + # module-shared row exactly as it found it (sibling tests assert on feed_name/provider). + seeded_feed.note = "stale note" + db_session.commit() + + # The fixture never sets `seasonal`; that is exactly the request shape under test. + assert update_request_gtfs_feed.seasonal is None + + api = OperationsApiImpl() + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) + assert response.status_code == 200 + + db_session.expire_all() + db_feed = ( + db_session.query(Gtfsfeed) + .filter(Gtfsfeed.stable_id == feed_mdb_40.stable_id) + .one() + ) + assert db_feed.note == feed_mdb_40.note + assert db_feed.seasonal is True diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py index 80d06d104..19afb09d5 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py @@ -32,7 +32,6 @@ def update_request_gtfs_rt_feed(): license_url=feed_mdb_41.license_url, ), redirects=[], - operational_status_action="no_change", entity_types=["vp"], official=True, ) @@ -94,7 +93,7 @@ async def test_update_gtfs_feed_static_change( async def test_update_gtfs_rt_feed_set_wip( mock_revalidation, update_request_gtfs_rt_feed, db_session ): - update_request_gtfs_rt_feed.operational_status_action = "wip" + update_request_gtfs_rt_feed.operational_status = "wip" api = OperationsApiImpl() response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 @@ -113,7 +112,7 @@ async def test_update_gtfs_rt_feed_set_wip( async def test_update_gtfs_rt_feed_set_published( mock_revalidation, update_request_gtfs_rt_feed, db_session ): - update_request_gtfs_rt_feed.operational_status_action = "published" + update_request_gtfs_rt_feed.operational_status = "published" api = OperationsApiImpl() response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 @@ -132,7 +131,7 @@ async def test_update_gtfs_rt_feed_set_published( async def test_update_gtfs_rt_feed_set_unpublished( mock_revalidation, update_request_gtfs_rt_feed, db_session ): - update_request_gtfs_rt_feed.operational_status_action = "unpublished" + update_request_gtfs_rt_feed.operational_status = "unpublished" api = OperationsApiImpl() response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 @@ -173,3 +172,68 @@ async def test_update_gtfs_rt_feed_official_field( .one() ) assert db_feed.official is True + + +@pytest.mark.asyncio +@patch("feeds_operations.impl.feeds_operations_impl.create_web_revalidation_task") +async def test_update_gtfs_rt_feed_seasonal_field( + mock_revalidation, update_request_gtfs_rt_feed, db_session +): + """An explicit `seasonal` in the request is persisted.""" + seeded_feed = ( + db_session.query(Gtfsrealtimefeed) + .filter(Gtfsrealtimefeed.stable_id == feed_mdb_41.stable_id) + .one() + ) + seeded_feed.seasonal = False + db_session.commit() + + update_request_gtfs_rt_feed.seasonal = True + api = OperationsApiImpl() + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + assert response.status_code == 200 + + db_session.expire_all() + db_feed = ( + db_session.query(Gtfsrealtimefeed) + .filter(Gtfsrealtimefeed.stable_id == feed_mdb_41.stable_id) + .one() + ) + assert db_feed.seasonal is True + + +@pytest.mark.asyncio +@patch("feeds_operations.impl.feeds_operations_impl.create_web_revalidation_task") +async def test_update_gtfs_rt_feed_omitted_seasonal_is_preserved( + mock_revalidation, update_request_gtfs_rt_feed, db_session +): + """A request that never mentions `seasonal` must not clear it.""" + seeded_feed = ( + db_session.query(Gtfsrealtimefeed) + .filter(Gtfsrealtimefeed.stable_id == feed_mdb_41.stable_id) + .one() + ) + seeded_feed.seasonal = True + db_session.commit() + + # Drive an unrelated edit by making the STORED note stale, rather than by changing the + # request. The write then restores `note` to its fixture value, so this test leaves the + # module-shared row exactly as it found it (sibling tests assert on feed_name/provider). + seeded_feed.note = "stale note" + db_session.commit() + + # The fixture never sets `seasonal`; that is exactly the request shape under test. + assert update_request_gtfs_rt_feed.seasonal is None + + api = OperationsApiImpl() + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + assert response.status_code == 200 + + db_session.expire_all() + db_feed = ( + db_session.query(Gtfsrealtimefeed) + .filter(Gtfsrealtimefeed.stable_id == feed_mdb_41.stable_id) + .one() + ) + assert db_feed.note == feed_mdb_41.note + assert db_feed.seasonal is True diff --git a/functions-python/tasks_executor/src/tasks/data_import/data_import_utils.py b/functions-python/tasks_executor/src/tasks/data_import/data_import_utils.py index 5454c1e97..b8f2bbb99 100644 --- a/functions-python/tasks_executor/src/tasks/data_import/data_import_utils.py +++ b/functions-python/tasks_executor/src/tasks/data_import/data_import_utils.py @@ -2,7 +2,7 @@ import os import uuid from datetime import datetime -from typing import Tuple, Type, TypeVar, Optional +from typing import Final, Tuple, Type, TypeVar, Optional from sqlalchemy import select from sqlalchemy.exc import IntegrityError @@ -20,6 +20,13 @@ logger = logging.getLogger(__name__) T = TypeVar("T", bound="Feed") +# Columns an operator sets by hand through the Operations API. No upstream source (TDG, +# ODPT, JBDA) carries a seasonality signal, so the importers must never assign these -- the +# only correct behaviour on every run is to leave the stored value alone, exactly as the +# catalog CSV treats an empty `is_seasonal` cell. Enforced by the per-importer +# `test_seasonal_survives_reimport` tests. +OPERATOR_OWNED_FEED_COLUMNS: Final[tuple[str, ...]] = ("seasonal",) + def get_or_create_entity_type(session: Session, entity_type_name: str) -> Entitytype: """Get or create an Entitytype by name.""" diff --git a/functions-python/tasks_executor/src/tasks/data_import/jbda/import_jbda_feeds.py b/functions-python/tasks_executor/src/tasks/data_import/jbda/import_jbda_feeds.py index 6a5783920..1467e35a2 100644 --- a/functions-python/tasks_executor/src/tasks/data_import/jbda/import_jbda_feeds.py +++ b/functions-python/tasks_executor/src/tasks/data_import/jbda/import_jbda_feeds.py @@ -135,6 +135,9 @@ def _update_common_feed_fields( feed: Feed, list_item: dict, detail: dict, producer_url: str ) -> None: """Update common fields of a Feed (Gtfsfeed or Gtfsrealtimefeed) from JBDA list item and detail.""" + # Only import-owned fields belong here: never assign anything in + # OPERATOR_OWNED_FEED_COLUMNS (data_import_utils), which an operator sets by hand + # and this source does not carry. logger.debug( "Updating common fields for feed id=%s stable_id=%s", getattr(feed, "id", None), diff --git a/functions-python/tasks_executor/src/tasks/data_import/odpt/import_odpt_feeds.py b/functions-python/tasks_executor/src/tasks/data_import/odpt/import_odpt_feeds.py index 64369cd01..f7858e52f 100644 --- a/functions-python/tasks_executor/src/tasks/data_import/odpt/import_odpt_feeds.py +++ b/functions-python/tasks_executor/src/tasks/data_import/odpt/import_odpt_feeds.py @@ -172,6 +172,9 @@ def _get_license_url(license_type: Optional[str]) -> Optional[str]: def _update_common_feed_fields(feed: Feed, item: dict, producer_url: str) -> None: """Update common fields of a Feed (Gtfsfeed or Gtfsrealtimefeed) from an ODPT feed item.""" + # Only import-owned fields belong here: never assign anything in + # OPERATOR_OWNED_FEED_COLUMNS (data_import_utils), which an operator sets by hand + # and this source does not carry. logger.debug( "Updating common fields for feed id=%s stable_id=%s", getattr(feed, "id", None), diff --git a/functions-python/tasks_executor/src/tasks/data_import/transportdatagouv/import_tdg_feeds.py b/functions-python/tasks_executor/src/tasks/data_import/transportdatagouv/import_tdg_feeds.py index 5be00faa2..82917cfab 100644 --- a/functions-python/tasks_executor/src/tasks/data_import/transportdatagouv/import_tdg_feeds.py +++ b/functions-python/tasks_executor/src/tasks/data_import/transportdatagouv/import_tdg_feeds.py @@ -262,6 +262,11 @@ def _delete_and_recreate_feed_if_type_changed( db_session.query(Feed).filter(Feed.stable_id == stable_id).one_or_none() ) + # Operator-owned columns (OPERATOR_OWNED_FEED_COLUMNS) are not in any TDG payload, so + # nothing upstream would ever set them again. Carry them across the delete/recreate by + # hand or a data_type flip silently clears them. + preserved_seasonal = None + if existing is not None and existing.data_type != feed_type: logger.info( "TDG feed type changed for stable_id=%s: db_data_type=%s -> new_data_type=%s. Deleting and recreating.", @@ -269,6 +274,7 @@ def _delete_and_recreate_feed_if_type_changed( getattr(existing, "data_type", None), feed_type, ) + preserved_seasonal = existing.seasonal db_session.delete(existing) # flush so the new insert doesn't collide on stable_id unique constraint db_session.flush() @@ -280,6 +286,13 @@ def _delete_and_recreate_feed_if_type_changed( feed_type, **get_or_create_kwargs, ) + if preserved_seasonal is not None: + logger.info( + "Carrying seasonal=%s onto recreated TDG feed stable_id=%s", + preserved_seasonal, + stable_id, + ) + feed.seasonal = preserved_seasonal return feed, is_new @@ -348,6 +361,9 @@ def _update_common_tdg_fields( Update common fields for both schedule GTFS and RT from TDG dataset + resource. Assumes required fields were validated earlier. """ + # Only import-owned fields belong here: never assign anything in + # OPERATOR_OWNED_FEED_COLUMNS (data_import_utils), which an operator sets by hand + # and this source does not carry. feed.feed_name = dataset.get("title") feed.provider = (dataset.get("publisher") or {}).get("name") feed.producer_url = producer_url diff --git a/functions-python/tasks_executor/tests/tasks/data_import/test_jbda_import.py b/functions-python/tasks_executor/tests/tasks/data_import/test_jbda_import.py index c77c4de98..90ae29cb3 100644 --- a/functions-python/tasks_executor/tests/tasks/data_import/test_jbda_import.py +++ b/functions-python/tasks_executor/tests/tasks/data_import/test_jbda_import.py @@ -1,5 +1,6 @@ import os import unittest +import uuid from typing import Any, Dict, List from unittest.mock import patch, MagicMock @@ -8,6 +9,7 @@ from test_shared.test_utils.database_utils import default_db_url from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( + Feed, Gtfsfeed, Gtfsrealtimefeed, Feedrelatedlink, @@ -131,6 +133,49 @@ def get(self, url, timeout=60): return _FakeResponse({}, 404) +class _FakeSessionSeasonal: + """ + A single valid feed in its own org/feed namespace, with no RT urls, used by the + `seasonal` preservation test. The feed name differs from the stale one seeded into the + DB so the schedule fingerprint cannot match and the importer must take its update path. + """ + + FEEDS_URL = "https://api.gtfs-data.jp/v2/feeds" + DETAIL_TMPL = "https://api.gtfs-data.jp/v2/organizations/{org_id}/feeds/{feed_id}" + ORG_ID = "orgseason" + FEED_ID = "feedseason" + + def get(self, url, timeout=60): + if url == self.FEEDS_URL: + return _FakeResponse( + { + "body": [ + { + "organization_id": self.ORG_ID, + "feed_id": self.FEED_ID, + "organization_name": "Season Org", + "organization_email": "season@example.com", + "feed_pref_id": 1, + "feed_memo": "season memo", + } + ] + } + ) + if url == self.DETAIL_TMPL.format(org_id=self.ORG_ID, feed_id=self.FEED_ID): + return _FakeResponse( + { + "body": { + "organization_id": self.ORG_ID, + "feed_id": self.FEED_ID, + "feed_name": "Season Feed", + "feed_license_url": "https://license.example/season", + "real_time": {}, + } + } + ) + return _FakeResponse({}, 404) + + class _FakeSessionError: def get(self, url, timeout=60): raise RuntimeError("network down") @@ -332,6 +377,72 @@ def test_import_http_failure_graceful(self, db_session: Session): self.assertEqual(out["linked_refs"], 0) self.assertEqual(out["total_processed_items"], 0) + @with_db_session(db_url=default_db_url) + def test_seasonal_survives_reimport(self, db_session: Session): + """`seasonal` is operator-owned, so a re-import must leave it alone. + + JBDA carries no seasonality signal, so if the importer ever wrote the column the + flag would be cleared on the next monthly run and the feed would silently start + failing the rolling 7-day coverage criterion again. + """ + stable_id = f"jbda-{_FakeSessionSeasonal.ORG_ID}-{_FakeSessionSeasonal.FEED_ID}" + current_url = ( + f"https://api.gtfs-data.jp/v2/organizations/{_FakeSessionSeasonal.ORG_ID}" + f"/feeds/{_FakeSessionSeasonal.FEED_ID}/files/feed.zip?rid=current" + ) + + def _head_side_effect(url, allow_redirects=True, timeout=15): + return _FakeResponse(status=200 if url == current_url else 404) + + try: + db_session.add( + Gtfsfeed( + id=str(uuid.uuid4()), + stable_id=stable_id, + data_type="gtfs", + # Stale on purpose: feed_name is part of the schedule fingerprint, so + # this forces the update path. Without it the importer short-circuits + # on "no change detected" and the test would pass vacuously. + feed_name="Stale feed name", + seasonal=True, + ) + ) + db_session.commit() + + with patch( + "tasks.data_import.jbda.import_jbda_feeds.requests.Session", + return_value=_FakeSessionSeasonal(), + ), patch( + "tasks.data_import.jbda.import_jbda_feeds.requests.head", + side_effect=_head_side_effect, + ), patch( + "tasks.data_import.jbda.import_jbda_feeds.REQUEST_TIMEOUT_S", 0.01 + ), patch( + "tasks.data_import.data_import_utils.trigger_dataset_download", + MagicMock(), + ), patch( + "tasks.data_import.data_import_utils.create_web_revalidation_task", + MagicMock(), + ), patch.dict( + os.environ, + {"COMMIT_BATCH_SIZE": "1", "ENVIRONMENT": "test"}, + clear=False, + ): + import_jbda_handler({"dry_run": False}) + + db_session.expire_all() + feed = ( + db_session.query(Gtfsfeed).filter(Gtfsfeed.stable_id == stable_id).one() + ) + # Proves the importer really rewrote this row, so the assertion below is real. + self.assertEqual(feed.feed_name, "Season Feed") + self.assertTrue(feed.seasonal) + finally: + db_session.query(Feed).filter(Feed.stable_id == stable_id).delete( + synchronize_session=False + ) + db_session.commit() + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/data_import/test_odpt_import.py b/functions-python/tasks_executor/tests/tasks/data_import/test_odpt_import.py index 41c136325..58cddb67e 100644 --- a/functions-python/tasks_executor/tests/tasks/data_import/test_odpt_import.py +++ b/functions-python/tasks_executor/tests/tasks/data_import/test_odpt_import.py @@ -204,6 +204,37 @@ def get(self, url, timeout=60): return _FakeResponse([], 404) +class _FakeSessionSeasonal: + """ + One org/dataset in its own namespace, with no RT urls, used by the `seasonal` + preservation test. The dataset name differs from the stale one seeded into the DB so + the schedule fingerprint cannot match and the importer must take its update path. + """ + + def get(self, url, timeout=60): + if url == METADATA_URL_TMPL.format("ccby4", "gtfs"): + return _FakeResponse( + [ + { + "label": "SeasonOrg", + "name_ja": "シーズン組織", + "name_en": "Season Org", + "datasets": [ + { + "label": "season_dataset", + "name_ja": "シーズンデータセット", + "name_en": "Season Dataset", + "license_type": "CC BY 4.0", + } + ], + } + ] + ) + if url == METADATA_URL_TMPL.format("cc0", "gtfs"): + return _FakeResponse([]) + return _FakeResponse([], 404) + + class _FakeSessionEmpty: """Successful HTTP 200 responses that happen to contain no datasets at all.""" @@ -721,6 +752,58 @@ def test_import_http_failure_graceful(self, db_session: Session): self.assertEqual(out["linked_refs"], 0) self.assertEqual(out["total_processed_items"], 0) + @with_db_session(db_url=default_db_url) + def test_seasonal_survives_reimport(self, db_session: Session): + """`seasonal` is operator-owned, so a re-import must leave it alone. + + ODPT carries no seasonality signal, so if the importer ever wrote the column the + flag would be cleared on the next monthly run and the feed would silently start + failing the rolling 7-day coverage criterion again. + """ + stable_id = "odpt-SeasonOrg-season_dataset" + try: + _seed_feed( + db_session, + Gtfsfeed, + stable_id, + "gtfs", + # Stale on purpose: feed_name is part of the schedule fingerprint, so this + # forces the update path. Without it the importer short-circuits on + # "no change detected" and the test would pass vacuously. + feed_name="Stale dataset name", + seasonal=True, + ) + db_session.commit() + + with patch( + "tasks.data_import.odpt.import_odpt_feeds.requests.Session", + return_value=_FakeSessionSeasonal(), + ), patch( + "tasks.data_import.odpt.import_odpt_feeds.REQUEST_TIMEOUT_S", 0.01 + ), patch( + "tasks.data_import.data_import_utils.trigger_dataset_download", + MagicMock(), + ), patch( + "tasks.data_import.data_import_utils.create_web_revalidation_task", + MagicMock(), + ), patch( + "tasks.data_import.odpt.import_odpt_feeds.deprecate_stale_feeds", + MagicMock(return_value=[]), + ), patch.dict( + os.environ, {"ENVIRONMENT": "test"}, clear=False + ): + import_odpt_handler({"dry_run": False}) + + db_session.expire_all() + feed = ( + db_session.query(Gtfsfeed).filter(Gtfsfeed.stable_id == stable_id).one() + ) + # Proves the importer really rewrote this row, so the assertion below is real. + self.assertEqual(feed.feed_name, "シーズンデータセット") + self.assertTrue(feed.seasonal) + finally: + _delete_feeds_like(db_session, f"{stable_id}%") + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/data_import/test_tdg_import.py b/functions-python/tasks_executor/tests/tasks/data_import/test_tdg_import.py index 093d8c0e6..69642258d 100644 --- a/functions-python/tasks_executor/tests/tasks/data_import/test_tdg_import.py +++ b/functions-python/tasks_executor/tests/tasks/data_import/test_tdg_import.py @@ -1,5 +1,6 @@ import os import unittest +import uuid from typing import Any, Dict, Optional from unittest.mock import patch, MagicMock @@ -7,6 +8,7 @@ from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( + Feed, Gtfsfeed, Gtfsrealtimefeed, ) @@ -104,6 +106,51 @@ def head(self, url, allow_redirects=True, timeout=15): ) +class _FakeSessionSeasonal: + """ + One dataset with a single GTFS resource, under its own resource id so the rows it + touches never overlap the happy-path test's. Used by the `seasonal` preservation + tests: the dataset title differs from the stale one seeded into the DB, so the + schedule fingerprint cannot match and the importer is forced down its update path + instead of the "no change detected" early return. + """ + + TDG_DATASETS_URL = "https://transport.data.gouv.fr/api/datasets?format=gtfs" + GTFS_URL = "https://tdg.example/seasonal.zip" + + def get(self, url, timeout=60, headers=None): + if url == self.TDG_DATASETS_URL: + return _FakeResponse( + [ + { + "id": "ds-seasonal", + "title": "Seasonal Dataset", + "publisher": {"name": "TDG Seasonal Org"}, + "licence": "odc-odbl", + "resources": [ + { + "id": "res-seasonal", + "title": "Static GTFS", + "format": "GTFS", + "url": self.GTFS_URL, + "metadata": {"end_date": "2999-12-31"}, + } + ], + } + ] + ) + return _FakeResponse({}, status=404) + + def head(self, url, allow_redirects=True, timeout=15): + if url == self.GTFS_URL: + return _FakeResponse( + status=200, headers={"Content-Type": "application/zip"} + ) + return _FakeResponse( + status=200, headers={"Content-Type": "application/octet-stream"} + ) + + class _FakeSessionError: """ Fake HTTP session that always fails on GET. @@ -297,6 +344,111 @@ def test_import_http_failure_graceful(self, db_session: Session): self.assertEqual(out["created_rt"], 0) self.assertEqual(out["total_processed_items"], 0) + SEASONAL_STABLE_ID = "tdg-res-seasonal" + + def _run_seasonal_import(self): + """Run the importer against _FakeSessionSeasonal with side effects stubbed out.""" + with patch( + "tasks.data_import.transportdatagouv.import_tdg_feeds.requests.Session", + return_value=_FakeSessionSeasonal(), + ), patch( + "tasks.data_import.transportdatagouv.import_tdg_feeds.REQUEST_TIMEOUT_S", + 0.01, + ), patch( + "tasks.data_import.data_import_utils.trigger_dataset_download", + MagicMock(), + ), patch( + "tasks.data_import.data_import_utils.create_web_revalidation_task", + MagicMock(), + ), patch( + # This test is not about the stale sweep. Stub it so it cannot deprecate the + # tdg- rows the happy-path test committed into the shared session-scoped DB. + "tasks.data_import.transportdatagouv.import_tdg_feeds._deprecate_stale_feeds", + MagicMock(return_value=[]), + ), patch.dict( + os.environ, + {"COMMIT_BATCH_SIZE": "1", "ENVIRONMENT": "test"}, + clear=False, + ): + return import_tdg_handler({"dry_run": False}) + + @with_db_session(db_url=default_db_url) + def test_seasonal_survives_reimport(self, db_session: Session): + """`seasonal` is operator-owned, so a re-import must leave it alone. + + No TDG payload carries a seasonality signal, so if the importer ever wrote the + column the flag would be cleared on the next monthly run and the feed would silently + start failing the rolling 7-day coverage criterion again. + """ + try: + db_session.add( + Gtfsfeed( + id=str(uuid.uuid4()), + stable_id=self.SEASONAL_STABLE_ID, + data_type="gtfs", + # Stale on purpose: feed_name is part of the schedule fingerprint, so + # this forces the update path. Without it the importer short-circuits + # on "no change detected" and the test would pass vacuously. + feed_name="Stale dataset title", + seasonal=True, + ) + ) + db_session.commit() + + self._run_seasonal_import() + + db_session.expire_all() + feed = ( + db_session.query(Gtfsfeed) + .filter(Gtfsfeed.stable_id == self.SEASONAL_STABLE_ID) + .one() + ) + # Proves the importer really rewrote this row, so the assertion below is real. + self.assertEqual(feed.feed_name, "Seasonal Dataset") + self.assertTrue(feed.seasonal) + finally: + db_session.query(Feed).filter( + Feed.stable_id == self.SEASONAL_STABLE_ID + ).delete(synchronize_session=False) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def test_seasonal_survives_data_type_change(self, db_session: Session): + """A GTFS-RT -> GTFS flip deletes and recreates the row; `seasonal` must carry over. + + _delete_and_recreate_feed_if_type_changed is the one path in any importer that drops + operator-set columns, because the new row is built from scratch. + """ + try: + db_session.add( + Gtfsrealtimefeed( + id=str(uuid.uuid4()), + stable_id=self.SEASONAL_STABLE_ID, + data_type="gtfs_rt", + feed_name="Stale dataset title", + seasonal=True, + ) + ) + db_session.commit() + + self._run_seasonal_import() + + db_session.expire_all() + feed = ( + db_session.query(Feed) + .filter(Feed.stable_id == self.SEASONAL_STABLE_ID) + .one() + ) + # The row was recreated as a schedule feed... + self.assertEqual(feed.data_type, "gtfs") + # ...and the operator-set flag survived the delete/recreate. + self.assertTrue(feed.seasonal) + finally: + db_session.query(Feed).filter( + Feed.stable_id == self.SEASONAL_STABLE_ID + ).delete(synchronize_session=False) + db_session.commit() + if __name__ == "__main__": unittest.main() diff --git a/scripts/api-operations-update-schema.sh b/scripts/api-operations-update-schema.sh index b1aafa0bf..50f97ca60 100755 --- a/scripts/api-operations-update-schema.sh +++ b/scripts/api-operations-update-schema.sh @@ -64,5 +64,71 @@ yq -i ' ) ' "${DEST}" +# yq's emitter re-serializes the whole file and inserts blank lines into every +# block scalar it writes: one after the block, and - in folded (">") blocks - one +# before every more-indented line. The latter accumulates, growing the same +# description by one blank line on every run, so strip both here. Blank lines +# separating two equally-indented paragraphs are authored and round-trip cleanly, +# so they are left alone. +python3 - "${DEST}" <<'PY' +import re +import sys + +path = sys.argv[1] +with open(path, encoding="utf-8") as f: + lines = f.read().splitlines(keepends=True) + + +def indent_of(line): + return len(line) - len(line.lstrip(" ")) + + +# Matches a block scalar header: "key: >", "key: |2-", etc. The chomping +# indicator matters - "+" keeps trailing newlines, so those blocks are left as is. +header = re.compile(r":[ \t]*([|>])([0-9]*)([-+]?)[ \t]*$") + +out = [] +i = 0 +while i < len(lines): + line = lines[i] + out.append(line) + i += 1 + match = header.search(line.rstrip("\n")) + if not match or match.group(3) == "+": + continue + style, block_indent = match.group(1), indent_of(line) + + # The body runs until the first non-blank line indented no deeper than the header. + body = [] + while i < len(lines): + following = lines[i] + if following.strip() and indent_of(following) <= block_indent: + break + body.append(following) + i += 1 + + while body and not body[-1].strip(): + body.pop() + + if style == ">": + kept, blanks, previous_indent = [], [], None + for body_line in body: + if not body_line.strip(): + blanks.append(body_line) + continue + current_indent = indent_of(body_line) + if previous_indent is None or current_indent <= previous_indent: + kept.extend(blanks) + blanks = [] + kept.append(body_line) + previous_indent = current_indent + body = kept + + out.extend(body) + +with open(path, "w", encoding="utf-8") as f: + f.write("".join(out)) +PY + echo "Synced schemas from ${SOURCE} -> ${DEST} (${DEST}.bak created)." echo "Note: Schemas in Operations with x-operation: true were preserved." \ No newline at end of file