Skip to content

Commit f891b40

Browse files
qcdyxqcdyx
committed
returned feed_id not id
1 parent 85a1a82 commit f891b40

1 file changed

Lines changed: 168 additions & 84 deletions

File tree

‎liquibase/changes/feat_1396.sql‎

Lines changed: 168 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,11 @@
11
-- Latest dataset id should return the feed_id, not the UID of the dataset
2-
DO
3-
'
4-
DECLARE
5-
BEGIN
6-
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = ''translationtype'') THEN
7-
CREATE TYPE TranslationType AS ENUM (''country'', ''subdivision_name'', ''municipality'');
8-
END IF;
9-
END;
10-
' LANGUAGE PLPGSQL;
11-
12-
CREATE TABLE IF NOT EXISTS Translation (
13-
type TranslationType NOT NULL,
14-
language_code VARCHAR(3) NOT NULL, -- ISO 639-2
15-
key VARCHAR(255) NOT NULL,
16-
value VARCHAR(255) NOT NULL,
17-
PRIMARY KEY (type, language_code, key)
18-
);
192

20-
-- Dropping the materialized view if it exists as we cannot update it
21-
DROP MATERIALIZED VIEW IF EXISTS FeedSearch;
3+
DROP MATERIALIZED VIEW IF EXISTS feedsearch;
4+
5+
ALTER TABLE feed
6+
ALTER COLUMN note TYPE TEXT;
227

8+
-- Recreate the FeedSearch materialized view
239
CREATE MATERIALIZED VIEW FeedSearch AS
2410
SELECT
2511
-- feed
@@ -37,12 +23,29 @@ SELECT
3723
Feed.api_key_parameter_name,
3824
Feed.license_url,
3925
Feed.provider,
26+
Feed.operational_status,
27+
-- official status
28+
Feed.official AS official,
29+
-- created_at
30+
Feed.created_at AS created_at,
4031
-- latest_dataset
4132
Latest_dataset.stable_id AS latest_dataset_id,
4233
Latest_dataset.hosted_url AS latest_dataset_hosted_url,
4334
Latest_dataset.downloaded_at AS latest_dataset_downloaded_at,
4435
Latest_dataset.bounding_box AS latest_dataset_bounding_box,
4536
Latest_dataset.hash AS latest_dataset_hash,
37+
Latest_dataset.agency_timezone AS latest_dataset_agency_timezone,
38+
Latest_dataset.service_date_range_start AS latest_dataset_service_date_range_start,
39+
Latest_dataset.service_date_range_end AS latest_dataset_service_date_range_end,
40+
-- Latest dataset features
41+
LatestDatasetFeatures AS latest_dataset_features,
42+
-- Latest dataset validation totals
43+
COALESCE(LatestDatasetValidationReportJoin.total_error, 0) as latest_total_error,
44+
COALESCE(LatestDatasetValidationReportJoin.total_warning, 0) as latest_total_warning,
45+
COALESCE(LatestDatasetValidationReportJoin.total_info, 0) as latest_total_info,
46+
COALESCE(LatestDatasetValidationReportJoin.unique_error_count, 0) as latest_unique_error_count,
47+
COALESCE(LatestDatasetValidationReportJoin.unique_warning_count, 0) as latest_unique_warning_count,
48+
COALESCE(LatestDatasetValidationReportJoin.unique_info_count, 0) as latest_unique_info_count,
4649
-- external_ids
4750
ExternalIdJoin.external_ids,
4851
-- redirect_ids
@@ -53,10 +56,11 @@ SELECT
5356
EntityTypeFeedJoin.entities,
5457
-- locations
5558
FeedLocationJoin.locations,
56-
-- translations
57-
FeedCountryTranslationJoin.translations AS country_translations,
58-
FeedSubdivisionNameTranslationJoin.translations AS subdivision_name_translations,
59-
FeedMunicipalityTranslationJoin.translations AS municipality_translations,
59+
-- osm locations grouped
60+
OsmLocationJoin.osm_locations,
61+
-- gbfs versions
62+
COALESCE(GbfsVersionsJoin.versions, '[]'::jsonb) AS versions,
63+
6064
-- full-text searchable document
6165
setweight(to_tsvector('english', coalesce(unaccent(Feed.feed_name), '')), 'C') ||
6266
setweight(to_tsvector('english', coalesce(unaccent(Feed.provider), '')), 'C') ||
@@ -70,40 +74,80 @@ SELECT
7074
)
7175
FROM json_array_elements(FeedLocationJoin.locations) AS location
7276
)), '')), 'A') ||
73-
setweight(to_tsvector('english', coalesce(unaccent((
74-
SELECT string_agg(
75-
coalesce(translation->>'value', ''),
76-
' '
77-
)
78-
FROM json_array_elements(FeedCountryTranslationJoin.translations) AS translation
79-
)), '')), 'A') ||
80-
setweight(to_tsvector('english', coalesce(unaccent((
81-
SELECT string_agg(
82-
coalesce(translation->>'value', ''),
83-
' '
84-
)
85-
FROM json_array_elements(FeedSubdivisionNameTranslationJoin.translations) AS translation
86-
)), '')), 'A') ||
87-
setweight(to_tsvector('english', coalesce(unaccent((
88-
SELECT string_agg(
89-
coalesce(translation->>'value', ''),
90-
' '
91-
)
92-
FROM json_array_elements(FeedMunicipalityTranslationJoin.translations) AS translation
93-
)), '')), 'A') AS document
77+
setweight(to_tsvector('english', coalesce(unaccent(OsmLocationNamesJoin.osm_location_names), '')), 'A')
78+
AS document
9479
FROM Feed
80+
81+
-- Latest dataset
82+
LEFT JOIN gtfsfeed gtf ON gtf.id = Feed.id AND Feed.data_type = 'gtfs'
83+
LEFT JOIN gtfsdataset Latest_dataset ON Latest_dataset.id = gtf.latest_dataset_id
84+
85+
-- Latest dataset features
86+
LEFT JOIN (
87+
SELECT
88+
GtfsDataset.id AS FeatureGtfsDatasetId,
89+
array_agg(DISTINCT FeatureValidationReport.feature) AS LatestDatasetFeatures
90+
FROM GtfsDataset
91+
JOIN ValidationReportGtfsDataset
92+
ON ValidationReportGtfsDataset.dataset_id = GtfsDataset.id
93+
JOIN (
94+
-- Pick latest ValidationReport per dataset based on validated_at
95+
SELECT DISTINCT ON (ValidationReportGtfsDataset.dataset_id)
96+
ValidationReportGtfsDataset.dataset_id,
97+
ValidationReport.id AS latest_validation_report_id
98+
FROM ValidationReportGtfsDataset
99+
JOIN ValidationReport
100+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
101+
ORDER BY
102+
ValidationReportGtfsDataset.dataset_id,
103+
ValidationReport.validated_at DESC
104+
) AS LatestReports
105+
ON LatestReports.latest_validation_report_id = ValidationReportGtfsDataset.validation_report_id
106+
JOIN FeatureValidationReport
107+
ON FeatureValidationReport.validation_id = ValidationReportGtfsDataset.validation_report_id
108+
GROUP BY FeatureGtfsDatasetId
109+
) AS LatestDatasetFeaturesJoin ON Latest_dataset.id = FeatureGtfsDatasetId
110+
111+
-- Latest dataset validation report
95112
LEFT JOIN (
96-
SELECT *
97-
FROM gtfsdataset
98-
WHERE latest = true
99-
) AS Latest_dataset ON Latest_dataset.feed_id = Feed.id AND Feed.data_type = 'gtfs'
113+
SELECT
114+
GtfsDataset.id AS ValidationReportGtfsDatasetId,
115+
ValidationReport.total_error,
116+
ValidationReport.total_warning,
117+
ValidationReport.total_info,
118+
ValidationReport.unique_error_count,
119+
ValidationReport.unique_warning_count,
120+
ValidationReport.unique_info_count
121+
FROM GtfsDataset
122+
JOIN ValidationReportGtfsDataset
123+
ON ValidationReportGtfsDataset.dataset_id = GtfsDataset.id
124+
JOIN (
125+
-- Pick latest ValidationReport per dataset based on validated_at
126+
SELECT DISTINCT ON (ValidationReportGtfsDataset.dataset_id)
127+
ValidationReportGtfsDataset.dataset_id,
128+
ValidationReport.id AS latest_validation_report_id
129+
FROM ValidationReportGtfsDataset
130+
JOIN ValidationReport
131+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
132+
ORDER BY
133+
ValidationReportGtfsDataset.dataset_id,
134+
ValidationReport.validated_at DESC
135+
) AS LatestReports
136+
ON LatestReports.latest_validation_report_id = ValidationReportGtfsDataset.validation_report_id
137+
JOIN ValidationReport
138+
ON ValidationReport.id = ValidationReportGtfsDataset.validation_report_id
139+
) AS LatestDatasetValidationReportJoin ON Latest_dataset.id = ValidationReportGtfsDatasetId
140+
141+
-- External ids
100142
LEFT JOIN (
101143
SELECT
102144
feed_id,
103145
json_agg(json_build_object('external_id', associated_id, 'source', source)) AS external_ids
104146
FROM externalid
105147
GROUP BY feed_id
106148
) AS ExternalIdJoin ON ExternalIdJoin.feed_id = Feed.id
149+
150+
-- feed reference ids
107151
LEFT JOIN (
108152
SELECT
109153
gtfs_rt_feed_id,
@@ -112,13 +156,17 @@ LEFT JOIN (
112156
LEFT JOIN Feed AS FeedReferenceJoinInnerQuery ON FeedReferenceJoinInnerQuery.id = FeedReference.gtfs_feed_id
113157
GROUP BY gtfs_rt_feed_id
114158
) AS FeedReferenceJoin ON FeedReferenceJoin.gtfs_rt_feed_id = Feed.id AND Feed.data_type = 'gtfs_rt'
159+
160+
-- Redirect ids
115161
LEFT JOIN (
116162
SELECT
117163
target_id,
118164
json_agg(json_build_object('target_id', target_id, 'comment', redirect_comment)) AS redirect_ids
119165
FROM RedirectingId
120166
GROUP BY target_id
121167
) AS RedirectingIdJoin ON RedirectingIdJoin.target_id = Feed.id
168+
169+
-- Feed locations
122170
LEFT JOIN (
123171
SELECT
124172
LocationFeed.feed_id,
@@ -128,50 +176,70 @@ LEFT JOIN (
128176
LEFT JOIN LocationFeed ON LocationFeed.location_id = Location.id
129177
GROUP BY LocationFeed.feed_id
130178
) AS FeedLocationJoin ON FeedLocationJoin.feed_id = Feed.id
179+
180+
-- Entity types
131181
LEFT JOIN (
132182
SELECT
133-
LocationFeed.feed_id,
134-
json_agg(json_build_object('value', Translation.value, 'key', Translation.key)) AS translations
135-
FROM Location
136-
LEFT JOIN Translation ON Location.country = Translation.key
137-
LEFT JOIN LocationFeed ON LocationFeed.location_id = Location.id
138-
WHERE Translation.language_code = 'en'
139-
AND Translation.type = 'country'
140-
AND Location.country IS NOT NULL
141-
GROUP BY LocationFeed.feed_id
142-
) AS FeedCountryTranslationJoin ON FeedCountryTranslationJoin.feed_id = Feed.id
183+
feed_id,
184+
array_agg(entity_name) AS entities
185+
FROM EntityTypeFeed
186+
GROUP BY feed_id
187+
) AS EntityTypeFeedJoin ON EntityTypeFeedJoin.feed_id = Feed.id AND Feed.data_type = 'gtfs_rt'
188+
189+
-- OSM locations
143190
LEFT JOIN (
191+
WITH locations_per_group AS (
192+
SELECT
193+
fog.feed_id,
194+
olg.group_name,
195+
jsonb_agg(
196+
DISTINCT jsonb_build_object(
197+
'admin_level', gp.admin_level,
198+
'name', gp.name
199+
)
200+
) AS locations
201+
FROM FeedOsmLocationGroup fog
202+
JOIN OsmLocationGroup olg ON olg.group_id = fog.group_id
203+
JOIN OsmLocationGroupGeopolygon olgg ON olgg.group_id = olg.group_id
204+
JOIN Geopolygon gp ON gp.osm_id = olgg.osm_id
205+
GROUP BY fog.feed_id, olg.group_name
206+
)
144207
SELECT
145-
LocationFeed.feed_id,
146-
json_agg(json_build_object('value', Translation.value, 'key', Translation.key)) AS translations
147-
FROM Location
148-
LEFT JOIN Translation ON Location.subdivision_name = Translation.key
149-
LEFT JOIN LocationFeed ON LocationFeed.location_id = Location.id
150-
WHERE Translation.language_code = 'en'
151-
AND Translation.type = 'subdivision_name'
152-
AND Location.subdivision_name IS NOT NULL
153-
GROUP BY LocationFeed.feed_id
154-
) AS FeedSubdivisionNameTranslationJoin ON FeedSubdivisionNameTranslationJoin.feed_id = Feed.id
208+
feed_id,
209+
jsonb_agg(
210+
jsonb_build_object(
211+
'group_name', group_name,
212+
'locations', locations
213+
)
214+
)::json AS osm_locations
215+
FROM locations_per_group
216+
GROUP BY feed_id
217+
) AS OsmLocationJoin ON OsmLocationJoin.feed_id = Feed.id
218+
219+
-- OSM location names
155220
LEFT JOIN (
156221
SELECT
157-
LocationFeed.feed_id,
158-
json_agg(json_build_object('value', Translation.value, 'key', Translation.key)) AS translations
159-
FROM Location
160-
LEFT JOIN Translation ON Location.municipality = Translation.key
161-
LEFT JOIN LocationFeed ON LocationFeed.location_id = Location.id
162-
WHERE Translation.language_code = 'en'
163-
AND Translation.type = 'municipality'
164-
AND Location.municipality IS NOT NULL
165-
GROUP BY LocationFeed.feed_id
166-
) AS FeedMunicipalityTranslationJoin ON FeedMunicipalityTranslationJoin.feed_id = Feed.id
222+
fog.feed_id,
223+
string_agg(DISTINCT gp.name, ' ') AS osm_location_names
224+
FROM FeedOsmLocationGroup fog
225+
JOIN OsmLocationGroup olg ON olg.group_id = fog.group_id
226+
JOIN OsmLocationGroupGeopolygon olgg ON olgg.group_id = olg.group_id
227+
JOIN Geopolygon gp ON gp.osm_id = olgg.osm_id
228+
WHERE gp.name IS NOT NULL
229+
GROUP BY fog.feed_id
230+
) AS OsmLocationNamesJoin ON OsmLocationNamesJoin.feed_id = Feed.id
231+
232+
-- GBFS versions
167233
LEFT JOIN (
168234
SELECT
169-
feed_id,
170-
array_agg(entity_name) AS entities
171-
FROM EntityTypeFeed
172-
GROUP BY feed_id
173-
) AS EntityTypeFeedJoin ON EntityTypeFeedJoin.feed_id = Feed.id AND Feed.data_type = 'gtfs_rt'
174-
;
235+
Feed.id AS feed_id,
236+
to_jsonb(array_agg(DISTINCT GbfsVersion.version ORDER BY GbfsVersion.version)) AS versions
237+
FROM Feed
238+
JOIN GbfsFeed ON GbfsFeed.id = Feed.id
239+
JOIN GbfsVersion ON GbfsVersion.feed_id = GbfsFeed.id
240+
WHERE Feed.data_type = 'gbfs'
241+
GROUP BY Feed.id
242+
) AS GbfsVersionsJoin ON GbfsVersionsJoin.feed_id = Feed.id;
175243

176244

177245
-- This index allows concurrent refresh on the materialized view avoiding table locks
@@ -182,3 +250,19 @@ CREATE INDEX feedsearch_document_idx ON FeedSearch USING GIN(document);
182250
CREATE INDEX feedsearch_feed_stable_id ON FeedSearch(feed_stable_id);
183251
CREATE INDEX feedsearch_data_type ON FeedSearch(data_type);
184252
CREATE INDEX feedsearch_status ON FeedSearch(status);
253+
254+
255+
-- Update search
256+
REFRESH MATERIALIZED VIEW CONCURRENTLY FeedSearch;
257+
258+
-- create new related links table
259+
CREATE TABLE IF NOT EXISTS FeedRelatedLink (
260+
feed_id varchar(255) NOT NULL REFERENCES Feed(id) ON DELETE CASCADE,
261+
description TEXT NOT NULL,
262+
code TEXT NOT NULL,
263+
url TEXT NOT NULL,
264+
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
265+
PRIMARY KEY (feed_id, code)
266+
);
267+
268+

0 commit comments

Comments
 (0)