Skip to content

Commit 280d927

Browse files
pidoubleyoupidoubleyou
committed
support zdftivi and topics without seasons like Herzkino
1 parent bc065c1 commit 280d927

File tree

7 files changed

+4841
-67
lines changed

7 files changed

+4841
-67
lines changed

‎src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfConstants.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,18 @@ public final class ZdfConstants {
3939
public static final String URL_TOPIC_PAGE_EXTENSIONS =
4040
"{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"9412a0f4ac55dc37d46975d461ec64bfd14380d815df843a1492348f77b5c99a\"}}";
4141

42+
public static final String URL_TOPIC_PAGE_NO_SEASON = URL_API_BASE + "/graphql?operationName=getMetaCollectionContent&" +
43+
"variables=%s&" +
44+
"extensions=%s";
45+
// "appId":"ffw-mt-web-879d5c17",
46+
// ,"filters":{"contentOwner":[],"fsk":[],"language":[]}
47+
// ,"user":{"abGroup":"gruppe-b","userSegment":"segment_0"}
48+
public static final String URL_TOPIC_PAGE_NO_SEASON_VARIABLES =
49+
"{\"collectionId\":\"%s\",\"input\":{\"appId\":\"ffw-mt-web-879d5c17\",\"filters\":{\"contentOwner\":[],\"fsk\":[],\"language\":[]},\"pagination\":{\"first\":%d,\"after\":%s},\"user\":{\"abGroup\":\"gruppe-d\",\"userSegment\":\"segment_0\"},\"tabId\":null}}";
50+
// todo woher kommt ie CollectionId? =>id
51+
public static final String URL_TOPIC_PAGE_NO_SEASON_EXTENSIONS =
52+
"{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"c85ca9c636258a65961a81124abd0dbef06ab97eaca9345cbdfde23b54117242\"}}";
53+
4254
public static final String URL_FILM_ENRY =
4355
URL_API_BASE + "/graphql?operationName=GetVideoMetaByCanonical&"
4456
+ "variables={\"canonical\"=\"%s\"}&"
@@ -69,6 +81,7 @@ public final class ZdfConstants {
6981
PARTNER_TO_SENDER.put("ZDFneo", Sender.ZDF_NEO);
7082
PARTNER_TO_SENDER.put("ZDF", Sender.ZDF);
7183
PARTNER_TO_SENDER.put("EMPTY", Sender.ZDF);
84+
PARTNER_TO_SENDER.put("ZDFtivi", Sender.ZDF_TIVI);
7285
// IGNORED Sender [KI.KA, WDR, PHOENIX, one, HR, 3sat, SWR, arte, BR, RBB, ARD, daserste, alpha, MDR, radiobremen, funk, ZDF, NDR, SR]
7386
}
7487

‎src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfUrlBuilder.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,11 @@ public static String buildTopicSeasonUrl(int seasonNumber, int pageSize, String
3535
URLEncoder.encode(String.format(ZdfConstants.URL_TOPIC_PAGE_VARIABLES_WITH_CURSOR, seasonNumber, pageSize, canonical, cursor), Charset.defaultCharset()),
3636
URLEncoder.encode(ZdfConstants.URL_TOPIC_PAGE_EXTENSIONS, Charset.defaultCharset()));
3737
}
38+
39+
public static String buildTopicNoSeasonUrl(int pageSize, String id, String cursor) {
40+
return String.format(
41+
ZdfConstants.URL_TOPIC_PAGE_NO_SEASON,
42+
URLEncoder.encode(String.format(ZdfConstants.URL_TOPIC_PAGE_NO_SEASON_VARIABLES, id, pageSize, cursor), Charset.defaultCharset()),
43+
URLEncoder.encode(ZdfConstants.URL_TOPIC_PAGE_NO_SEASON_EXTENSIONS, Charset.defaultCharset()));
44+
}
3845
}

‎src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfLetterPageDeserializer.java‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,19 +40,27 @@ public PagedElementListDTO<ZdfTopicUrlDto> deserialize(
4040
final Optional<String> topic = JsonUtils.getElementValueAsString(node, "title");
4141
final Optional<String> countSeasons = JsonUtils.getElementValueAsString(node, "countSeasons");
4242
if (ZdfConstants.PARTNER_TO_SENDER.containsKey(sender.orElse("ZDF"))) {
43-
// todo: gibt es den Fall, dass es keine Season gibt??
4443
if (countSeasons.isEmpty()) {
45-
LOG.error("no season found for {}", node.get("canonical").getAsString());
46-
}
47-
for (int i = 0; i < Integer.parseInt(countSeasons.orElse("0")); i++) {
48-
String canonical = node.get("canonical").getAsString();
49-
result.addElement(
50-
new ZdfTopicUrlDto(
51-
topic.orElse(""),
52-
i,
53-
canonical,
54-
ZdfUrlBuilder.buildTopicSeasonUrl(
55-
i, ZdfConstants.EPISODES_PAGE_SIZE, canonical)));
44+
final Optional<String> id = JsonUtils.getElementValueAsString(node, "id");
45+
id.ifPresent(s -> result.addElement(
46+
new ZdfTopicUrlDto(
47+
topic.orElse(""),
48+
0,
49+
s,
50+
ZdfUrlBuilder.buildTopicNoSeasonUrl(
51+
ZdfConstants.EPISODES_PAGE_SIZE, s, ZdfConstants.NO_CURSOR)
52+
)));
53+
} else {
54+
for (int i = 0; i < Integer.parseInt(countSeasons.orElse("0")); i++) {
55+
String canonical = node.get("canonical").getAsString();
56+
result.addElement(
57+
new ZdfTopicUrlDto(
58+
topic.orElse(""),
59+
i,
60+
canonical,
61+
ZdfUrlBuilder.buildTopicSeasonUrl(
62+
i, ZdfConstants.EPISODES_PAGE_SIZE, canonical)));
63+
}
5664
}
5765
}
5866
}

‎src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfTopicSeasonDeserializer.java‎

Lines changed: 84 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -39,60 +39,84 @@ public PagedElementListDTO<ZdfFilmDto> deserialize(
3939
final PagedElementListDTO<ZdfFilmDto> films = new PagedElementListDTO<>();
4040

4141
JsonObject rootNode = jsonElement.getAsJsonObject();
42-
JsonArray nodes =
43-
rootNode
44-
.getAsJsonObject("data")
45-
.getAsJsonObject("smartCollectionByCanonical")
46-
.getAsJsonObject("seasons")
47-
.getAsJsonArray("nodes");
48-
49-
for (JsonElement element : nodes) {
50-
51-
final JsonObject episodes = element.getAsJsonObject().getAsJsonObject("episodes");
52-
final JsonArray episodeNodes = episodes.getAsJsonArray("nodes");
53-
for (JsonElement episode : episodeNodes) {
54-
final JsonObject episodeObject = episode.getAsJsonObject();
55-
final Optional<String> title = parseTitle(episodeObject);
56-
final Optional<String> website =
57-
JsonUtils.getAttributeAsString(episodeObject, "sharingUrl");
58-
final Optional<LocalDateTime> time = parseDate(episodeObject);
59-
final Optional<String> description =
60-
JsonUtils.getAttributeAsString(episodeObject.getAsJsonObject("teaser"), "description");
61-
final Optional<String> sender = parseSender(episodeObject);
62-
63-
// streamingoptions relevant, um zu erkennen ob uhd/dgs/ad/ov...?
64-
final Map<String, String> downloadUrls = new HashMap<>();
65-
66-
final JsonArray mediaNodes =
67-
episodeObject.getAsJsonObject("currentMedia").getAsJsonArray("nodes");
68-
for (JsonElement media : mediaNodes) {
69-
final JsonObject mediaObject = media.getAsJsonObject();
70-
final Optional<String> mediaType =
71-
JsonUtils.getAttributeAsString(mediaObject, "vodMediaType");
72-
final Optional<String> url = JsonUtils.getAttributeAsString(mediaObject, "ptmdTemplate");
73-
if (mediaType.isPresent() && url.isPresent()) {
74-
downloadUrls.put(mediaType.get(), finalizeDownloadUrl(url.get()));
75-
}
76-
}
42+
final JsonObject data = rootNode.getAsJsonObject("data");
43+
if (data.isJsonNull()) {
44+
LOG.error("ZdfTopicSeasonDeserializer: No data found in response");
45+
return films;
46+
}
7747

78-
if (title.isPresent()) {
79-
films.addElements(
80-
createFilm(
81-
ZdfConstants.PARTNER_TO_SENDER.get(sender.orElse("EMPTY")),
82-
title.get(),
83-
description,
84-
website,
85-
time,
86-
downloadUrls));
87-
} else {
88-
LOG.error("ZdfTopicSeasonDeserializer: no title found");
48+
if (data.has("smartCollectionByCanonical")) {
49+
final JsonArray seasonNodes = data.getAsJsonObject("smartCollectionByCanonical")
50+
.getAsJsonObject("seasons")
51+
.getAsJsonArray("nodes");
52+
for (JsonElement element : seasonNodes) {
53+
final JsonObject episodes = element.getAsJsonObject().getAsJsonObject("episodes");
54+
final JsonArray episodeNodes = episodes.getAsJsonArray("nodes");
55+
addFilms(films, episodeNodes);
56+
films.setNextPage(parseNextPage(episodes.getAsJsonObject("pageInfo")));
57+
}
58+
}
59+
else if (data.has("metaCollectionContent")) {
60+
final JsonObject metaCollectionContent = data.getAsJsonObject("metaCollectionContent");
61+
final JsonArray collectionNodes = metaCollectionContent
62+
.getAsJsonArray("smartCollections");
63+
addFilms(films, collectionNodes);
64+
films.setNextPage(parseNextPage(metaCollectionContent.getAsJsonObject("pageInfo")));
65+
} else {
66+
LOG.error("ZdfTopicSeasonDeserializer: No valid entry nodes found");
67+
}
68+
return films;
69+
}
70+
71+
private void addFilms(PagedElementListDTO<ZdfFilmDto> films, JsonArray episodeNodes) {
72+
for (JsonElement episode : episodeNodes) {
73+
final JsonObject episodeObject = episode.getAsJsonObject();
74+
final Optional<String> title = parseTitle(episodeObject);
75+
final Optional<String> website =
76+
JsonUtils.getAttributeAsString(episodeObject, "sharingUrl");
77+
final Optional<LocalDateTime> time = parseDate(episodeObject);
78+
final Optional<String> description =
79+
JsonUtils.getAttributeAsString(episodeObject.getAsJsonObject("teaser"), "description");
80+
final Optional<String> sender = parseSender(episodeObject);
81+
82+
// streamingoptions relevant, um zu erkennen ob uhd/dgs/ad/ov...?
83+
final Map<String, String> downloadUrls = new HashMap<>();
84+
85+
final JsonArray mediaNodes = getMediaNodes(episodeObject);
86+
for (JsonElement media : mediaNodes) {
87+
final JsonObject mediaObject = media.getAsJsonObject();
88+
final Optional<String> mediaType =
89+
JsonUtils.getAttributeAsString(mediaObject, "vodMediaType");
90+
final Optional<String> url = JsonUtils.getAttributeAsString(mediaObject, "ptmdTemplate");
91+
if (mediaType.isPresent() && url.isPresent()) {
92+
downloadUrls.put(mediaType.get(), finalizeDownloadUrl(url.get()));
8993
}
9094
}
9195

92-
films.setNextPage(parseNextPage(episodes.getAsJsonObject("pageInfo")));
96+
if (title.isPresent()) {
97+
films.addElements(
98+
createFilm(
99+
ZdfConstants.PARTNER_TO_SENDER.get(sender.orElse("EMPTY")),
100+
title.get(),
101+
description,
102+
website,
103+
time,
104+
downloadUrls));
105+
} else {
106+
LOG.error("ZdfTopicSeasonDeserializer: no title found");
107+
}
93108
}
109+
}
94110

95-
return films;
111+
private JsonArray getMediaNodes(JsonObject episodeObject) {
112+
JsonObject videoRootObject = episodeObject;
113+
if (episodeObject.has("video")) {
114+
videoRootObject = episodeObject.getAsJsonObject("video");
115+
}
116+
if (!videoRootObject.has("currentMedia")) {
117+
return new JsonArray();
118+
}
119+
return videoRootObject.getAsJsonObject("currentMedia").getAsJsonArray("nodes");
96120
}
97121

98122
private Optional<String> parseSender(JsonObject episodeObject) {
@@ -130,13 +154,18 @@ private Optional<String> parseTitle(final JsonObject episodeObject) {
130154
Optional<String> resultingTitle = formatTitle(title, subtitle);
131155

132156
if (resultingTitle.isPresent()) {
133-
final Optional<Integer> season =
134-
JsonUtils.getAttributeAsInt(episodeObject.getAsJsonObject("episodeInfo"), "seasonNumber");
135-
final Optional<Integer> episode =
136-
JsonUtils.getAttributeAsInt(
137-
episodeObject.getAsJsonObject("episodeInfo"), "episodeNumber");
138-
final Optional<String> seasonEpisodeTitle = formatEpisodeTitle(season, episode);
139-
return cleanupTitle((resultingTitle.get() + " " + seasonEpisodeTitle.orElse("")).trim());
157+
if (episodeObject.has("episodeInfo")) {
158+
final Optional<Integer> season =
159+
JsonUtils.getAttributeAsInt(
160+
episodeObject.getAsJsonObject("episodeInfo"), "seasonNumber");
161+
final Optional<Integer> episode =
162+
JsonUtils.getAttributeAsInt(
163+
episodeObject.getAsJsonObject("episodeInfo"), "episodeNumber");
164+
final Optional<String> seasonEpisodeTitle = formatEpisodeTitle(season, episode);
165+
return cleanupTitle((resultingTitle.get() + " " + seasonEpisodeTitle.orElse("")).trim());
166+
} else {
167+
return cleanupTitle(resultingTitle.get());
168+
}
140169
}
141170
return Optional.empty();
142171
}

‎src/test/java/de/mediathekview/mserver/crawler/zdf/ZdfUrlBuilderTest.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,11 @@ void testBuildTopicSeasonUrlWithCursor() {
3030
ZdfUrlBuilder.buildTopicSeasonUrl(18, 24, "soko-wismar-104", "WyJmZTlmZTc5ZDU1IiwiMjAyMy0xMi0wOFQxOTo0NjowNi4wOTIwMDArMDA6MDAiLCJwYWdlLXZpZGVvLWFyZF92aWRlb19hcmRfZFhKdU9tRnlaRHB3ZFdKc2FXTmhkR2x2YmpvMVlqSXhZV1E1TmpFeFpqVTRNalZqIl0="));
3131
}
3232

33+
@Test
34+
void testBuildTopicUrlNoSeason() {
35+
assertEquals("https://api.zdf.de/graphql?operationName=getMetaCollectionContent&variables=%7B%22collectionId%22%3A%22e5acd149-ee98-44b9-8b08-0b70357b4b46%22%2C%22input%22%3A%7B%22pagination%22%3A%7B%22first%22%3A18%2C%22after%22%3Anull%7D%2C%22tabId%22%3Anull%7D%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%22c85ca9c636258a65961a81124abd0dbef06ab97eaca9345cbdfde23b54117242%22%7D%7D",
36+
ZdfUrlBuilder.buildTopicNoSeasonUrl(18, "e5acd149-ee98-44b9-8b08-0b70357b4b46", ZdfConstants.NO_CURSOR));
37+
}
38+
39+
3340
}

‎src/test/java/de/mediathekview/mserver/crawler/zdf/json/ZdfTopicSeasonDeserializerTest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,13 @@ void deserializeZdfNeo() {
5454
assertEquals(13, actual.getElements().size());
5555
actual.getElements().forEach(element -> assertEquals(Sender.ZDF_NEO, element.getSender()));
5656
}
57+
@Test
58+
void deserializeNoSeason() {
59+
final JsonObject json = JsonFileReader.readJson("/zdf/zdf_topic_page_no_season.json");
60+
final PagedElementListDTO<ZdfFilmDto> actual = target.deserialize(json, null, null);
61+
62+
assertEquals(Optional.empty(), actual.getNextPage());
63+
assertEquals(11, actual.getElements().size());
64+
actual.getElements().forEach(element -> assertEquals(Sender.ZDF, element.getSender()));
65+
}
5766
}

0 commit comments

Comments
 (0)