Skip to content

Commit 5704fc0

Browse files
pidoubleyoupidoubleyou
committed
add special categories
1 parent 6b6fbdc commit 5704fc0

File tree

11 files changed

+5290
-198
lines changed

11 files changed

+5290
-198
lines changed

‎src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfConstants.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ public final class ZdfConstants {
1919
public static final String NO_CURSOR = "null";
2020
public static final int EPISODES_PAGE_SIZE = 24;
2121

22-
// todo: itemsfilter so ok?
2322
public static final String URL_LETTER_PAGE =
2423
URL_API_BASE
2524
+ "/graphql?operationName=specialPageByCanonical&" +
@@ -30,7 +29,6 @@ public final class ZdfConstants {
3029
public static final String URL_LETTER_PAGE_EXTENSIONS =
3130
"{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"7d33167e7700ba57779f48b28b5d485c8ada0a1d5dfbdc8a261b7bd62ca28ba7\"}}";
3231

33-
// todo filter raus => ok? "filterBy":{"idIn":["13-fragen-_season_1"]}
3432
public static final String URL_TOPIC_PAGE = URL_API_BASE + "/graphql?operationName=seasonByCanonical&" +
3533
"variables=%s&" +
3634
"extensions=%s";
@@ -42,12 +40,8 @@ public final class ZdfConstants {
4240
public static final String URL_TOPIC_PAGE_NO_SEASON = URL_API_BASE + "/graphql?operationName=getMetaCollectionContent&" +
4341
"variables=%s&" +
4442
"extensions=%s";
45-
// "appId":"ffw-mt-web-879d5c17",
46-
// ,"filters":{"contentOwner":[],"fsk":[],"language":[]}
47-
// ,"user":{"abGroup":"gruppe-b","userSegment":"segment_0"}
4843
public static final String URL_TOPIC_PAGE_NO_SEASON_VARIABLES =
4944
"{\"collectionId\":\"%s\",\"input\":{\"appId\":\"ffw-mt-web-879d5c17\",\"filters\":{\"contentOwner\":[],\"fsk\":[],\"language\":[]},\"pagination\":{\"first\":%d,\"after\":%s},\"user\":{\"abGroup\":\"gruppe-d\",\"userSegment\":\"segment_0\"},\"tabId\":null}}";
50-
// todo woher kommt ie CollectionId? =>id
5145
public static final String URL_TOPIC_PAGE_NO_SEASON_EXTENSIONS =
5246
"{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"c85ca9c636258a65961a81124abd0dbef06ab97eaca9345cbdfde23b54117242\"}}";
5347

@@ -75,14 +69,19 @@ public final class ZdfConstants {
7569
public static final String LANGUAGE_GERMAN_DGS = LANGUAGE_GERMAN + LANGUAGE_SUFFIX_DGS;
7670

7771
public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
72+
public static final Map<String, String> SPECIAL_COLLECTION_IDS = new HashMap<>();
7873

7974
static {
8075
PARTNER_TO_SENDER.put("ZDFinfo", Sender.ZDF_INFO);
8176
PARTNER_TO_SENDER.put("ZDFneo", Sender.ZDF_NEO);
82-
PARTNER_TO_SENDER.put("ZDF", Sender.ZDF);
77+
PARTNER_TO_SENDER.put("ZDF", Sender.ZDF);
8378
PARTNER_TO_SENDER.put("EMPTY", Sender.ZDF);
8479
PARTNER_TO_SENDER.put("ZDFtivi", Sender.ZDF_TIVI);
8580
// IGNORED Sender [KI.KA, WDR, PHOENIX, one, HR, 3sat, SWR, arte, BR, RBB, ARD, daserste, alpha, MDR, radiobremen, funk, ZDF, NDR, SR]
81+
82+
SPECIAL_COLLECTION_IDS.put("pub-form-10004", "Filme");
83+
SPECIAL_COLLECTION_IDS.put("pub-form-10003", "Dokus");
84+
SPECIAL_COLLECTION_IDS.put("pub-form-10010", "Serien");
8685
}
8786

8887
private ZdfConstants() {}

‎src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfCrawler.java‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,10 @@
88
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
99
import de.mediathekview.mserver.crawler.zdf.tasks.ZdfFilmTask;
1010
import de.mediathekview.mserver.crawler.zdf.tasks.ZdfLetterPageTask;
11+
import de.mediathekview.mserver.crawler.zdf.tasks.ZdfPubFormTask;
1112
import de.mediathekview.mserver.crawler.zdf.tasks.ZdfTopicSeasonTask;
1213
import de.mediathekview.mserver.progress.listeners.SenderProgressListener;
13-
import java.util.Collection;
14-
import java.util.Queue;
15-
import java.util.Set;
14+
import java.util.*;
1615
import java.util.concurrent.ConcurrentLinkedQueue;
1716
import java.util.concurrent.ExecutionException;
1817
import java.util.concurrent.ForkJoinPool;
@@ -25,7 +24,11 @@ public class ZdfCrawler extends AbstractCrawler {
2524
private static final Logger LOG = LogManager.getLogger(ZdfCrawler.class);
2625
private static final int MAX_LETTER_PAGEGS = 27;
2726

28-
public ZdfCrawler(ForkJoinPool aForkJoinPool, Collection<MessageListener> aMessageListeners, Collection<SenderProgressListener> aProgressListeners, MServerConfigManager rootConfig) {
27+
public ZdfCrawler(
28+
ForkJoinPool aForkJoinPool,
29+
Collection<MessageListener> aMessageListeners,
30+
Collection<SenderProgressListener> aProgressListeners,
31+
MServerConfigManager rootConfig) {
2932
super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
3033
}
3134

@@ -39,18 +42,35 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
3942

4043
final String authKey = "aa3noh4ohz9eeboo8shiesheec9ciequ9Quah7el";
4144
try {
42-
ZdfLetterPageTask letterPageTask = new ZdfLetterPageTask(this, createLetterPageUrls(), authKey);
45+
final Set<ZdfFilmDto> shows = new HashSet<>();
46+
47+
ZdfLetterPageTask letterPageTask =
48+
new ZdfLetterPageTask(this, createLetterPageUrls(), authKey);
4349
final Set<ZdfTopicUrlDto> topicUrls = forkJoinPool.submit(letterPageTask).get();
4450

4551
printMessage(
46-
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), topicUrls.size());
52+
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), topicUrls.size());
53+
54+
final ZdfPubFormTask pubFormTask = new ZdfPubFormTask(this, createPubFormUrls(), authKey);
55+
final Set<ZdfPubFormResult> pubFormUrls = forkJoinPool.submit(pubFormTask).get();
56+
57+
printMessage(
58+
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName() + " - PubForm:", pubFormUrls.size());
59+
60+
pubFormUrls.forEach(
61+
pubFormResult -> {
62+
topicUrls.addAll(pubFormResult.getTopics().getElements());
63+
shows.addAll(pubFormResult.getFilms());
64+
});
65+
printMessage(
66+
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName() + " - PubForm-Topics integrated: ", topicUrls.size());
4767

4868
ZdfTopicSeasonTask topicSeasonTask =
4969
new ZdfTopicSeasonTask(this, new ConcurrentLinkedQueue<>(topicUrls), authKey);
50-
final Set<ZdfFilmDto> shows = forkJoinPool.submit(topicSeasonTask).get();
70+
shows.addAll(forkJoinPool.submit(topicSeasonTask).get());
5171

5272
printMessage(
53-
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), shows.size());
73+
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), shows.size());
5474

5575
return new ZdfFilmTask(this, new ConcurrentLinkedQueue<>(shows), authKey);
5676
} catch (final InterruptedException ex) {
@@ -62,6 +82,17 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
6282
return null;
6383
}
6484

85+
private Queue<ZdfPubFormDto> createPubFormUrls() {
86+
Queue<ZdfPubFormDto> urls = new ConcurrentLinkedQueue<>();
87+
ZdfConstants.SPECIAL_COLLECTION_IDS.forEach((collectionId, topic) -> {
88+
final String url =
89+
ZdfUrlBuilder.buildTopicNoSeasonUrl(
90+
ZdfConstants.EPISODES_PAGE_SIZE, collectionId, ZdfConstants.NO_CURSOR);
91+
urls.add(new ZdfPubFormDto(topic, collectionId, url));
92+
});
93+
return urls;
94+
}
95+
6596
private Queue<ZdfLetterDto> createLetterPageUrls() {
6697
final Queue<ZdfLetterDto> urls = new ConcurrentLinkedQueue<>();
6798
for (int i = 0; i < MAX_LETTER_PAGEGS; i++) {

‎src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfPubFormDto.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package de.mediathekview.mserver.crawler.zdf;
2+
3+
import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
4+
5+
import java.util.Objects;
6+
7+
public class ZdfPubFormDto extends CrawlerUrlDTO {
8+
private final String topic;
9+
private final String collectionId;
10+
11+
public ZdfPubFormDto(String topic,String collectionId, String url) {
12+
super(url);
13+
this.topic = topic;
14+
this.collectionId = collectionId;
15+
}
16+
17+
public String getCollectionId() {
18+
return collectionId;
19+
}
20+
21+
22+
public String getTopic() {
23+
return topic;
24+
}
25+
26+
@Override
27+
public boolean equals(Object o) {
28+
if (o == null || getClass() != o.getClass()) return false;
29+
if (!super.equals(o)) return false;
30+
31+
ZdfPubFormDto that = (ZdfPubFormDto) o;
32+
return Objects.equals(topic, that.topic) && Objects.equals(collectionId, that.collectionId);
33+
}
34+
35+
@Override
36+
public int hashCode() {
37+
int result = super.hashCode();
38+
result = 31 * result + Objects.hashCode(topic);
39+
result = 31 * result + Objects.hashCode(collectionId);
40+
return result;
41+
}
42+
}

‎src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfPubFormResult.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package de.mediathekview.mserver.crawler.zdf;
2+
3+
import de.mediathekview.mserver.crawler.basic.PagedElementListDTO;
4+
5+
import java.util.HashSet;
6+
import java.util.Optional;
7+
import java.util.Set;
8+
9+
public class ZdfPubFormResult {
10+
private final PagedElementListDTO<ZdfTopicUrlDto> topics;
11+
private final Set<ZdfFilmDto> films;
12+
13+
public ZdfPubFormResult() {
14+
this.topics = new PagedElementListDTO<>();
15+
this.films = new HashSet<>();
16+
}
17+
18+
public PagedElementListDTO<ZdfTopicUrlDto> getTopics() {
19+
return topics;
20+
}
21+
22+
public Set<ZdfFilmDto> getFilms() {
23+
return films;
24+
}
25+
26+
public void addTopic(ZdfTopicUrlDto topicUrlDto) {
27+
this.topics.addElement(topicUrlDto);
28+
}
29+
30+
public void setNextPage(Optional<String> pageInfo) {
31+
this.topics.setNextPage(pageInfo);
32+
}
33+
34+
public void addFilms(Set<ZdfFilmDto> films) {
35+
this.films.addAll(films);
36+
}
37+
}

‎src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfPubFormDeserializer.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package de.mediathekview.mserver.crawler.zdf.json;
2+
3+
import com.google.gson.JsonArray;
4+
import com.google.gson.JsonDeserializationContext;
5+
import com.google.gson.JsonDeserializer;
6+
import com.google.gson.JsonElement;
7+
import com.google.gson.JsonObject;
8+
import com.google.gson.JsonParseException;
9+
import de.mediathekview.mserver.base.utils.JsonUtils;
10+
import de.mediathekview.mserver.crawler.zdf.ZdfConstants;
11+
import de.mediathekview.mserver.crawler.zdf.ZdfPubFormResult;
12+
import de.mediathekview.mserver.crawler.zdf.ZdfTopicUrlDto;
13+
import de.mediathekview.mserver.crawler.zdf.ZdfUrlBuilder;
14+
import java.lang.reflect.Type;
15+
import java.util.Optional;
16+
import org.apache.logging.log4j.LogManager;
17+
import org.apache.logging.log4j.Logger;
18+
19+
public class ZdfPubFormDeserializer extends ZdfTopicBaseClass
20+
implements JsonDeserializer<ZdfPubFormResult> {
21+
22+
@Override
23+
public ZdfPubFormResult deserialize(
24+
JsonElement json, Type typeOfT, JsonDeserializationContext context)
25+
throws JsonParseException {
26+
final ZdfPubFormResult result = new ZdfPubFormResult();
27+
JsonObject rootNode = json.getAsJsonObject();
28+
29+
JsonObject content = rootNode.getAsJsonObject("data").getAsJsonObject("metaCollectionContent");
30+
JsonArray nodes = content.getAsJsonArray("smartCollections");
31+
32+
for (JsonElement element : nodes) {
33+
JsonObject node = element.getAsJsonObject();
34+
final Optional<String> sender =
35+
JsonUtils.getElementValueAsString(node.getAsJsonObject("contentOwner"), "title");
36+
final Optional<String> topic = JsonUtils.getElementValueAsString(node, "title");
37+
final Optional<String> countSeasons = JsonUtils.getElementValueAsString(node, "countSeasons");
38+
if (ZdfConstants.PARTNER_TO_SENDER.containsKey(sender.orElse("ZDF"))) {
39+
if (countSeasons.isEmpty()) {
40+
final Optional<String> collectionType =
41+
JsonUtils.getElementValueAsString(node, "collectionType");
42+
if (collectionType.isPresent() && collectionType.get().equals("MOVIE")) {
43+
result.addFilms(deserializeMovie(element));
44+
}
45+
} else {
46+
for (int i = 0; i < Integer.parseInt(countSeasons.orElse("0")); i++) {
47+
String canonical = node.get("canonical").getAsString();
48+
result.addTopic(
49+
new ZdfTopicUrlDto(
50+
topic.orElse(""),
51+
i,
52+
canonical,
53+
ZdfUrlBuilder.buildTopicSeasonUrl(
54+
i, ZdfConstants.EPISODES_PAGE_SIZE, canonical)));
55+
}
56+
}
57+
}
58+
}
59+
60+
result.setNextPage(parseNextPage(content.getAsJsonObject("pageInfo")));
61+
62+
return result;
63+
}
64+
65+
private Optional<String> parseNextPage(JsonObject pageInfo) {
66+
if (!pageInfo.isJsonNull()) {
67+
final Optional<String> hasNextPage = JsonUtils.getAttributeAsString(pageInfo, "hasNextPage");
68+
if (hasNextPage.isPresent() && hasNextPage.get().equals("true")) {
69+
return JsonUtils.getAttributeAsString(pageInfo, "endCursor");
70+
}
71+
}
72+
return Optional.empty();
73+
}
74+
}

0 commit comments

Comments
 (0)