Skip to content

arte: fix rate limit problems on crawler1 #1120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alex1702 merged 5 commits into from
Jan 10, 2026
Merged

arte: fix rate limit problems on crawler1 #1120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5ae3a79
wait till SR+PHOENIX-Crawler finishes - fix necessary because SR+PHOE…
pidoubleyou Jan 10, 2026
ef8ac04
fix rate limiting problem with requests to api.arte.tv/opa
pidoubleyou Jan 10, 2026
ae69e69
filter video kind
pidoubleyou Jan 10, 2026
aac9742
reduce search deepth
pidoubleyou Jan 10, 2026
4f0c1bc
move arte.fr to step 1
pidoubleyou Jan 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/mServer/crawler/FilmeSuchen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public FilmeSuchen() {
}
if (crawlerList.contains("ARTE")) {
mediathekListe.add(new ArteCrawler(this, 0));
mediathekListe.add(new ArteCrawler_FR(this, 0));
mediathekListe.add(new ArteCrawler_FR(this, 1));
mediathekListe.add(new ArteCrawler_EN(this, 1));
mediathekListe.add(new ArteCrawler_ES(this, 1));
mediathekListe.add(new ArteCrawler_PL(this, 1));
Expand Down
43 changes: 18 additions & 25 deletions src/main/java/mServer/crawler/GetUrl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,26 +57,19 @@ public void setDelay(long delay, TimeUnit delayUnit) {
}

@Deprecated
public MSStringBuilder getUri_Utf(String sender, String addr, MSStringBuilder seite, String meldung) {
return getUri(sender, addr, StandardCharsets.UTF_8, 1 /* versuche */, seite, meldung);
public MSStringBuilder getUri_Utf(String runIdentifier, String sender, String addr, MSStringBuilder seite, String meldung) {
return getUri(runIdentifier, sender, addr, StandardCharsets.UTF_8, 1 /* versuche */, seite, meldung);
}

public MSStringBuilder getUri_Iso(String sender, String addr, MSStringBuilder seite, String meldung) {
return getUri(sender, addr, StandardCharsets.ISO_8859_1, 1 /* versuche */, seite, meldung);
public MSStringBuilder getUri_Iso(String runIdentifier, String sender, String addr, MSStringBuilder seite, String meldung) {
return getUri(runIdentifier, sender, addr, StandardCharsets.ISO_8859_1, 1 /* versuche */, seite, meldung);
}

public MSStringBuilder getUri(String sender, String addr, Charset encoding, int maxVersuche, MSStringBuilder seite, String meldung) {
return getUri(sender, addr, encoding, maxVersuche, seite, meldung, "");
public MSStringBuilder getUri(String runIdentifier, String sender, String addr, Charset encoding, int maxVersuche, MSStringBuilder seite, String meldung) {
return getUri(runIdentifier, sender, addr, encoding, maxVersuche, seite, meldung, "");
}

public MSStringBuilder getUriWithDelay(String sender, String addr, Charset encoding, int maxVersuche, MSStringBuilder seite, String meldung,
long delay, TimeUnit delayUnit) {
setDelay(delay, delayUnit);

return getUri(sender, addr, encoding, maxVersuche, seite, meldung);
}

public MSStringBuilder getUri(String sender, String addr, Charset encoding, int maxVersuche, MSStringBuilder seite, String meldung, String token) {
public MSStringBuilder getUri(String runIdentifier, String sender, String addr, Charset encoding, int maxVersuche, MSStringBuilder seite, String meldung, String token) {
int aktVer = 0;
boolean letzterVersuch;

Expand All @@ -96,22 +89,22 @@ public MSStringBuilder getUri(String sender, String addr, Charset encoding, int
TimeUnit.MILLISECONDS.sleep(PAUSE);
}
letzterVersuch = (aktVer >= maxVersuche);
seite = getUriNew(sender, addr, seite, encoding, meldung, maxVersuche, letzterVersuch, token);
seite = getUriNew(runIdentifier, sender, addr, seite, encoding, meldung, maxVersuche, letzterVersuch, token);
if (seite.length() > 0) {
// und nix wie weiter
if (Config.debug && aktVer > 1) {
String text = sender + " [" + aktVer + '/' + maxVersuche + "] ~~~> " + addr;
Log.sysLog(text);
}
// nur dann zählen
FilmeSuchen.listeSenderLaufen.inc(sender, RunSender.Count.ANZAHL);
FilmeSuchen.listeSenderLaufen.inc(runIdentifier, RunSender.Count.ANZAHL);
return seite;
} else {
FilmeSuchen.listeSenderLaufen.inc(sender, RunSender.Count.FEHLVERSUCHE);
FilmeSuchen.listeSenderLaufen.inc(sender, RunSender.Count.WARTEZEIT_FEHLVERSUCHE, delayVal);
FilmeSuchen.listeSenderLaufen.inc(runIdentifier, RunSender.Count.FEHLVERSUCHE);
FilmeSuchen.listeSenderLaufen.inc(runIdentifier, RunSender.Count.WARTEZEIT_FEHLVERSUCHE, delayVal);
if (letzterVersuch) {
// dann wars leider nichts
FilmeSuchen.listeSenderLaufen.inc(sender, RunSender.Count.FEHLER);
FilmeSuchen.listeSenderLaufen.inc(runIdentifier, RunSender.Count.FEHLER);
}
}
} catch (Exception ex) {
Expand All @@ -121,11 +114,11 @@ public MSStringBuilder getUri(String sender, String addr, Charset encoding, int
return seite;
}

private void updateStatistics(final String sender, final long bytesWritten) {
FilmeSuchen.listeSenderLaufen.inc(sender, RunSender.Count.SUM_DATA_BYTE, bytesWritten);
FilmeSuchen.listeSenderLaufen.inc(sender, RunSender.Count.SUM_TRAFFIC_BYTE, bytesWritten);
private void updateStatistics(final String runIdentifier, final long bytesWritten) {
FilmeSuchen.listeSenderLaufen.inc(runIdentifier, RunSender.Count.SUM_DATA_BYTE, bytesWritten);
FilmeSuchen.listeSenderLaufen.inc(runIdentifier, RunSender.Count.SUM_TRAFFIC_BYTE, bytesWritten);

FilmeSuchen.listeSenderLaufen.inc(sender, RunSender.Count.SUM_TRAFFIC_LOADART_NIX, bytesWritten);
FilmeSuchen.listeSenderLaufen.inc(runIdentifier, RunSender.Count.SUM_TRAFFIC_LOADART_NIX, bytesWritten);
}

private long transferData(ResponseBody body, Charset encoding, MSStringBuilder seite) throws IOException {
Expand Down Expand Up @@ -165,7 +158,7 @@ private Request.Builder createRequestBuilder(String addr, String token) {
return builder;
}

private MSStringBuilder getUriNew(String sender, String addr, MSStringBuilder seite,
private MSStringBuilder getUriNew(String runIdentifier, String sender, String addr, MSStringBuilder seite,
Charset encoding, String meldung, int versuch, boolean lVersuch,
String token) {

Expand All @@ -192,7 +185,7 @@ private MSStringBuilder getUriNew(String sender, String addr, MSStringBuilder se
Log.errorLog(973969801, ex, "");
}

updateStatistics(sender, load);
updateStatistics(runIdentifier, load);

return seite;
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/mServer/crawler/ListeRunSender.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
public class ListeRunSender extends LinkedList<RunSender> {

private final static String TRENNER = " | ";
private static final String SENDER = " Sender ";
private static final String SENDER = " Sender ";

public boolean listeFertig() {
// liefert true wenn alle Sender fertig sind
Expand Down Expand Up @@ -151,7 +151,7 @@ public ArrayList<String> getTextCount(ArrayList<String> ret) {

public void getTextSum(ArrayList<String> retArray) {
//wird ausgeführt wenn Sender beendet ist
final String[] titel1 = {" Sender ", " [min] ", " [kB/s] ", "s/Seite", "Threads", "Wait"};
final String[] titel1 = {" Sender ", " [min] ", " [kB/s] ", "s/Seite", "Threads", "Wait"};
String zeile = "";
String[] names = new String[titel1.length];
for (int i = 0; i < titel1.length; ++i) {
Expand Down
Loading