Skip to content

Commit b3e92bc

Browse files
author
root
committed
Merge branch 'main' of https://github.com/cabcode-id/TALAS 
2 parents 3aff147 + 4cf5e73 commit b3e92bc

10 files changed

Lines changed: 776 additions & 29 deletions

File tree

‎Development/crawler/crawling-news/antara.py‎

Lines changed: 114 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,67 @@
33
import re
44
import time
55
from bs4 import BeautifulSoup
6-
from datetime import datetime
6+
from datetime import datetime, timezone, timedelta
77

88
# Tanggal yang dimasukkan ke csv.
99
def get_raw_date():
1010
return datetime.today().strftime("%Y-%m-%d")
1111

1212
# Membuat dictionary berisi judul dan link yang dibuat beberapa jam lalu atau kemarin.
1313
def find_links(url):
14+
print(f"Crawling URL: {url}")
1415
response = requests.get(url)
1516
soup = BeautifulSoup(response.content, 'html.parser')
1617

1718
divs = soup.find_all('div', class_='col-md-8')
1819

1920
span_elements = []
2021

22+
# Mengambil waktu saat ini dalam zona waktu Jakarta (GMT+7)
23+
current_hour = datetime.now(timezone(timedelta(hours=7))).hour
24+
# Menghitung jam yang sudah berlalu dari jam saat ini untuk crawl dari awal hari ini.
25+
hour_patterns = [f"\\b{i} jam lalu\\b" for i in range(1, current_hour + 1)]
26+
minute_pattern = "\\b\\d+ menit lalu\\b"
27+
time_pattern = f"{minute_pattern}|{('|'.join(hour_patterns))}"
28+
2129
for div in divs:
22-
spans = div.find_all('span', string=re.compile(r'jam|kemarin', re.IGNORECASE))
30+
spans = div.find_all('span', string=re.compile(time_pattern, re.IGNORECASE))
2331
span_elements.extend(spans)
2432

2533
links_dict = {}
2634
for span in span_elements:
2735
parent = span.find_parent()
36+
37+
# Check if this span is related to content we want to filter out
38+
should_skip = False
39+
checking_element = span
40+
for _ in range(5): # Check up to 5 levels up to find slugs
41+
if not checking_element:
42+
break
43+
44+
# Check if there's a slug element
45+
slug_element = checking_element.find_previous('h4', class_='slug')
46+
if slug_element:
47+
slug_link = slug_element.find('a')
48+
if slug_link and 'video' in slug_link.text.lower():
49+
should_skip = True
50+
break
51+
52+
checking_element = checking_element.parent
53+
54+
if should_skip:
55+
continue
56+
2857
while parent:
2958
link = parent.find('a', href=True)
3059
if link:
60+
# Skip link yang ada slug.
61+
if link.find_parent('h4', class_='slug'):
62+
break
63+
3164
title = link.get('title', '')
32-
links_dict[title] = link['href']
65+
if title: # Only add non-empty titles
66+
links_dict[title] = link['href']
3367
break
3468
parent = parent.find_parent()
3569
return links_dict
@@ -38,7 +72,7 @@ def find_links(url):
3872
def collect_articles(links_dict):
3973
data = []
4074
raw_date = get_raw_date()
41-
source = "antara"
75+
source = "Antara"
4276
for title, link in links_dict.items():
4377
try:
4478
get_content = requests.get(link, timeout=10)
@@ -58,13 +92,43 @@ def find_content(url):
5892
response = requests.get(url)
5993
soup = BeautifulSoup(response.content, 'html.parser')
6094

61-
# Extract image URL
95+
# Find the image with the largest area instead of by class
6296
image_url = ""
63-
image_div = soup.find('div', class_='wrap__article-detail-image')
64-
if image_div:
65-
img_tag = image_div.find('img', class_='img-fluid')
66-
if img_tag and img_tag.has_attr('src'):
67-
image_url = img_tag['src']
97+
max_area = 0
98+
for img in soup.find_all('img'):
99+
# Skip small icons, emoticons etc
100+
if not img.has_attr('src') or img['src'].startswith('data:'):
101+
continue
102+
103+
# Get width and height attributes
104+
width_str = img.get('width', '0')
105+
height_str = img.get('height', '0')
106+
107+
# Remove 'px' suffix if present
108+
width_str = width_str.replace('px', '') if isinstance(width_str, str) else width_str
109+
height_str = height_str.replace('px', '') if isinstance(height_str, str) else height_str
110+
111+
try:
112+
width = int(width_str) or int(re.search(r'width:\s*(\d+)px', img.get('style', '') or '') or [0, 0])[1]
113+
height = int(height_str) or int(re.search(r'height:\s*(\d+)px', img.get('style', '') or '') or [0, 0])[1]
114+
except (ValueError, TypeError):
115+
width, height = 0, 0
116+
117+
# Calculate area
118+
area = width * height
119+
120+
# Update max area and image URL if this image is larger
121+
if area > max_area:
122+
max_area = area
123+
image_url = img['src']
124+
125+
# If no area detected images, fallback to traditional method
126+
if not image_url:
127+
image_div = soup.find('div', class_='wrap__article-detail-image mt-4')
128+
if image_div:
129+
img_tag = image_div.find('img', class_='img-fluid')
130+
if img_tag and img_tag.has_attr('src'):
131+
image_url = img_tag['src']
68132

69133
parent_p_count = {}
70134
# Mencari semua parent yang memiliki p. Jumlah p yang tertinggi = main content yang ingin diambil.
@@ -104,9 +168,41 @@ def extract_content(text):
104168
return match.group(2).strip() if match else text
105169

106170
def crawl_antara():
107-
url = 'https://www.antaranews.com/tag/cek-fakta/'
108-
links_dict = find_links(url)
109-
data = collect_articles(links_dict)
171+
all_links = {}
172+
page = 1
173+
max_pages = 10 # Set a reasonable maximum to avoid infinite crawling
174+
consecutive_empty_pages = 0
175+
176+
while page <= max_pages:
177+
url = f'https://www.antaranews.com/terkini/{page}'
178+
try:
179+
links_dict = find_links(url)
180+
181+
# If no articles found or we got an empty dictionary
182+
if not links_dict:
183+
consecutive_empty_pages += 1
184+
print(f"No articles found on page {page}")
185+
186+
# Stop after 2 consecutive empty pages
187+
if consecutive_empty_pages >= 2:
188+
print("Reached end of articles, stopping crawler")
189+
break
190+
else:
191+
consecutive_empty_pages = 0 # Reset counter when we find articles
192+
all_links.update(links_dict)
193+
print(f"Found {len(links_dict)} articles on page {page}")
194+
195+
# Print each article title and link
196+
for title, link in links_dict.items():
197+
print(f" - {title}: {link}")
198+
199+
page += 1
200+
except requests.exceptions.RequestException as e:
201+
print(f"Error accessing page {page}: {e}")
202+
break
203+
204+
# Process all collected links
205+
data = collect_articles(all_links)
110206

111207
# Format data for return
112208
formatted_data = []
@@ -126,4 +222,8 @@ def main():
126222
return crawl_antara()
127223

128224
if __name__ == "__main__":
129-
main()
225+
articles = main()
226+
titles = [article['title'] for article in articles]
227+
print(titles)
228+
229+

‎Development/crawler/crawling-news/antarahoax.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
import csv
2+
import requests
3+
import re
4+
import time
5+
from bs4 import BeautifulSoup
6+
from datetime import datetime
7+
8+
# Tanggal yang dimasukkan ke csv.
9+
def get_raw_date():
10+
return datetime.today().strftime("%Y-%m-%d")
11+
12+
# Membuat dictionary berisi judul dan link yang dibuat beberapa jam lalu atau kemarin.
13+
def find_links(url):
14+
response = requests.get(url)
15+
soup = BeautifulSoup(response.content, 'html.parser')
16+
17+
divs = soup.find_all('div', class_='col-md-8')
18+
19+
span_elements = []
20+
21+
for div in divs:
22+
spans = div.find_all('span', string=re.compile(r'jam|kemarin', re.IGNORECASE))
23+
span_elements.extend(spans)
24+
25+
links_dict = {}
26+
for span in span_elements:
27+
parent = span.find_parent()
28+
while parent:
29+
link = parent.find('a', href=True)
30+
if link:
31+
title = link.get('title', '')
32+
links_dict[title] = link['href']
33+
break
34+
parent = parent.find_parent()
35+
return links_dict
36+
37+
# Mengumpulkan artikel dari link yang ditemukan.
38+
def collect_articles(links_dict):
39+
data = []
40+
raw_date = get_raw_date()
41+
source = "antara"
42+
for title, link in links_dict.items():
43+
try:
44+
get_content = requests.get(link, timeout=10)
45+
get_content.raise_for_status()
46+
date = raw_date
47+
content, image_url = find_content(link)
48+
# Process content directly
49+
processed_content = extract_content(content)
50+
data.append([title, source, link, image_url, date, processed_content])
51+
except requests.exceptions.RequestException as e:
52+
print(f"Error fetching content for {link}: {e}")
53+
time.sleep(2)
54+
return data
55+
56+
# Mencari konten dari link berita.
57+
def find_content(url):
58+
response = requests.get(url)
59+
soup = BeautifulSoup(response.content, 'html.parser')
60+
61+
# Extract image URL
62+
image_url = ""
63+
image_div = soup.find('div', class_='wrap__article-detail-image')
64+
if image_div:
65+
img_tag = image_div.find('img', class_='img-fluid')
66+
if img_tag and img_tag.has_attr('src'):
67+
image_url = img_tag['src']
68+
69+
parent_p_count = {}
70+
# Mencari semua parent yang memiliki p. Jumlah p yang tertinggi = main content yang ingin diambil.
71+
for p in soup.find_all('p'):
72+
# Skip paragraphs that have any class attribute
73+
if p.has_attr('class'):
74+
continue
75+
# Skip paragraphs that contain a span with class 'baca-juga'
76+
if p.find('span', class_='baca-juga'):
77+
continue
78+
79+
parent = p.find_parent()
80+
if parent:
81+
parent_p_count[parent] = parent_p_count.get(parent, 0) + 1
82+
83+
content_text = ""
84+
if parent_p_count:
85+
max_parent = max(parent_p_count, key=parent_p_count.get)
86+
paragraphs = []
87+
for p in max_parent.find_all('p'):
88+
if p.has_attr('class'):
89+
continue
90+
if p.find('span', class_='baca-juga'):
91+
continue
92+
text = p.get_text(" ", strip=True)
93+
paragraphs.append(text)
94+
content_text = " ".join(paragraphs)
95+
96+
return content_text, image_url
97+
98+
def extract_content(text):
99+
text = text.lower()
100+
start_phrases = ["jakarta (antara) - "]
101+
start_pattern = "|".join(map(re.escape, start_phrases))
102+
pattern = f"({start_pattern})(.*)"
103+
match = re.search(pattern, text, re.DOTALL)
104+
return match.group(2).strip() if match else text
105+
106+
def crawl_antara():
107+
url = 'https://www.antaranews.com/tag/cek-fakta/'
108+
links_dict = find_links(url)
109+
data = collect_articles(links_dict)
110+
111+
# Format data for return
112+
formatted_data = []
113+
for row in data:
114+
formatted_data.append({
115+
'title': row[0],
116+
'source': row[1],
117+
'url': row[2],
118+
'image': row[3],
119+
'date': row[4],
120+
'content': row[5]
121+
})
122+
123+
return formatted_data
124+
125+
def main():
126+
return crawl_antara()
127+
128+
if __name__ == "__main__":
129+
main()

Development/crawler/crawling-news/antara2.py renamed to Development/crawler/crawling-news/antarapantai.py

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,17 @@ def find_links(url):
1919

2020
span_elements = []
2121

22-
# Mengambil waktu saat ini dalam zona waktu Jakarta (GMT+7)
23-
current_hour = datetime.now(timezone(timedelta(hours=7))).hour
24-
# Menghitung jam yang sudah berlalu dari jam saat ini untuk crawl dari awal hari ini.
25-
hour_patterns = [f"\\b{i} jam lalu\\b" for i in range(1, current_hour + 1)]
26-
minute_pattern = "\\b\\d+ menit lalu\\b"
27-
time_pattern = f"{minute_pattern}|{('|'.join(hour_patterns))}"
22+
# G pake range waktu dulu
23+
# # Mengambil waktu saat ini dalam zona waktu Jakarta (GMT+7)
24+
# current_hour = datetime.now(timezone(timedelta(hours=7))).hour
25+
# # Menghitung jam yang sudah berlalu dari jam saat ini untuk crawl dari awal hari ini.
26+
# hour_patterns = [f"\\b{i} jam lalu\\b" for i in range(1, current_hour + 1)]
27+
# minute_pattern = "\\b\\d+ menit lalu\\b"
28+
# time_pattern = f"{minute_pattern}|{('|'.join(hour_patterns))}"
2829

2930
for div in divs:
30-
spans = div.find_all('span', string=re.compile(time_pattern, re.IGNORECASE))
31+
# spans = div.find_all('span', string=re.compile(time_pattern, re.IGNORECASE))
32+
spans = div.find_all('span')
3133
span_elements.extend(spans)
3234

3335
links_dict = {}
@@ -72,7 +74,7 @@ def find_links(url):
7274
def collect_articles(links_dict):
7375
data = []
7476
raw_date = get_raw_date()
75-
source = "Antara"
77+
source = "Antara Sanur"
7678
for title, link in links_dict.items():
7779
try:
7880
get_content = requests.get(link, timeout=10)
@@ -174,7 +176,7 @@ def crawl_antara():
174176
consecutive_empty_pages = 0
175177

176178
while page <= max_pages:
177-
url = f'https://www.antaranews.com/terkini/{page}'
179+
url = f'https://www.antaranews.com/tag/pantai-sanur-bali/{page}'
178180
try:
179181
links_dict = find_links(url)
180182

0 commit comments

Comments
 (0)