1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
|
from bs4 import BeautifulSoup
from datetime import datetime
from urllib.parse import urljoin
def parse_exceed_gear_news_site(html: str, base_url: str):
soup = BeautifulSoup(html, 'html.parser')
news_list = soup.select('.tab ul.news li')
entries = []
for li in news_list:
date = li.select_one('strong')
pre = li.select_one('pre')
if not date or not pre:
continue
date_str = date.text.strip()
try:
dt = datetime.strptime(date_str, "%Y.%m.%d")
timestamp = int(dt.timestamp())
except ValueError:
timestamp = None
headline = li.select_one('p.notice')
headline_text = headline.text.strip() if headline else None
for tag in pre.select('font, b, u, span'):
tag.unwrap()
content = pre.get_text(separator='\n', strip=True)
images = []
for img in pre.select('img'):
src = img.get('data-original') or img.get('src')
if not src or src.startswith('data:'):
continue
src = urljoin(base_url, src)
parent = img.find_parent('a')
href = urljoin(base_url, parent['href']) if parent and parent.has_attr('href') else None
images.append({'image': src, 'link': href})
entries.append({
'date': date_str,
'identifier': 'SOUND_VOLTEX_EXCEED_GEAR',
'type': None,
'timestamp': timestamp,
'headline': headline_text,
'content': content,
"url": None,
'images': images
})
return entries
|
