1
0
Fork 0
forked from Chay/scrp

add error handler by internal request

This commit is contained in:
lelkins 2024-04-20 14:07:34 +03:00
parent 53609fbb90
commit e42f68947e

15
main.py
View file

@ -15,6 +15,7 @@ class NewsScraper:
self.content_selector = content_selector
def scrape(self):
try:
if self.enable_js:
session = HTMLSession()
response = session.get(self.url)
@ -69,8 +70,18 @@ class NewsScraper:
news_items.append(news_item_html)
return news_items
except requests.exceptions.RequestException as e:
print(f"Error fetching URL: {e}")
return []
except AttributeError as e:
print(f"Error parsing HTML: {e}")
return []
except Exception as e:
print(f"Unexpected error: {e}")
return []
def write_to_html(self, template_path, output_path, news_items, heading):
try:
with open(template_path, 'r') as f:
html_template = f.read()
@ -79,6 +90,10 @@ class NewsScraper:
with open(output_path, 'a') as f:
f.write(html_output)
except IOError as e:
print(f"Error writing to file: {e}")
except Exception as e:
print(f"Unexpected error: {e}")
# Define the scrapers for each news source