add error handler by internal request
This commit is contained in:
parent
53609fbb90
commit
e42f68947e
15
main.py
15
main.py
|
@ -15,6 +15,7 @@ class NewsScraper:
|
|||
self.content_selector = content_selector
|
||||
|
||||
def scrape(self):
|
||||
try:
|
||||
if self.enable_js:
|
||||
session = HTMLSession()
|
||||
response = session.get(self.url)
|
||||
|
@ -69,8 +70,18 @@ class NewsScraper:
|
|||
news_items.append(news_item_html)
|
||||
|
||||
return news_items
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"Error fetching URL: {e}")
|
||||
return []
|
||||
except AttributeError as e:
|
||||
print(f"Error parsing HTML: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"Unexpected error: {e}")
|
||||
return []
|
||||
|
||||
def write_to_html(self, template_path, output_path, news_items, heading):
|
||||
try:
|
||||
with open(template_path, 'r') as f:
|
||||
html_template = f.read()
|
||||
|
||||
|
@ -79,6 +90,10 @@ class NewsScraper:
|
|||
|
||||
with open(output_path, 'a') as f:
|
||||
f.write(html_output)
|
||||
except IOError as e:
|
||||
print(f"Error writing to file: {e}")
|
||||
except Exception as e:
|
||||
print(f"Unexpected error: {e}")
|
||||
|
||||
# Define the scrapers for each news source
|
||||
|
||||
|
|
Loading…
Reference in a new issue