vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Use a proper HTML parser with BS and catch exceptions

Browse source 
Fixes #473
This commit is contained in:
Joseph Schorr 2015-09-10 16:14:29 -04:00
parent 14107893a6
commit 6ca33ca108
1 changed files with 8 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
util/seo.py
View file

@ -32,11 +32,13 @@ def render_snapshot(url):
# Remove script tags # Remove script tags
logger.info('Removing script tags: %s' % url) logger.info('Removing script tags: %s' % url)
soup = BeautifulSoup(out_html.decode('utf8')) try:
to_extract = soup.findAll('script') soup = BeautifulSoup(out_html.decode('utf8'), 'html.parser')
for item in to_extract: to_extract = soup.findAll('script')
item.extract() for item in to_extract:
item.extract()
logger.info('Snapshotted url: %s' % url) except:
logger.exception('Exception when trying to parse served HTML')
return out_html
return str(soup) return str(soup)