vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Use a proper HTML parser with BS and catch exceptions

Browse source 
Fixes #473
This commit is contained in:
Joseph Schorr 2015-09-10 16:14:29 -04:00
parent 14107893a6
commit 6ca33ca108
1 changed files with 8 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
util/seo.py
View file

@ -32,11 +32,13 @@ def render_snapshot(url):
# Remove script tags # Remove script tags
logger.info('Removing script tags: %s' % url) logger.info('Removing script tags: %s' % url)
soup = BeautifulSoup(out_html.decode('utf8')) try:
soup = BeautifulSoup(out_html.decode('utf8'), 'html.parser')
to_extract = soup.findAll('script') to_extract = soup.findAll('script')
for item in to_extract: for item in to_extract:
item.extract() item.extract()
except:
logger.info('Snapshotted url: %s' % url) logger.exception('Exception when trying to parse served HTML')
return out_html
return str(soup) return str(soup)