vbatts/microblog.pub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

More url checking

Browse source
This commit is contained in:
Thomas Sileo 2018-05-26 00:03:30 +02:00
parent 06f4f824d8
commit a3267971e8
3 changed files with 10 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
utils/opengraph.py
View file

@ -5,7 +5,7 @@ import opengraph
import requests
from bs4 import BeautifulSoup
from .urlutils import is_url_valid
from .urlutils import is_url_valid, check_url
def links_from_note(note):
@ -38,6 +38,7 @@ def fetch_og_metadata(user_agent, col, remote_id):
# FIXME(tsileo): set the user agent by giving HTML directly to OpenGraph
htmls = []
for l in links:
check_url(l)
r = requests.get(l, headers={'User-Agent': user_agent})
r.raise_for_status()
htmls.append(r.text)