vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add automatic retry to external lib downloader

Browse source
This commit is contained in:
Joseph Schorr 2019-02-26 16:21:24 -05:00
parent 07fccb8703
commit ab3a1983f9
1 changed files with 29 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
external_libraries.py
View file

@ -1,10 +1,17 @@
import logging
import logging.config
import urllib2 import urllib2
import re import re
import os import os
from _init import STATIC_FONTS_DIR, STATIC_LDN_DIR from _init import STATIC_FONTS_DIR, STATIC_LDN_DIR
from util.log import logfile_path
LOCAL_PATH = '/static/ldn/' LOCAL_PATH = '/static/ldn/'
MAX_RETRY_COUNT = 3
EXTERNAL_JS = [ EXTERNAL_JS = [
'code.jquery.com/jquery.js', 'code.jquery.com/jquery.js',
'netdna.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js', 'netdna.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js',
@ -56,6 +63,9 @@ EXTERNAL_CSS_FONTS = [
] ]
logger = logging.getLogger(__name__)
def get_external_javascript(local=False): def get_external_javascript(local=False):
if local: if local:
return [LOCAL_PATH + format_local_name(src) for src in EXTERNAL_JS] return [LOCAL_PATH + format_local_name(src) for src in EXTERNAL_JS]
@ -82,11 +92,24 @@ def format_local_name(url):
return filename return filename
def _download_url(url):
for index in range(0, MAX_RETRY_COUNT):
try:
response = urllib2.urlopen(url)
return response.read()
except urllib2.URLError:
logger.exception('Got exception when trying to download URL %s (try #%s)', url, index + 1)
raise Exception('Aborted due to maximum retries reached')
if __name__ == '__main__': if __name__ == '__main__':
logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False)
for url in EXTERNAL_JS + EXTERNAL_CSS: for url in EXTERNAL_JS + EXTERNAL_CSS:
print 'Downloading %s' % url print 'Downloading %s' % url
response = urllib2.urlopen('https://' + url) contents = _download_url('https://' + url)
contents = response.read()
filename = format_local_name(url) filename = format_local_name(url)
print 'Writing %s' % filename print 'Writing %s' % filename
@ -95,16 +118,16 @@ if __name__ == '__main__':
for url in EXTERNAL_CSS_FONTS: for url in EXTERNAL_CSS_FONTS:
print 'Downloading %s' % url print 'Downloading %s' % url
response = urllib2.urlopen('https://' + url) contents = _download_url('https://' + url)
filename = os.path.basename(url).split('?')[0] filename = os.path.basename(url).split('?')[0]
with open(STATIC_LDN_DIR + filename, "wb") as local_file: with open(STATIC_LDN_DIR + filename, "wb") as local_file:
local_file.write(response.read()) local_file.write(contents)
for url in EXTERNAL_FONTS: for url in EXTERNAL_FONTS:
print 'Downloading %s' % url print 'Downloading %s' % url
response = urllib2.urlopen('https://' + url) contents = _download_url('https://' + url)
filename = os.path.basename(url).split('?')[0] filename = os.path.basename(url).split('?')[0]
with open(STATIC_FONTS_DIR + filename, "wb") as local_file: with open(STATIC_FONTS_DIR + filename, "wb") as local_file:
local_file.write(response.read()) local_file.write(contents)