vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

initial import for Open Source 🎉

Browse source
This commit is contained in:
Jimmy Zelinskie 2019-11-12 11:09:47 -05:00
parent 1898c361f3
commit 9c0dd3b722
2048 changed files with 218743 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

71
conf/init/02_get_kube_certs.py Normal file
View file

@ -0,0 +1,71 @@
import json
import os
import base64
from requests import Request, Session
QUAYPATH = os.environ.get('QUAYPATH', '.')
KUBE_EXTRA_CA_CERTDIR = os.environ.get('KUBE_EXTRA_CA_CERTDIR', '%s/conf/kube_extra_certs' % QUAYPATH)
KUBERNETES_API_HOST = os.environ.get('KUBERNETES_SERVICE_HOST', '')
port = os.environ.get('KUBERNETES_SERVICE_PORT')
if port:
KUBERNETES_API_HOST += ':' + port
SERVICE_ACCOUNT_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token'
QE_NAMESPACE = os.environ.get('QE_K8S_NAMESPACE', 'quay-enterprise')
QE_CONFIG_SECRET = os.environ.get('QE_K8S_CONFIG_SECRET', 'quay-enterprise-config-secret')
EXTRA_CA_DIRECTORY_PREFIX = 'extra_ca_certs_'
def _lookup_secret(service_token):
secret_url = 'namespaces/%s/secrets/%s' % (QE_NAMESPACE, QE_CONFIG_SECRET)
response = _execute_k8s_api(service_token, 'GET', secret_url)
if response.status_code != 200:
raise Exception('Cannot get the config secret')
return json.loads(response.text)
def _execute_k8s_api(service_account_token, method, relative_url, data=None, api_prefix='api/v1', content_type='application/json'):
headers = {
'Authorization': 'Bearer ' + service_account_token
}
if data:
headers['Content-Type'] = content_type
data = json.dumps(data) if data else None
session = Session()
url = 'https://%s/%s/%s' % (KUBERNETES_API_HOST, api_prefix, relative_url)
request = Request(method, url, data=data, headers=headers)
return session.send(request.prepare(), verify=False, timeout=2)
def is_extra_cert(key):
return key.find(EXTRA_CA_DIRECTORY_PREFIX) == 0
def main():
# Load the service account token from the local store.
if not os.path.exists(SERVICE_ACCOUNT_TOKEN_PATH):
raise Exception('Cannot load Kubernetes service account token')
with open(SERVICE_ACCOUNT_TOKEN_PATH, 'r') as f:
service_token = f.read()
secret_data = _lookup_secret(service_token).get('data', {})
cert_keys = filter(is_extra_cert, secret_data.keys())
for cert_key in cert_keys:
if not os.path.exists(KUBE_EXTRA_CA_CERTDIR):
os.mkdir(KUBE_EXTRA_CA_CERTDIR)
cert_value = base64.b64decode(secret_data[cert_key])
cert_filename = cert_key.replace(EXTRA_CA_DIRECTORY_PREFIX, '')
print "Found an extra cert %s in config-secret, copying to kube ca dir"
with open(os.path.join(KUBE_EXTRA_CA_CERTDIR, cert_filename), 'w') as f:
f.write(cert_value)
if __name__ == '__main__':
main()

12
conf/init/02_get_kube_certs.sh Executable file
View file

@ -0,0 +1,12 @@
#!/usr/bin/env bash
QUAYDIR=${QUAYDIR:-"/"}
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd $QUAYDIR
if [[ "$KUBERNETES_SERVICE_HOST" != "" ]];then
echo "Running on kubernetes, attempting to retrieve extra certs from secret"
python $QUAYCONF/init/02_get_kube_certs.py
fi

0
conf/init/__init__.py Normal file
View file

15
conf/init/certs_create.sh Executable file
View file

@ -0,0 +1,15 @@
#! /bin/bash
set -e
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd ${QUAYDIR:-"/"}
SYSTEM_CERTDIR=${SYSTEM_CERTDIR:-"/etc/pki/ca-trust/source/anchors"}
# Create certs for jwtproxy to mitm outgoing TLS connections
# echo '{"CN":"CA","key":{"algo":"rsa","size":2048}}' | cfssl gencert -initca - | cfssljson -bare mitm
mkdir -p /certificates; cd /certificates
openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \
-subj "/C=US/ST=NY/L=NYC/O=Dis/CN=self-signed" \
-keyout mitm-key.pem -out mitm.pem
cp /certificates/mitm-key.pem $QUAYCONF/mitm.key
cp /certificates/mitm.pem $QUAYCONF/mitm.cert
cp /certificates/mitm.pem $SYSTEM_CERTDIR/mitm.crt

52
conf/init/certs_install.sh Executable file
View file

@ -0,0 +1,52 @@
#! /bin/bash
set -e
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
QUAYCONFIG=${QUAYCONFIG:-"$QUAYCONF/stack"}
CERTDIR=${CERTDIR:-"$QUAYCONFIG/extra_ca_certs"}
SYSTEM_CERTDIR=${SYSTEM_CERTDIR:-"/etc/pki/ca-trust/source/anchors"}
PYTHON_ROOT=${PYTHON_ROOT:-"/opt/rh/python27/root/usr/lib/python2.7"}
# If we're running under kube, the previous script (02_get_kube_certs.sh) will put the certs in a different location
if [[ "$KUBERNETES_SERVICE_HOST" != "" ]];then
CERTDIR=${KUBE_EXTRA_CA_CERTDIR:-"$QUAYPATH/conf/kube_extra_certs"}
fi
cd ${QUAYDIR:-"/quay-registry"}
# Add the custom LDAP certificate
if [ -e $QUAYCONFIG/ldap.crt ]
then
cp $QUAYCONFIG/ldap.crt ${SYSTEM_CERTDIR}/ldap.crt
fi
# Add extra trusted certificates (as a directory)
if [ -d $CERTDIR ]; then
if test "$(ls -A "$CERTDIR")"; then
echo "Installing extra certificates found in $CERTDIR directory"
cp $CERTDIR/* ${SYSTEM_CERTDIR}
cat $CERTDIR/* >> $PYTHON_ROOT/site-packages/requests/cacert.pem
cat $CERTDIR/* >> $PYTHON_ROOT/site-packages/certifi/cacert.pem
fi
fi
# Add extra trusted certificates (as a file)
if [ -f $CERTDIR ]; then
echo "Installing extra certificates found in $CERTDIR file"
csplit -z -f ${SYSTEM_CERTDIR}/extra-ca- $CERTDIR '/-----BEGIN CERTIFICATE-----/' '{*}'
cat $CERTDIR >> $PYTHON_ROOT/site-packages/requests/cacert.pem
cat $CERTDIR >> $PYTHON_ROOT/site-packages/certifi/cacert.pem
fi
# Add extra trusted certificates (prefixed)
for f in $(find $QUAYCONFIG/ -maxdepth 1 -type f -name "extra_ca*")
do
echo "Installing extra cert $f"
cp "$f" ${SYSTEM_CERTDIR}
cat "$f" >> $PYTHON_ROOT/site-packages/requests/cacert.pem
cat "$f" >> $PYTHON_ROOT/site-packages/certifi/cacert.pem
done
# Update all CA certificates.
update-ca-trust extract

16
conf/init/copy_config_files.sh Executable file
View file

@ -0,0 +1,16 @@
#! /bin/sh
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd ${QUAYDIR:-"/"}
if [ -e $QUAYCONF/stack/robots.txt ]
then
cp $QUAYCONF/stack/robots.txt $QUAYPATH/templates/robots.txt
fi
if [ -e $QUAYCONF/stack/favicon.ico ]
then
cp $QUAYCONF/stack/favicon.ico $QUAYPATH/static/favicon.ico
fi

42
conf/init/logrotate.conf Normal file
View file

@ -0,0 +1,42 @@
#
# This file exists because of a bug in phusion/baseimage:0.9.19 where the su
# directive below is configured to use the nonexistant syslog user.
#
# see "man logrotate" for details
# rotate log files weekly
weekly
# use the syslog group by default, since this is the owning group
# of /var/log/syslog.
su root root
# keep 4 weeks worth of backlogs
rotate 4
# create new (empty) log files after rotating old ones
create
# uncomment this if you want your log files compressed
#compress
# packages drop log rotation information into this directory
include /etc/logrotate.d
# no packages own wtmp, or btmp -- we'll rotate them here
/var/log/wtmp {
missingok
monthly
create 0664 root utmp
rotate 1
}
/var/log/btmp {
missingok
monthly
create 0660 root utmp
rotate 1
}
# system-specific logs may be configured here

126
conf/init/nginx_conf_create.py Normal file
View file

@ -0,0 +1,126 @@
import os
import os.path
import yaml
import jinja2
QUAYPATH = os.getenv("QUAYPATH", ".")
QUAYDIR = os.getenv("QUAYDIR", "/")
QUAYCONF_DIR = os.getenv("QUAYCONF", os.path.join(QUAYDIR, QUAYPATH, "conf"))
STATIC_DIR = os.path.join(QUAYDIR, 'static')
SSL_PROTOCOL_DEFAULTS = ['TLSv1', 'TLSv1.1', 'TLSv1.2']
SSL_CIPHER_DEFAULTS = [
'ECDHE-RSA-AES128-GCM-SHA256',
'ECDHE-ECDSA-AES128-GCM-SHA256',
'ECDHE-RSA-AES256-GCM-SHA384',
'ECDHE-ECDSA-AES256-GCM-SHA384',
'DHE-RSA-AES128-GCM-SHA256',
'DHE-DSS-AES128-GCM-SHA256',
'kEDH+AESGCM',
'ECDHE-RSA-AES128-SHA256',
'ECDHE-ECDSA-AES128-SHA256',
'ECDHE-RSA-AES128-SHA',
'ECDHE-ECDSA-AES128-SHA',
'ECDHE-RSA-AES256-SHA384',
'ECDHE-ECDSA-AES256-SHA384',
'ECDHE-RSA-AES256-SHA',
'ECDHE-ECDSA-AES256-SHA',
'DHE-RSA-AES128-SHA256',
'DHE-RSA-AES128-SHA',
'DHE-DSS-AES128-SHA256',
'DHE-RSA-AES256-SHA256',
'DHE-DSS-AES256-SHA',
'DHE-RSA-AES256-SHA',
'AES128-GCM-SHA256',
'AES256-GCM-SHA384',
'AES128-SHA256',
'AES256-SHA256',
'AES128-SHA',
'AES256-SHA',
'AES',
'CAMELLIA',
'!3DES',
'!aNULL',
'!eNULL',
'!EXPORT',
'!DES',
'!RC4',
'!MD5',
'!PSK',
'!aECDH',
'!EDH-DSS-DES-CBC3-SHA',
'!EDH-RSA-DES-CBC3-SHA',
'!KRB5-DES-CBC3-SHA',
]
def write_config(filename, **kwargs):
with open(filename + ".jnj") as f:
template = jinja2.Template(f.read())
rendered = template.render(kwargs)
with open(filename, 'w') as f:
f.write(rendered)
def generate_nginx_config(config):
"""
Generates nginx config from the app config
"""
config = config or {}
use_https = os.path.exists(os.path.join(QUAYCONF_DIR, 'stack/ssl.key'))
use_old_certs = os.path.exists(os.path.join(QUAYCONF_DIR, 'stack/ssl.old.key'))
v1_only_domain = config.get('V1_ONLY_DOMAIN', None)
enable_rate_limits = config.get('FEATURE_RATE_LIMITS', False)
ssl_protocols = config.get('SSL_PROTOCOLS', SSL_PROTOCOL_DEFAULTS)
ssl_ciphers = config.get('SSL_CIPHERS', SSL_CIPHER_DEFAULTS)
write_config(os.path.join(QUAYCONF_DIR, 'nginx/nginx.conf'), use_https=use_https,
use_old_certs=use_old_certs,
enable_rate_limits=enable_rate_limits,
v1_only_domain=v1_only_domain,
ssl_protocols=ssl_protocols,
ssl_ciphers=':'.join(ssl_ciphers))
def generate_server_config(config):
"""
Generates server config from the app config
"""
config = config or {}
tuf_server = config.get('TUF_SERVER', None)
tuf_host = config.get('TUF_HOST', None)
signing_enabled = config.get('FEATURE_SIGNING', False)
maximum_layer_size = config.get('MAXIMUM_LAYER_SIZE', '20G')
enable_rate_limits = config.get('FEATURE_RATE_LIMITS', False)
write_config(
os.path.join(QUAYCONF_DIR, 'nginx/server-base.conf'), tuf_server=tuf_server, tuf_host=tuf_host,
signing_enabled=signing_enabled, maximum_layer_size=maximum_layer_size,
enable_rate_limits=enable_rate_limits,
static_dir=STATIC_DIR)
def generate_rate_limiting_config(config):
"""
Generates rate limiting config from the app config
"""
config = config or {}
non_rate_limited_namespaces = config.get('NON_RATE_LIMITED_NAMESPACES') or set()
enable_rate_limits = config.get('FEATURE_RATE_LIMITS', False)
write_config(
os.path.join(QUAYCONF_DIR, 'nginx/rate-limiting.conf'),
non_rate_limited_namespaces=non_rate_limited_namespaces,
enable_rate_limits=enable_rate_limits,
static_dir=STATIC_DIR)
if __name__ == "__main__":
if os.path.exists(os.path.join(QUAYCONF_DIR, 'stack/config.yaml')):
with open(os.path.join(QUAYCONF_DIR, 'stack/config.yaml'), 'r') as f:
config = yaml.load(f)
else:
config = None
generate_rate_limiting_config(config)
generate_server_config(config)
generate_nginx_config(config)

8
conf/init/nginx_conf_create.sh Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
QUAYDIR=${QUAYDIR:-"/"}
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd $QUAYDIR
python $QUAYCONF/init/nginx_conf_create.py

10
conf/init/runmigration.sh Executable file
View file

@ -0,0 +1,10 @@
#!/bin/bash
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
set -e
cd ${QUAYDIR:-"/"}
# Run the database migration
PYTHONPATH=${QUAYPATH:-"."} python $QUAYCONF/init/v3_migration.py > revision_head
PYTHONPATH=${QUAYPATH:-"."} alembic upgrade `cat revision_head`

4
conf/init/service/batch/blobuploadcleanupworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t blobuploadcleanupworker

10
conf/init/service/batch/blobuploadcleanupworker/run Executable file
View file

@ -0,0 +1,10 @@
#! /bin/bash
echo 'Starting Blob upload cleanup worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.blobuploadcleanupworker.blobuploadcleanupworker 2>&1
echo 'Blob upload cleanup exited'

4
conf/init/service/batch/buildlogsarchiver/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t buildlogsarchiver

9
conf/init/service/batch/buildlogsarchiver/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting build logs archiver worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.buildlogsarchiver.buildlogsarchiver 2>&1
echo 'Diffs worker exited'

4
conf/init/service/batch/buildmanager/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t buildmanager

11
conf/init/service/batch/buildmanager/run Executable file
View file

@ -0,0 +1,11 @@
#! /bin/bash
echo 'Starting internal build manager'
# Run the build manager.
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
export PYTHONPATH=$QUAYPATH
exec venv/bin/python -m buildman.builder 2>&1
echo 'Internal build manager exited'

4
conf/init/service/batch/chunkcleanupworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t chunkcleanupworker

9
conf/init/service/batch/chunkcleanupworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting chunk cleanup worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.chunkcleanupworker 2>&1
echo 'Chunk cleanup worker exited'

4
conf/init/service/batch/expiredappspecifictokenworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t expiredappspecifictokenworker

9
conf/init/service/batch/expiredappspecifictokenworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting Expired app specific token GC worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.expiredappspecifictokenworker 2>&1
echo 'Expired app specific token GC exited'

4
conf/init/service/batch/exportactionlogsworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t exportactionlogsworker

9
conf/init/service/batch/exportactionlogsworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting Export Actions Log worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.exportactionlogsworker 2>&1
echo 'Export Actions Log worker exited'

4
conf/init/service/batch/gcworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t gcworker

9
conf/init/service/batch/gcworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting GC worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.gc.gcworker 2>&1
echo 'Repository GC exited'

4
conf/init/service/batch/globalpromstats/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t globalpromstats

9
conf/init/service/batch/globalpromstats/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting global prometheus stats worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.globalpromstats.globalpromstats
echo 'Global prometheus stats exited'

4
conf/init/service/batch/labelbackfillworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t labelbackfillworker

9
conf/init/service/batch/labelbackfillworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting label backfill worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.labelbackfillworker 2>&1
echo 'Repository label backfill exited'

4
conf/init/service/batch/logrotateworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t logrotateworker

9
conf/init/service/batch/logrotateworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting log rotation worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.logrotateworker
echo 'Log rotation worker exited'

4
conf/init/service/batch/namespacegcworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t namespacegcworker

9
conf/init/service/batch/namespacegcworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting Namespace GC worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.namespacegcworker 2>&1
echo 'Namespace GC exited'

4
conf/init/service/batch/notificationworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t notificationworker

10
conf/init/service/batch/notificationworker/run Executable file
View file

@ -0,0 +1,10 @@
#! /bin/bash
echo 'Starting notification worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.notificationworker.notificationworker
echo 'Notification worker exited'

4
conf/init/service/batch/queuecleanupworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t queuecleanupworker

9
conf/init/service/batch/queuecleanupworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting Queue cleanup worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.queuecleanupworker 2>&1
echo 'Repository Queue cleanup exited'

4
conf/init/service/batch/repositoryactioncounter/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t repositoryactioncounter

9
conf/init/service/batch/repositoryactioncounter/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting repository action count worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.repositoryactioncounter 2>&1
echo 'Repository action worker exited'

4
conf/init/service/batch/security_notification_worker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t security_notification_worker

9
conf/init/service/batch/security_notification_worker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting security scanner notification worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.security_notification_worker 2>&1
echo 'Security scanner notification worker exited'

4
conf/init/service/batch/securityworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t securityworker

9
conf/init/service/batch/securityworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting security scanner worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.securityworker.securityworker 2>&1
echo 'Security scanner worker exited'

4
conf/init/service/batch/storagereplication/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t storagereplication

9
conf/init/service/batch/storagereplication/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting storage replication worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.storagereplication 2>&1
echo 'Repository storage replication exited'

4
conf/init/service/batch/tagbackfillworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t tagbackfillworker

9
conf/init/service/batch/tagbackfillworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting tag backfill worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.tagbackfillworker 2>&1
echo 'Repository tag backfill exited'

4
conf/init/service/batch/teamsyncworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t teamsyncworker

9
conf/init/service/batch/teamsyncworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting team synchronization worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.teamsyncworker.teamsyncworker 2>&1
echo 'Team synchronization worker exited'

4
conf/init/service/interactive/dnsmasq/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t dnsmasq

7
conf/init/service/interactive/dnsmasq/run Executable file
View file

@ -0,0 +1,7 @@
#! /bin/bash
echo 'Starting dnsmasq'
/usr/sbin/dnsmasq --no-daemon --user=root --listen-address=127.0.0.1
echo 'dnsmasq'

4
conf/init/service/interactive/gunicorn_registry/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t gunicorn_registry

12
conf/init/service/interactive/gunicorn_registry/run Executable file
View file

@ -0,0 +1,12 @@
#! /bin/bash
echo 'Starting gunicon'
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
DB_CONNECTION_POOLING=${DB_CONNECTION_POOLING:-"true"}
cd ${QUAYDIR:-"/"}
DB_CONNECTION_POOLING=$DB_CONNECTION_POOLING PYTHONPATH=$QUAYPATH nice -n 10 venv/bin/gunicorn -c $QUAYCONF/gunicorn_registry.py registry:application
echo 'Gunicorn exited'

4
conf/init/service/interactive/gunicorn_secscan/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t gunicorn_secscan

11
conf/init/service/interactive/gunicorn_secscan/run Executable file
View file

@ -0,0 +1,11 @@
#! /bin/bash
echo 'Starting gunicon'
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/gunicorn -c $QUAYCONF/gunicorn_secscan.py secscan:application
echo 'Gunicorn exited'

4
conf/init/service/interactive/gunicorn_verbs/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t gunicorn_verbs

11
conf/init/service/interactive/gunicorn_verbs/run Executable file
View file

@ -0,0 +1,11 @@
#! /bin/bash
echo 'Starting gunicon'
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH nice -n 10 venv/bin/gunicorn -c $QUAYCONF/gunicorn_verbs.py verbs:application
echo 'Gunicorn exited'

4
conf/init/service/interactive/gunicorn_web/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t gunicorn_web

11
conf/init/service/interactive/gunicorn_web/run Executable file
View file

@ -0,0 +1,11 @@
#! /bin/bash
echo 'Starting gunicon'
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/gunicorn -c $QUAYCONF/gunicorn_web.py web:application
echo 'Gunicorn exited'

4
conf/init/service/interactive/jwtproxy/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t jwtproxy

16
conf/init/service/interactive/jwtproxy/run Executable file
View file

@ -0,0 +1,16 @@
#! /bin/bash
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
if [ -f $QUAYCONF/jwtproxy_conf.yaml ];
then
echo 'Starting jwtproxy'
/usr/local/bin/jwtproxy --config $QUAYCONF/jwtproxy_conf.yaml
rm /tmp/jwtproxy_secscan.sock
echo 'Jwtproxy exited'
else
sleep 1
fi

7
conf/init/service/interactive/memcached/log/run Executable file
View file

@ -0,0 +1,7 @@
#!/bin/sh
# Ensure dependencies start before the logger
sv check syslog-ng > /dev/null || exit 1
# Start the logger
exec logger -i -t memcached

12
conf/init/service/interactive/memcached/run Executable file
View file

@ -0,0 +1,12 @@
#! /bin/bash
echo 'Starting memcached'
if [ "$DEBUGLOG" == "true" ]
then
memcached -u memcached -m 64 -vv -l 127.0.0.1 -p 18080
else
memcached -u memcached -m 64 -l 127.0.0.1 -p 18080
fi
echo 'memcached exited'

4
conf/init/service/interactive/nginx/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t nginx

12
conf/init/service/interactive/nginx/run Executable file
View file

@ -0,0 +1,12 @@
#! /bin/bash
echo 'Starting nginx'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
/usr/sbin/nginx -c $QUAYCONF/nginx/nginx.conf
echo 'Nginx exited'

4
conf/init/service/interactive/prometheus-aggregator/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t prometheus-aggregator

7
conf/init/service/interactive/prometheus-aggregator/run Executable file
View file

@ -0,0 +1,7 @@
#! /bin/bash
echo 'Starting prometheus aggregator'
/usr/local/bin/prometheus-aggregator
echo 'Prometheus aggregator exited'

4
conf/init/service/interactive/servicekeyworker/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t service_key_worker

9
conf/init/service/interactive/servicekeyworker/run Executable file
View file

@ -0,0 +1,9 @@
#! /bin/bash
echo 'Starting service key worker'
QUAYPATH=${QUAYPATH:-"."}
cd ${QUAYDIR:-"/"}
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.servicekeyworker.servicekeyworker 2>&1
echo 'Service key worker exited'

147
conf/init/supervisord_conf_create.py Normal file
View file

@ -0,0 +1,147 @@
import os
import os.path
import jinja2
QUAYPATH = os.getenv("QUAYPATH", ".")
QUAYDIR = os.getenv("QUAYDIR", "/")
QUAYCONF_DIR = os.getenv("QUAYCONF", os.path.join(QUAYDIR, QUAYPATH, "conf"))
QUAY_SERVICES = os.getenv("QUAY_SERVICES", [])
QUAY_OVERRIDE_SERVICES = os.getenv("QUAY_OVERRIDE_SERVICES", [])
def default_services():
return {
"blobuploadcleanupworker": {
"autostart": "true"
},
"buildlogsarchiver": {
"autostart": "true"
},
"builder": {
"autostart": "true"
},
"chunkcleanupworker": {
"autostart": "true"
},
"expiredappspecifictokenworker": {
"autostart": "true"
},
"exportactionlogsworker": {
"autostart": "true"
},
"gcworker": {
"autostart": "true"
},
"globalpromstats": {
"autostart": "true"
},
"labelbackfillworker": {
"autostart": "true"
},
"logrotateworker": {
"autostart": "true"
},
"namespacegcworker": {
"autostart": "true"
},
"notificationworker": {
"autostart": "true"
},
"queuecleanupworker": {
"autostart": "true"
},
"repositoryactioncounter": {
"autostart": "true"
},
"security_notification_worker": {
"autostart": "true"
},
"securityworker": {
"autostart": "true"
},
"storagereplication": {
"autostart": "true"
},
"tagbackfillworker": {
"autostart": "true"
},
"teamsyncworker": {
"autostart": "true"
},
"dnsmasq": {
"autostart": "true"
},
"gunicorn-registry": {
"autostart": "true"
},
"gunicorn-secscan": {
"autostart": "true"
},
"gunicorn-verbs": {
"autostart": "true"
},
"gunicorn-web": {
"autostart": "true"
},
"ip-resolver-update-worker": {
"autostart": "true"
},
"jwtproxy": {
"autostart": "true"
},
"memcache": {
"autostart": "true"
},
"nginx": {
"autostart": "true"
},
"prometheus-aggregator": {
"autostart": "true"
},
"servicekey": {
"autostart": "true"
},
"repomirrorworker": {
"autostart": "false"
}
}
def generate_supervisord_config(filename, config):
with open(filename + ".jnj") as f:
template = jinja2.Template(f.read())
rendered = template.render(config=config)
with open(filename, 'w') as f:
f.write(rendered)
def limit_services(config, enabled_services):
if enabled_services == []:
return
for service in config.keys():
if service in enabled_services:
config[service]["autostart"] = "true"
else:
config[service]["autostart"] = "false"
def override_services(config, override_services):
if override_services == []:
return
for service in config.keys():
if service + "=true" in override_services:
config[service]["autostart"] = "true"
elif service + "=false" in override_services:
config[service]["autostart"] = "false"
if __name__ == "__main__":
config = default_services()
limit_services(config, QUAY_SERVICES)
override_services(config, QUAY_OVERRIDE_SERVICES)
generate_supervisord_config(os.path.join(QUAYCONF_DIR, 'supervisord.conf'), config)

8
conf/init/supervisord_conf_create.sh Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
QUAYDIR=${QUAYDIR:-"/"}
QUAYPATH=${QUAYPATH:-"."}
QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
cd $QUAYDIR
python $QUAYCONF/init/supervisord_conf_create.py

0
conf/init/test/__init__.py Normal file
View file

778
conf/init/test/test_supervisord_conf_create.py Normal file
View file

@ -0,0 +1,778 @@
import os
import pytest
import json
import yaml
import jinja2
from ..supervisord_conf_create import QUAYCONF_DIR, default_services, limit_services
def render_supervisord_conf(config):
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../supervisord.conf.jnj")) as f:
template = jinja2.Template(f.read())
return template.render(config=config)
def test_supervisord_conf_create_defaults():
config = default_services()
limit_services(config, [])
rendered = render_supervisord_conf(config)
expected = """[supervisord]
nodaemon=true
[unix_http_server]
file=%(ENV_QUAYCONF)s/supervisord.sock
user=root
[supervisorctl]
serverurl=unix:///%(ENV_QUAYCONF)s/supervisord.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[eventlistener:stdout]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command = supervisor_stdout
buffer_size = 1024
events = PROCESS_LOG
result_handler = supervisor_stdout:event_handler
;;; Run batch scripts
[program:blobuploadcleanupworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.blobuploadcleanupworker.blobuploadcleanupworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:buildlogsarchiver]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.buildlogsarchiver.buildlogsarchiver
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:builder]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m buildman.builder
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:chunkcleanupworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.chunkcleanupworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:expiredappspecifictokenworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.expiredappspecifictokenworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:exportactionlogsworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.exportactionlogsworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gcworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.gc.gcworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:globalpromstats]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.globalpromstats.globalpromstats
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:labelbackfillworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.labelbackfillworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:logrotateworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.logrotateworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:namespacegcworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.namespacegcworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:notificationworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.notificationworker.notificationworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:queuecleanupworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.queuecleanupworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:repositoryactioncounter]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.repositoryactioncounter
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:security_notification_worker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.security_notification_worker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:securityworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.securityworker.securityworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:storagereplication]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.storagereplication
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:tagbackfillworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.tagbackfillworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:teamsyncworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.teamsyncworker.teamsyncworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
;;; Run interactive scripts
[program:dnsmasq]
command=/usr/sbin/dnsmasq --no-daemon --user=root --listen-address=127.0.0.1 --port=8053
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-registry]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s,
DB_CONNECTION_POOLING=%(ENV_DB_CONNECTION_POOLING_REGISTRY)s
command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_registry.py registry:application
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-secscan]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=gunicorn -c %(ENV_QUAYCONF)s/gunicorn_secscan.py secscan:application
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-verbs]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_verbs.py verbs:application
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-web]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=gunicorn -c %(ENV_QUAYCONF)s/gunicorn_web.py web:application
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:jwtproxy]
command=/usr/local/bin/jwtproxy --config %(ENV_QUAYCONF)s/jwtproxy_conf.yaml
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:memcache]
command=memcached -u memcached -m 64 -l 127.0.0.1 -p 18080
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:nginx]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=nginx -c %(ENV_QUAYCONF)s/nginx/nginx.conf
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:prometheus-aggregator]
command=/usr/local/bin/prometheus-aggregator
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:servicekey]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.servicekeyworker.servicekeyworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:repomirrorworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.repomirrorworker.repomirrorworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
# EOF NO NEWLINE"""
assert rendered == expected
def test_supervisord_conf_create_all_overrides():
config = default_services()
limit_services(config, "servicekey,prometheus-aggregator")
rendered = render_supervisord_conf(config)
expected = """[supervisord]
nodaemon=true
[unix_http_server]
file=%(ENV_QUAYCONF)s/supervisord.sock
user=root
[supervisorctl]
serverurl=unix:///%(ENV_QUAYCONF)s/supervisord.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[eventlistener:stdout]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command = supervisor_stdout
buffer_size = 1024
events = PROCESS_LOG
result_handler = supervisor_stdout:event_handler
;;; Run batch scripts
[program:blobuploadcleanupworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.blobuploadcleanupworker.blobuploadcleanupworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:buildlogsarchiver]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.buildlogsarchiver.buildlogsarchiver
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:builder]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m buildman.builder
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:chunkcleanupworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.chunkcleanupworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:expiredappspecifictokenworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.expiredappspecifictokenworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:exportactionlogsworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.exportactionlogsworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gcworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.gc.gcworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:globalpromstats]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.globalpromstats.globalpromstats
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:labelbackfillworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.labelbackfillworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:logrotateworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.logrotateworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:namespacegcworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.namespacegcworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:notificationworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.notificationworker.notificationworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:queuecleanupworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.queuecleanupworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:repositoryactioncounter]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.repositoryactioncounter
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:security_notification_worker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.security_notification_worker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:securityworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.securityworker.securityworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:storagereplication]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.storagereplication
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:tagbackfillworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.tagbackfillworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:teamsyncworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.teamsyncworker.teamsyncworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
;;; Run interactive scripts
[program:dnsmasq]
command=/usr/sbin/dnsmasq --no-daemon --user=root --listen-address=127.0.0.1 --port=8053
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-registry]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s,
DB_CONNECTION_POOLING=%(ENV_DB_CONNECTION_POOLING_REGISTRY)s
command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_registry.py registry:application
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-secscan]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=gunicorn -c %(ENV_QUAYCONF)s/gunicorn_secscan.py secscan:application
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-verbs]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_verbs.py verbs:application
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-web]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=gunicorn -c %(ENV_QUAYCONF)s/gunicorn_web.py web:application
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:jwtproxy]
command=/usr/local/bin/jwtproxy --config %(ENV_QUAYCONF)s/jwtproxy_conf.yaml
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:memcache]
command=memcached -u memcached -m 64 -l 127.0.0.1 -p 18080
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:nginx]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=nginx -c %(ENV_QUAYCONF)s/nginx/nginx.conf
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:prometheus-aggregator]
command=/usr/local/bin/prometheus-aggregator
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:servicekey]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.servicekeyworker.servicekeyworker
autostart = true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
[program:repomirrorworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.repomirrorworker.repomirrorworker
autostart = false
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_events_enabled = true
stderr_events_enabled = true
# EOF NO NEWLINE"""
assert rendered == expected

15
conf/init/v3_migration.py Normal file
View file

@ -0,0 +1,15 @@
from app import app
from active_migration import ActiveDataMigration
if not app.config.get('SETUP_COMPLETE', False):
print 'head'
else:
v3_upgrade_mode = app.config.get('V3_UPGRADE_MODE')
if v3_upgrade_mode == 'background':
raise Exception('V3_UPGRADE_MODE must be "complete". This requires a full upgrade to Quay:v3.0. See https://access.qa.redhat.com/documentation/en-us/red_hat_quay/3/html/upgrade_quay/index')
elif v3_upgrade_mode == 'production-transition':
print '481623ba00ba'
elif v3_upgrade_mode == 'post-oci-rollout' or v3_upgrade_mode == 'post-oci-roll-back-compat' or v3_upgrade_mode == 'complete':
print ActiveDataMigration.alembic_migration_revision
else:
raise Exception('Unknown V3_UPGRADE_MODE: %s' % v3_upgrade_mode)

4
conf/init/zz_boot.sh Executable file
View file

@ -0,0 +1,4 @@
#!/bin/bash
cd ${QUAYDIR:-"/"}
python ${QUAYPATH:-"."}/boot.py