initial import for Open Source 🎉
This commit is contained in:
Jimmy Zelinskie
parent
1898c361f3
commit
9c0dd3b722
2048 changed files with 218743 additions and 0 deletions
35
.dockerignore
Normal file
35
.dockerignore
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
./ci/
|
||||||
|
conf/stack
|
||||||
|
conf/stack/**
|
||||||
|
conf/stack/config.yaml
|
||||||
|
config.yaml
|
||||||
|
screenshots
|
||||||
|
tools
|
||||||
|
test/data/registry
|
||||||
|
venv
|
||||||
|
.git
|
||||||
|
!.git/HEAD
|
||||||
|
.gitignore
|
||||||
|
.github
|
||||||
|
Bobfile
|
||||||
|
README.md
|
||||||
|
ROADMAP.md
|
||||||
|
requirements-nover.txt
|
||||||
|
run-local.sh
|
||||||
|
.DS_Store
|
||||||
|
**/*.pyc
|
||||||
|
.tox
|
||||||
|
htmlcov
|
||||||
|
.coverage
|
||||||
|
coverage
|
||||||
|
.cache
|
||||||
|
.npm-debug.log
|
||||||
|
test/__pycache__
|
||||||
|
__pycache__
|
||||||
|
**/__pycache__
|
||||||
|
static/build/**
|
||||||
|
.gitlab-ci/*
|
||||||
|
.gitlab-ci.*
|
||||||
|
docker-compose.yaml
|
||||||
|
test/dockerclients/**
|
||||||
|
node_modules
|
||||||
29
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
29
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
|
||||||
|
### Description of Changes
|
||||||
|
|
||||||
|
* details about the implementation of the changes
|
||||||
|
* motivation for the change (broken code, new feature, etc)
|
||||||
|
* contrast with previous behavior
|
||||||
|
|
||||||
|
|
||||||
|
#### Changes:
|
||||||
|
|
||||||
|
* ..
|
||||||
|
* ..
|
||||||
|
|
||||||
|
#### Issue: <link to story or task>
|
||||||
|
|
||||||
|
|
||||||
|
**TESTING** ->
|
||||||
|
|
||||||
|
**BREAKING CHANGE** ->
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reviewer Checklist
|
||||||
|
|
||||||
|
- [ ] It works!
|
||||||
|
- [ ] Comments provide sufficient explanations for the next contributor
|
||||||
|
- [ ] Tests cover changes and corner cases
|
||||||
|
- [ ] Follows Quay syntax patterns and format
|
||||||
31
.gitignore
vendored
Normal file
31
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
*.pyc
|
||||||
|
venv
|
||||||
|
screenshots/screenshots/
|
||||||
|
conf/stack
|
||||||
|
*/node_modules
|
||||||
|
dist
|
||||||
|
dest
|
||||||
|
node_modules
|
||||||
|
static/ldn
|
||||||
|
static/fonts
|
||||||
|
static/build
|
||||||
|
stack_local
|
||||||
|
test/data/registry/
|
||||||
|
GIT_HEAD
|
||||||
|
.idea
|
||||||
|
.python-version
|
||||||
|
.pylintrc
|
||||||
|
.coverage
|
||||||
|
coverage
|
||||||
|
htmlcov
|
||||||
|
.tox
|
||||||
|
.cache
|
||||||
|
.npm-debug.log
|
||||||
|
Dockerfile-e
|
||||||
|
build/
|
||||||
|
.vscode
|
||||||
|
*.iml
|
||||||
|
.DS_Store
|
||||||
|
.pytest_cache/*
|
||||||
|
test/dockerclients/Vagrantfile
|
||||||
|
test/dockerclients/.*
|
||||||
37
.style.yapf
Normal file
37
.style.yapf
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
[style]
|
||||||
|
based_on_style = chromium
|
||||||
|
COLUMN_LIMIT=99
|
||||||
|
INDENT_WIDTH=2
|
||||||
|
BLANK_LINE_BEFORE_CLASS_DOCSTRING=False
|
||||||
|
#True
|
||||||
|
ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True
|
||||||
|
# False
|
||||||
|
ALLOW_MULTILINE_DICTIONARY_KEYS=True
|
||||||
|
# False
|
||||||
|
BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False
|
||||||
|
# False
|
||||||
|
COALESCE_BRACKETS=True
|
||||||
|
DEDENT_CLOSING_BRACKETS=False
|
||||||
|
CONTINUATION_INDENT_WIDTH=2
|
||||||
|
# False
|
||||||
|
INDENT_DICTIONARY_VALUE=True
|
||||||
|
JOIN_MULTIPLE_LINES=False
|
||||||
|
# True
|
||||||
|
SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=False
|
||||||
|
# True
|
||||||
|
SPLIT_BEFORE_NAMED_ASSIGNS=False
|
||||||
|
SPLIT_PENALTY_AFTER_OPENING_BRACKET=30
|
||||||
|
SPLIT_PENALTY_AFTER_UNARY_OPERATOR=10000
|
||||||
|
SPLIT_PENALTY_BEFORE_IF_EXPR=0
|
||||||
|
SPLIT_PENALTY_BITWISE_OPERATOR=300
|
||||||
|
SPLIT_PENALTY_EXCESS_CHARACTER=10000
|
||||||
|
SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=30
|
||||||
|
SPLIT_PENALTY_IMPORT_NAMES=450
|
||||||
|
SPLIT_PENALTY_LOGICAL_OPERATOR=300
|
||||||
|
USE_TABS=False
|
||||||
|
SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=False
|
||||||
|
# Align closing bracket with visual indentation.
|
||||||
|
align_closing_bracket_with_visual_indent=True
|
||||||
|
|
||||||
|
# Allow lambdas to be formatted on more than one line.
|
||||||
|
allow_multiline_lambdas=True
|
||||||
103
.travis.yml
Normal file
103
.travis.yml
Normal file
|
|
@ -0,0 +1,103 @@
|
||||||
|
---
|
||||||
|
language: python
|
||||||
|
python: 2.7
|
||||||
|
|
||||||
|
sudo: required
|
||||||
|
|
||||||
|
services:
|
||||||
|
- docker
|
||||||
|
|
||||||
|
install: true
|
||||||
|
|
||||||
|
branches:
|
||||||
|
only:
|
||||||
|
- master
|
||||||
|
|
||||||
|
# Stop default database instances here to avoid port conflicts.
|
||||||
|
before_script:
|
||||||
|
- sudo service mysql stop
|
||||||
|
- sudo service postgresql stop
|
||||||
|
|
||||||
|
# Clean the cache if any step fails.
|
||||||
|
before_cache:
|
||||||
|
- scripts/ci fail-clean
|
||||||
|
|
||||||
|
cache:
|
||||||
|
timeout: 1000
|
||||||
|
directories:
|
||||||
|
- $HOME/docker
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- build
|
||||||
|
- test
|
||||||
|
- clean
|
||||||
|
|
||||||
|
# We should label the steps if Travis ever supports it:
|
||||||
|
# https://github.com/travis-ci/travis-ci/issues/5898
|
||||||
|
jobs:
|
||||||
|
include:
|
||||||
|
- stage: build
|
||||||
|
name: Build
|
||||||
|
script: scripts/ci build
|
||||||
|
|
||||||
|
# To further shard, change the script to shard_X_of_XS and add new steps
|
||||||
|
- stage: test
|
||||||
|
name: Unit tests (shard 1)
|
||||||
|
script: scripts/ci unit shard_1_of_2
|
||||||
|
- stage: test
|
||||||
|
name: Unit tests (shard 2)
|
||||||
|
script: scripts/ci unit shard_2_of_2
|
||||||
|
|
||||||
|
- stage: test
|
||||||
|
name: Registry tests (shard 1)
|
||||||
|
script: scripts/ci registry shard_1_of_5
|
||||||
|
- stage: test
|
||||||
|
name: Registry tests (shard 2)
|
||||||
|
script: scripts/ci registry shard_2_of_5
|
||||||
|
- stage: test
|
||||||
|
name: Registry tests (shard 3)
|
||||||
|
script: scripts/ci registry shard_3_of_5
|
||||||
|
- stage: test
|
||||||
|
name: Registry tests (shard 4)
|
||||||
|
script: scripts/ci registry shard_4_of_5
|
||||||
|
- stage: test
|
||||||
|
name: Registry tests (shard 5)
|
||||||
|
script: scripts/ci registry shard_5_of_5
|
||||||
|
|
||||||
|
- stage: test
|
||||||
|
name: Legacy registry tests
|
||||||
|
script: scripts/ci registry_old
|
||||||
|
|
||||||
|
- stage: test
|
||||||
|
name: Custom TLS certs test
|
||||||
|
script: scripts/ci certs_test
|
||||||
|
|
||||||
|
- stage: test
|
||||||
|
name: Gunicorn worker test
|
||||||
|
script: scripts/ci gunicorn_test
|
||||||
|
|
||||||
|
- stage: test
|
||||||
|
name: MySQL unit tests (shard 1)
|
||||||
|
script: scripts/ci mysql shard_1_of_2
|
||||||
|
- stage: test
|
||||||
|
name: MySQL unit tests (shard 2)
|
||||||
|
script: scripts/ci mysql shard_2_of_2
|
||||||
|
|
||||||
|
- stage: test
|
||||||
|
name: Postgres unit tests (shard 1)
|
||||||
|
script: scripts/ci postgres shard_1_of_2
|
||||||
|
- stage: test
|
||||||
|
name: Postgres unit tests (shard 2)
|
||||||
|
script: scripts/ci postgres shard_2_of_2
|
||||||
|
|
||||||
|
- stage: clean
|
||||||
|
name: Cleanup
|
||||||
|
script: scripts/ci clean
|
||||||
|
|
||||||
|
notifications:
|
||||||
|
slack:
|
||||||
|
rooms:
|
||||||
|
- secure: "fBR3YMXaOkoX2Iz7oSJVAw9zrcDoqwadiMEWTWhx7Ic0zoM8IieD2EWIcDHAoGpqf3ixHkc1v/iLBpbWHgvK7TkrSrGEbFyEmu/uomuHU8oGTiazWCbMWg9T2mhWYFyVaKtt8bzMbFo8k72kYK/NWV8bR4W/Qe/opkH2GGzfhZA="
|
||||||
|
on_success: change
|
||||||
|
on_failure: always
|
||||||
|
on_pull_requests: false
|
||||||
879
CHANGELOG.md
Normal file
879
CHANGELOG.md
Normal file
|
|
@ -0,0 +1,879 @@
|
||||||
|
### v3.1.2
|
||||||
|
|
||||||
|
- Fixed: Repository mirroring properly updates status
|
||||||
|
- Fixed: Application repositories in public namespaces shown in UI
|
||||||
|
- Fixed: Description of log operations in UI
|
||||||
|
- Fixed: Quay V3 upgrade fails with "id field missing from v1Compatibility JSON"
|
||||||
|
- Fixed: Security token for storage proxy properly URL encoded
|
||||||
|
|
||||||
|
### v3.1.1
|
||||||
|
|
||||||
|
- Fixed: Quoting of username/password for repository mirror
|
||||||
|
- Fixed: Changing next sync date in repository mirror UI
|
||||||
|
- Fixed: Enable cancel button in repository mirror UI
|
||||||
|
|
||||||
|
### v3.1.0
|
||||||
|
|
||||||
|
- Added: New Repository Mirror functionality to continously synchronize repositories from external source registries into Quay
|
||||||
|
- Added: New Repository Mode setting (Normal, Mirrored, Read-Only) to indicate how a repository is updated
|
||||||
|
- Added: New Quay Setup Operator (Dev Preview) to automate configuring Quay on OpenShift
|
||||||
|
- Added: Support for using Red Hat OpenShift Container Storage 3 as a Quay storage backend
|
||||||
|
- Added: Support for using the Crunchy Data Operator to deploy Postgresql as Quay database
|
||||||
|
- Added: Ability to use build ARGS as first line in Dockerfiles in Quay builds
|
||||||
|
- Added: New Red Hat color scheme in Quay web UI
|
||||||
|
- Fixed: Display of repo_verb logs in logs panel
|
||||||
|
- Fixed: Ensure robot accounts being granted access actually belongs in same namespace
|
||||||
|
- Fixed: Numerous documentation improvements
|
||||||
|
|
||||||
|
### v3.0.5
|
||||||
|
|
||||||
|
- Fixed: LDAP config error when user search results exceeds 1000 objects (#1736)[https://jira.coreos.com/browse/QUAY-1736]
|
||||||
|
- Fixed: Remove obsolete 01_copy_syslog_config.sh (#1768)[https://jira.coreos.com/browse/QUAY-1768)
|
||||||
|
- Fixed: Config tool fails to set up database when password string contains "$" (#1510)[https://jira.coreos.com/browse/QUAY-1510)
|
||||||
|
- Added: Config flag to disable TLSv1.0 support (#1726)[https://jira.coreos.com/browse/QUAY-1726]
|
||||||
|
|
||||||
|
### v3.0.4
|
||||||
|
|
||||||
|
- Fixed: Package vulnerability notifications now shown in UI
|
||||||
|
- Fixed: Error deleting manifest after pushing new tag
|
||||||
|
- Fixed: Manifest now shown in UI for all types
|
||||||
|
- Fixed: CSRF rotation corrected
|
||||||
|
- Fixed: nginx access and error logs now to stdout
|
||||||
|
|
||||||
|
### v3.0.3
|
||||||
|
|
||||||
|
- Fixed: Security scan notifications endpoint not working (part #2) (#3472)
|
||||||
|
- Fixed: Exception raised during parallel pushes of same manifest on Postgres (#3478)
|
||||||
|
- Fixed: Connection pooling was ignoring environment variable (#3480)
|
||||||
|
- Fixed: Exception when in OAuth approval flow (#3491)
|
||||||
|
|
||||||
|
### v3.0.2
|
||||||
|
|
||||||
|
- Fixed: Configuration tool now operates in disconnected environments (#3468)
|
||||||
|
- Fixed: Security scan notifications endpoint not working (#3472)
|
||||||
|
|
||||||
|
### v3.0.1
|
||||||
|
|
||||||
|
- Fixed: Instance health endpoint (`/health/instance`) (#3467)
|
||||||
|
|
||||||
|
### v3.0.0
|
||||||
|
|
||||||
|
**IMPORTANT NOTE:** This release is a **major** release and has special upgrade instructions. Please see the upgrade instructions documentation.
|
||||||
|
|
||||||
|
- Added: Full support for Docker Manifest Version 2, Schema 2, including support for manifest lists and Windows images
|
||||||
|
- Added: New, distinct configuration tool for Quay that can be run outside of Quay itself and perform in-place configuration changes
|
||||||
|
- Added: Disabling of V1 push support by default and support for whitelist-enabling specific namespaces for this legacy protocol (#3398)
|
||||||
|
- Added: Full support for blob mounting via the Docker protocol (#3057)
|
||||||
|
- Added: Have all registry operations be disabled if a namespace is disabled (#3091)
|
||||||
|
- Added: Allow syncing of team members from LDAP/Keystone groups, even if user creation is disabled (#3089)
|
||||||
|
- Added: Add a feature flag to allow username confirmation to be disabled (#3099)
|
||||||
|
- Added: New indexes which should result in significant database performance when accessing lists of tags
|
||||||
|
- Added: Add support for POST on OIDC endpoints, to support those providers that POST back (#3246)
|
||||||
|
- Added: Add support for configuration of the claims required for OIDC authentication (#3246)
|
||||||
|
- Added: Have the instance health check verify the disk space available to ensure it doesn’t run out and cause problems for nginx (#3241)
|
||||||
|
- Added: Support for basic auth on security scanner API endpoints (#3255)
|
||||||
|
- Added: Support for geo-blocking pulls in a namespace from a country (#3300)
|
||||||
|
|
||||||
|
- Fixed: Ensure that starred public repositories appear in the starred repositories list (#3098)
|
||||||
|
- Fixed: Add rate limiting to the catalog endpoint (#3106)
|
||||||
|
- Fixed: Have the catalog endpoint return empty for a namespace if it is disabled (#3106)
|
||||||
|
- Fixed: Have user logs start writing to a new LogEntry3 table, which has a BigInteger ID column, to ensure no overflow
|
||||||
|
- Fixed: Improve loading of action logs to be less jumpy (#3299)
|
||||||
|
- Fixed: Ensure that all upload segments are deleted in Swift storage engine once no longer necessary (#3260)
|
||||||
|
- Fixed: Handling of unicode in manifests (#3325)
|
||||||
|
- Fixed: Unauthorized request handling under podman for public repositories when anonymous access is disabled (#3365)
|
||||||
|
|
||||||
|
### v2.9.2
|
||||||
|
|
||||||
|
**IMPORTANT NOTE:** This release fixes a bug in which the deletion of namespaces did not result in the deletion of robot accounts under that namespace. While this is not a security issue (no permissions or credentials are leaked), it can appear unusual to users, so an upgrade is highly recommended. This change also includes a migration that cleans up the aforementioned robot accounts, so the migration step can take **several minutes**. Please plan accordingly.
|
||||||
|
|
||||||
|
- Added: Support for custom query parameters on OIDC endpoints (#3050)
|
||||||
|
- Added: Configurable options for search page length and maximum number of pages (#3060)
|
||||||
|
- Added: Better messaging for when the maximum search page is reached (#3060)
|
||||||
|
- Added: Support for browser notifications (#3068)
|
||||||
|
|
||||||
|
- Fixed: Robot accounts were not being immediately deleted under namespaces (#3071)
|
||||||
|
- Fixed: Setup under latest versions of Kubernetes (#3051)
|
||||||
|
- Fixed: Viewing of logs in repositories with many, many logs (#3082)
|
||||||
|
- Fixed: Filtering of deleting users and organizations in superuser panel (#3080)
|
||||||
|
- Fixed: Incorrect information displayed for builds triggered by deleted build triggers (#3078)
|
||||||
|
- Fixed: Robots could not be created with empty descriptions (#3073)
|
||||||
|
- Fixed: Inability to find Dockerfile in certain archives (#3072)
|
||||||
|
- Fixed: Display of empty tab in credentials dialog under certain circumstances (#3061)
|
||||||
|
- Fixed: Overflow of robot names when extremely long (#3062)
|
||||||
|
- Fixed: Respect CPU affinity when determining number of workers to run (#3064)
|
||||||
|
- Fixed: Breakage in RECATPCHA support (#3065)
|
||||||
|
|
||||||
|
### v2.9.1
|
||||||
|
|
||||||
|
**IMPORTANT NOTE:** This release fixes the 2.9.0 migration. If you experienced an error during the 2.9.0 migration, manually rollback and then upgrade your quay instance to 2.9.1.
|
||||||
|
|
||||||
|
- Fixed: Specify default server value for new integer fields added (#3052)
|
||||||
|
- Fixed: Overflow of repository grid UI (#3049)
|
||||||
|
|
||||||
|
### v2.9.0
|
||||||
|
|
||||||
|
- Added: Automatic cleanup of expired external application tokens (#3002)
|
||||||
|
- Added: Make deletions of namespaces occur in the background (#3014)
|
||||||
|
- Added: Ability to disable build triggers (#2892)
|
||||||
|
- Added: Have repeatedly failing build triggers be automatically disabled (#2892)
|
||||||
|
- Added: Automatic caching of registry Blob data for faster pull operations (#3022)
|
||||||
|
- Added: Creation date/time, last usage date/time and other metadata for robot accounts (#3024)
|
||||||
|
- Added: Collaborators view under organizations, for viewing non-members (#3025)
|
||||||
|
|
||||||
|
- Fixed: Make superusers APIs for users and organizations visible in the API browser (#3017)
|
||||||
|
- Fixed: Better messaging when attempting to create a team that already exists (#3006)
|
||||||
|
- Fixed: Prevent possible reflected text attacks by limiting API access (#2987)
|
||||||
|
- Fixed: Have checkable menus in UI respect filters (#3013)
|
||||||
|
- Fixed: Users being invited to a new organization must always be invited (#3029)
|
||||||
|
- Fixed: Removed all license requirements in Quay (#3031)
|
||||||
|
- Fixed: Squashed images with hard links pointing to deleted files no longer fail (#3032)
|
||||||
|
- Fixed: 500 error when trying to pull certain images via torrent (#3036)
|
||||||
|
|
||||||
|
### v2.8.0
|
||||||
|
|
||||||
|
- Added: Support for Azure Blob Storage (#2902)
|
||||||
|
- Added: Ability to filter out disabled users in users list API (#2954)
|
||||||
|
- Added: Image ID in expanded tags view (#2965)
|
||||||
|
- Added: Processes auto-scale based on CPU count (#2971, 2978)
|
||||||
|
- Added: Health checks for all workers (#2977)
|
||||||
|
- Added: Health checks and auto-rotation for service keys (#2909)
|
||||||
|
- Added: Ability to back GitHub or Google login with LDAP/Keystone (#2983)
|
||||||
|
- Added: Configurable page size for Docker Registry V2 API pagination (#2993)
|
||||||
|
|
||||||
|
- Fixed: Anonymous calls to API discovery endpoint (#2953)
|
||||||
|
- Fixed: Optimized creation of repositories
|
||||||
|
- Fixed: Optimized manifest pushing
|
||||||
|
- Fixed: LDAP password input is now password field (#2970)
|
||||||
|
- Fixed: 500 raised when sending an invalid release name for app repos (#2979)
|
||||||
|
- Fixed: Deletion of expired external app tokens (#2981)
|
||||||
|
- Fixed: Sizing of OIDC login buttons (#2990)
|
||||||
|
- Fixed: Hide build-related UI when builds are not enabled (#2991)
|
||||||
|
- Fixed: Incorrect caching of external application token expiration (#2996)
|
||||||
|
- Fixed: Warning bar should not be displayed for already expired application tokens (#3003)
|
||||||
|
|
||||||
|
### v2.7.0
|
||||||
|
|
||||||
|
**NOTE:** This release *removes* support for the OIDC token internal authentication mechanism and replaces it with support for a new app-specific token system. All customers using the old OIDC token auth mechanism must change their configuration after updating manually in `config.yaml`.
|
||||||
|
|
||||||
|
- Added: Support for external application tokens to be used on the Docker CLI (#2942)
|
||||||
|
- Added: Explore tab for browsing visible repositories (#2921)
|
||||||
|
- Added: Ability to view and copy full manifest SHAs in tags view (#2898)
|
||||||
|
- Added: Support for robot tokens in App Registry pushes and pulls (#2899)
|
||||||
|
|
||||||
|
- Fixed: Failure when attempting to use Skopeo tool to access the registry (#2950)
|
||||||
|
- Fixed: Ordering of segments in Swift to match spec (#2920)
|
||||||
|
- Fixed: Squashed image downloading when using Postgres DB (#2930)
|
||||||
|
- Fixed: Hide "Start Build" button if the action is not allowed (#2916)
|
||||||
|
- Fixed: Exception when pushing certain labels with JSON-like contents (#2912)
|
||||||
|
- Fixed: Don't add password required notification for non-database auth (#2910)
|
||||||
|
- Fixed: Tags UI spacing on small displays (#2904)
|
||||||
|
- Fixed: Push updated notification now shows correct tags (#2897)
|
||||||
|
- Fixed: "Restart Container" button in superuser config panel (#2928)
|
||||||
|
- Fixed: Various small JavaScript security fixes
|
||||||
|
|
||||||
|
### v2.6.2
|
||||||
|
|
||||||
|
- Fixed: Failure to register uploaded TLS certificates (#2946)
|
||||||
|
|
||||||
|
### v2.6.1
|
||||||
|
|
||||||
|
- Added: Optimized overhead for direct downloads from Swift storage (#2889)
|
||||||
|
- Fixed: Immediately expire image builds that fail to start (#2887)
|
||||||
|
- Fixed: Failure to list all GitHub Enterprise namespaces (#2894)
|
||||||
|
- Fixed: Incorrect links to builds in notifications (#2895)
|
||||||
|
- Fixed: Failure to delete certain app repositories (#2893)
|
||||||
|
- Fixed: Inability to display Tag Signing status (#2890)
|
||||||
|
- Fixed: Broken health check for OIDC authentication (#2888)
|
||||||
|
|
||||||
|
### v2.6.0
|
||||||
|
|
||||||
|
- Added: Ability to use OIDC token for CLI login (#2695)
|
||||||
|
- Added: Documentation for OIDC callback URLs in setup tool
|
||||||
|
- Added: Ability for users to change their family and given name and company info (#2870)
|
||||||
|
- Added: Support for invite-only user sign up (#2867)
|
||||||
|
- Added: Option to disable partial autocompletion of users (#2864)
|
||||||
|
- Added: Georeplication support in Swift storage (#2874)
|
||||||
|
- Fixed: Namespace links ending in slashes (#2871)
|
||||||
|
- Fixed: Contact info setup in setup tool (#2866)
|
||||||
|
- Fixed: Lazy loading of teams and robots (#2883)
|
||||||
|
- Fixed: OIDC auth headers (#2695)
|
||||||
|
|
||||||
|
### v2.5.0
|
||||||
|
|
||||||
|
- Added: Better TLS caching (#2860)
|
||||||
|
- Added: Feature flag to allow read-only users to see build logs (#2850)
|
||||||
|
- Added: Feature flag to enable team sync setup when not a superuser (#2813)
|
||||||
|
- Added: Preferred public organizations list (#2850)
|
||||||
|
- Added: OIDC support for OIDC implementations without user info endpoint (#2817)
|
||||||
|
- Added: Support for tag expiration, in UI and view a special `quay.expires-after` label (#2718)
|
||||||
|
- Added: Health checks report failure reasons (#2636)
|
||||||
|
- Added: Enable database connection pooling (#2834)
|
||||||
|
|
||||||
|
- Fixed: setting of team resync option
|
||||||
|
- Fixed: Purge repository on very large repositories
|
||||||
|
|
||||||
|
### v2.4.0
|
||||||
|
|
||||||
|
- Added: Kubernetes Applications Support
|
||||||
|
- Added: Full-page search UI (#2529)
|
||||||
|
- Added: Always generate V2 manifests for tag operations in UI (#2608)
|
||||||
|
- Added: Option to enable public repositories in v2 catalog API (#2654)
|
||||||
|
- Added: Disable repository notifications after 3 failures (#2652)
|
||||||
|
- Added: Remove requirement for flash for copy button in UI (#2667)
|
||||||
|
|
||||||
|
- Fixed: Upgrade support for Markdown (#2624)
|
||||||
|
- Fixed: Kubernetes secret generation with secrets with CAPITAL names (#2640)
|
||||||
|
- Fixed: Content-Length reporting on HEAD requests (#2616)
|
||||||
|
- Fixed: Use configured email address as the sender in email notifications (#2635)
|
||||||
|
- Fixed: Better peformance on permissions lookup (#2628)
|
||||||
|
- Fixed: Disable federated login for new users if user creation is disabled (#2623)
|
||||||
|
- Fixed: Show build logs timestamps by default (#2647)
|
||||||
|
- Fixed: Custom TLS certificates tooling in superuser panel under Kubernetes (#2646, #2663)
|
||||||
|
- Fixed: Disable debug logs in superuser panel when under multiple instances (#2663)
|
||||||
|
- Fixed: External Notification Modal UI bug (#2650)
|
||||||
|
- Fixed: Security worker thrashing when security scanner not available
|
||||||
|
- Fixed: Torrent validation in superuser config panel (#2694)
|
||||||
|
- Fixed: Expensive database call in build badges (#2688)
|
||||||
|
|
||||||
|
### v2.3.4
|
||||||
|
|
||||||
|
- Added: Always show tag expiration options in superuser panel
|
||||||
|
|
||||||
|
### v2.3.3
|
||||||
|
|
||||||
|
- Added: Prometheus metric for queued builds (#2596)
|
||||||
|
|
||||||
|
- Fixed: Allow selection of Gitlab repository when Gitlab sends no permissions (#2601)
|
||||||
|
- Fixed: Failure when viewing Gitlab repository with unexpected schema (#2599)
|
||||||
|
- Fixed: LDAP stability fixes (#2598, #2584, #2595)
|
||||||
|
- Fixed: Viewing of repositories with trust enabled caused a 500 (#2594, #2593)
|
||||||
|
- Fixed: Failure in setup tool when time machine config is not set (#2589)
|
||||||
|
|
||||||
|
### v2.3.2
|
||||||
|
|
||||||
|
- Added: Configuration of time machine in UI (#2516)
|
||||||
|
|
||||||
|
- Fixed: Auth header in OIDC login UserInfo call (#2585)
|
||||||
|
- Fixed: Flash of red error box on loading (#2562)
|
||||||
|
- Fixed: Search under postgres (#2568)
|
||||||
|
- Fixed: Gitlab namespaces with null avatars (#2570)
|
||||||
|
- Fixed: Build log archiver race condition which results in missing logs (#2575)
|
||||||
|
- Fixed: Team synchronization when encountering a user with a shared email address (#2580)
|
||||||
|
- Fixed: Create New tooltip hiding dropdown menu (#2579)
|
||||||
|
- Fixed: Ensure build logs archive lookup URL checks build permissions (#2578)
|
||||||
|
|
||||||
|
### v2.3.1
|
||||||
|
|
||||||
|
**IMPORTANT NOTE:** This release fixes the 2.3.0 migration. If you experienced an error during the 2.3.0 migration, manually rollback and then upgrade your quay instance to 2.3.1.
|
||||||
|
- Fixed: Specify default server value for new bool field added to the repository table
|
||||||
|
|
||||||
|
### v2.3.0
|
||||||
|
|
||||||
|
- Added: LDAP Team Sync support (#2387, #2527)
|
||||||
|
- Added: Improved search performance through pre-computed scores (#2441, #2531, #2533, #2539)
|
||||||
|
- Added: Ability to allow pulls even if audit logging fails (#2306)
|
||||||
|
- Added: Full error information for build errors in Superuser panel (#2505)
|
||||||
|
- Added: Better error messages passed to the Docker client (#2499)
|
||||||
|
- Added: Custom git triggers can specify separate build context directory (#2517, #2509)
|
||||||
|
- Added: Improved performance on repository list API (#2542, #2544, #2546)
|
||||||
|
|
||||||
|
- Fixed: Handle undefined case in build message (#2501)
|
||||||
|
- Fixed: OIDC configuration in Superuser panel (#2520)
|
||||||
|
- Fixed: Ability to invite team members by email address (#2522)
|
||||||
|
- Fixed: Avatars for non-owner namespaces in GitLab (#2507, #2532)
|
||||||
|
- Fixed: Update dependencies and remove warnings (#2518, #2511, #2535, #2545, #2553)
|
||||||
|
- Fixed: Remove link to blog (#2523)
|
||||||
|
- Fixed: Better handling for unavailable frontend dependencies (#2503)
|
||||||
|
- Fixed: Top level redirect logic for missing repositories (#2540)
|
||||||
|
- Fixed: Remove extra slash from missing base image permissions error in build logs (#2548)
|
||||||
|
- Fixed: Backfill replication script when adjusting replication destinations (#2555)
|
||||||
|
- Fixed: Errors when deleting repositories without security scanning enabled (#2554)
|
||||||
|
|
||||||
|
### v2.2.0
|
||||||
|
|
||||||
|
**IMPORTANT NOTE:** This release contains a migration which adds a new feature to the build system. This requires shutting down the entire cluster _including builders_ and running one instance to migrate the database forward. You _must_ use a v2.2.0 builder with a v2.2.0 Quay cluster.
|
||||||
|
|
||||||
|
- Added: Separate build contexts from Dockerfile locations (#2398, #2410, #2438, #2449, #2480, #2481)
|
||||||
|
- Added: Configuration and enforcement of maximum layer size (#2388)
|
||||||
|
- Added: OIDC configuration in the Super User Panel (#2393)
|
||||||
|
- Added: Batching of Security Scanner notifications (#2397)
|
||||||
|
- Added: Auth Failures now display messages on the docker client (#2428, #2474)
|
||||||
|
- Added: Redesigned Tags page to include Labels, Image ID Type, and more informative Security Scanner information (#2416)
|
||||||
|
|
||||||
|
- Fixed: Parsing new docker client version format (#2378)
|
||||||
|
- Fixed: Improved repository search performance (#2392, #2440)
|
||||||
|
- Fixed: Miscellaneous Build Trigger page issues (#2405, #2406, #2407, #2408, #2409, #2414, #2418, #2445)
|
||||||
|
- Fixed: Remove all actionable CVEs from the docker image (#2422, #2468)
|
||||||
|
- Fixed: Minor bugs in Repository views (#2423, #2430, #2431)
|
||||||
|
- Fixed: Improve performance by deleting keys in redis rather than expiring (#2439)
|
||||||
|
- Fixed: Better error messages when configuring cloud storage (#2444)
|
||||||
|
- Fixed: Validation and installation of custom TLS certificates (#2473)
|
||||||
|
- Fixed: Garbage Collection corner case (#2404)
|
||||||
|
|
||||||
|
### v2.1.0
|
||||||
|
|
||||||
|
**IMPORTANT NOTE FOR POSTGRES USERS:** This release contains a migration which adds full-text searching capabilities to Red Hat Quay. In order to support this feature, the migration will attempt to create the `pg_trgm` extension in the database. This operation requires **superuser access** to run and requires the extension to be installed. See https://coreos.com/quay-enterprise/docs/latest/postgres-additional-modules.html for more information on installing the extension.
|
||||||
|
|
||||||
|
If the user given to Red Hat Quay is not a superuser, please temporarily grant superuser access to the Red Hat Quay user in the database (or change the user in config) **before** upgrading.
|
||||||
|
|
||||||
|
- Added: Full text search support (#2272)
|
||||||
|
- Added: OIDC support (#2300, #2348)
|
||||||
|
- Added: API for lookup of security status of a manifest (#2334)
|
||||||
|
- Added: More descriptive logs (#2358)
|
||||||
|
|
||||||
|
- Fixed: Datetime bug in logs view (#2318)
|
||||||
|
- Fixed: Display bug in logs view (#2345)
|
||||||
|
- Fixed: Display of expiration date for licenses with multiple entries (#2354)
|
||||||
|
- Fixed: V1 search compatibility (#2344)
|
||||||
|
|
||||||
|
### v2.0.5
|
||||||
|
|
||||||
|
- Added: Build logs viewer in superuser panel
|
||||||
|
|
||||||
|
- Fixed: Support for wildcard certs in the superuser config panel
|
||||||
|
|
||||||
|
### v2.0.4
|
||||||
|
|
||||||
|
- Added: Expand allowed length of namespaces to be between 2 and 255 characters (#2291)
|
||||||
|
- Added: Better messaging for namespaces (#2283)
|
||||||
|
- Added: More customization of Message Of The Day (MOTD) (#2282)
|
||||||
|
- Added: Configurable and default timeout for LDAP (#2247)
|
||||||
|
- Added: Custom SSL certificate panel in superuser panel (#2271, #2274)
|
||||||
|
- Added: User and Organization list pagination on superuser panel (#2250)
|
||||||
|
- Added: Performance improvements for georeplication queuing (#2254)
|
||||||
|
- Added: Automatic garbage collection in security scanner (#2257)
|
||||||
|
- Added: RECAPTCHA support during create account flow (#2245)
|
||||||
|
- Added: Always display full git error in build logs (#2277)
|
||||||
|
- Added: Superuser config clarification warnings (#2279)
|
||||||
|
- Added: Performance improvements around queues (#2276, #2286, #2287)
|
||||||
|
- Added: Automatic retry for security scanning (#2242)
|
||||||
|
- Added: Better error messaging on security scanner lookup failure (#2235)
|
||||||
|
- Added: Ensure robot accounts show at top of entity autocomplete (#2243)
|
||||||
|
|
||||||
|
- Fixed: Exception when autocompleting users in teams (#2255)
|
||||||
|
- Fixed: Port mapping in ACI conversion (#2251, #2273)
|
||||||
|
- Fixed: Error messaging for attempting to join a team with invalid email (#2240)
|
||||||
|
- Fixed: Prometheus metrics for scale (#2237)
|
||||||
|
- Fixed: Security scanner notification pagination (#2233, #2249)
|
||||||
|
|
||||||
|
- Regressed: Support for wildcard certs in the superuser config panel
|
||||||
|
|
||||||
|
### v2.0.3
|
||||||
|
|
||||||
|
- Added: Allow extra_ca_certs to be a folder or a file (#2180)
|
||||||
|
|
||||||
|
- Fixed: Cancelling builds (#2203)
|
||||||
|
- Fixed: Allow license to be set in setup tool (#2200)
|
||||||
|
- Fixed: Improve queue performance (#2207, #2211)
|
||||||
|
- Fixed: Improve security scan performance (#2209)
|
||||||
|
- Fixed: Fix user lookup for external auth engines (#2206)
|
||||||
|
|
||||||
|
### v2.0.2
|
||||||
|
|
||||||
|
- Added: Ability to cancel builds that are already building. (#2041, #2127, #2186, #2189, #2190)
|
||||||
|
- Added: Notifications when a build is canceled (#2173, #2184)
|
||||||
|
- Added: Remove deprecated email flag from generated `docker login` commands (#2146)
|
||||||
|
- Added: Upgrade nginx to v1.11.5 (#2140)
|
||||||
|
- Added: Improve performance of robots management UI (#2145)
|
||||||
|
- Added: Add data about specific manifest or tag pulled in audit logs (#2152)
|
||||||
|
- Added: Debug nginx logs from non-proxy protocol connection (#2167)
|
||||||
|
- Added: Accept multiple team invitations simultaneously (#2169)
|
||||||
|
- Added: Password recovery defaults to resetting password (#2170)
|
||||||
|
- Added: Gzip javascript and svg assets (#2171)
|
||||||
|
- Added: Add support for custom ports in RADOS and S3 storage engines (#2185)
|
||||||
|
- Added: Prometheus metric for number of unscanned images (#2183)
|
||||||
|
|
||||||
|
- Fixed: Fix entity search under Postgres (regression in v2.0.0) (#2172)
|
||||||
|
- Fixed: Error displayed for OAuth if an existing token already matches scopes (#2139)
|
||||||
|
- Fixed: Reduce timeouts of the build manager when under heavy load (#2143, #2157)
|
||||||
|
- Fixed: Fix guage metrics on prometheus endpoint (#2153)
|
||||||
|
- Fixed: Disable CoreOS update-engine on ephemeral Kubernetes builders (#2159)
|
||||||
|
- Fixed: Fix notifications generated by the build manager (#2163)
|
||||||
|
- Fixed: JSON encoding for chunk cleanup in Swift storage engine (#2162)
|
||||||
|
- Fixed: Fix configuration validator when setting up storage engine (#2176)
|
||||||
|
- Fixed: Multiline message of the day to not cover the search box (#2181)
|
||||||
|
|
||||||
|
- Regressed: User lookup for external auth engines broken
|
||||||
|
|
||||||
|
### v2.0.1
|
||||||
|
|
||||||
|
- Added: A defined timeout on all HTTP calls in notification methods
|
||||||
|
- Added: Customized Build start timeouts and better debug logs
|
||||||
|
- Added: A warning bar when the license will become invalid in a week
|
||||||
|
- Added: Collection of user metadata: name and company
|
||||||
|
- Added: New Prometheus metrics
|
||||||
|
- Added: Support for temp usernames and an interstitial to confirm username
|
||||||
|
- Added: Missing parameter on RADOS storage
|
||||||
|
- Added: Stagger worker startup
|
||||||
|
- Added: Make email addresses optional in external auth if email feature is turned off
|
||||||
|
- Added: External auth emails to entity search
|
||||||
|
- Added: Banner bar message when license has expired or is invalid
|
||||||
|
|
||||||
|
- Fixed: Make sure to check for user before redirecting in update user
|
||||||
|
- Fixed: 500 on get label endpoint and add a test
|
||||||
|
- Fixed: KeyError in Github trigger setup
|
||||||
|
- Fixed: Change LDAP errors into debug statements to reduce log clutter
|
||||||
|
- Fixed: Bugs due to conflicting operation names in the API
|
||||||
|
- Fixed: Cannot-use-robot for private base image bug in build dialog
|
||||||
|
- Fixed: Swift exception reporting on deletion and add async chunk cleanup
|
||||||
|
- Fixed: Logs view for dates that start in zero
|
||||||
|
- Fixed: Small JS error fixes
|
||||||
|
- Fixed: A bug with accessing the su config panel without a license
|
||||||
|
- Fixed: Buildcomponent: raise heartbeat timeout to 60s
|
||||||
|
- Fixed: KeyError in config when not present in BitBucket trigger
|
||||||
|
- Fixed: Namespace lookup in V1 registry search
|
||||||
|
- Fixed: Build notification ref filtering setup in UI
|
||||||
|
- Fixed: Entity search API to not IndexError
|
||||||
|
- Fixed: Remove setup and superuser routes when SUPER_USERS is not enabled
|
||||||
|
- Fixed: TypeError in Gitlab trigger when user not found
|
||||||
|
|
||||||
|
- Regressed: Superuser config panel cannot save
|
||||||
|
|
||||||
|
### v2.0.0
|
||||||
|
|
||||||
|
This release is a **required release** and must be run before attempting an upgrade to v2.0.0+.
|
||||||
|
|
||||||
|
In order to upgrade to this version, your cluster must contain a valid license, which can be found and downloaded at: [tectonic.com](https://account.tectonic.com)
|
||||||
|
|
||||||
|
- Added: Require valid license to enable registry actions (#2009, #2018)
|
||||||
|
- Added: The ability to delete users and organizations (#1698)
|
||||||
|
- Added: Add option to properly handle TLS terminated outside of the container (#1986)
|
||||||
|
- Added: Updated run trigger/build dialog (#1895)
|
||||||
|
- Added: Update dependencies to latest versions (#2012)
|
||||||
|
- Added: Ability to use dots and dashes in namespaces intended for use with newer Docker clients (#1852)
|
||||||
|
- Added: Changed dead queue item cleanup from 7 days to 1 day (#2019)
|
||||||
|
- Added: Add a default database timeout to prevent failed DB connections from hanging registry and API operations (#1764)
|
||||||
|
|
||||||
|
- Fixed: Fix error if a vulnerability notification doesn't have a level filter (#1995)
|
||||||
|
- Fixed: Registry WWW-Authenticate and Link headers are now Registry API compliant (#2004)
|
||||||
|
- Fixed: Small fixes for Message of the Day feature (#2005, #2006)
|
||||||
|
- Fixed: Disallow underscores at the beginning of namespaces (#1852)
|
||||||
|
- Fixed: Installation tool liveness checks during container restarts (#2023)
|
||||||
|
|
||||||
|
- Regressed: Entity search broken under Postgres
|
||||||
|
|
||||||
|
### v1.18.0
|
||||||
|
|
||||||
|
- Added: Add message of the day (#1953)
|
||||||
|
- Added: Add repository list pagination (#1858)
|
||||||
|
- Added: Better 404 (and 403) pages (#1857)
|
||||||
|
|
||||||
|
- Fixed: Always use absolute URLs in Location headers to fix blob uploads on nonstandard ports (#1957)
|
||||||
|
- Fixed: Improved reliability of several JS functions (#1959) (#1980) (#1981)
|
||||||
|
- Fixed: Handle unicode in entity search (#1939)
|
||||||
|
- Fixed: Fix tags API pagination (#1926)
|
||||||
|
- Fixed: Add configurable timeout and debug flags to Keystone users (#1867)
|
||||||
|
- Fixed: Build notifications were failing to fire (#1859)
|
||||||
|
- Fixed: Add feature flag to turn off requirement for team invitations (#1845)
|
||||||
|
- Fixed: Don't exception log for expected 404s in Swift storage (#1851)
|
||||||
|
|
||||||
|
### v1.17.1
|
||||||
|
|
||||||
|
- Added: Repository admins can now invoke build triggers manually (#1822)
|
||||||
|
- Added: Improved notifications UI and features (#1839)
|
||||||
|
- Added: Improved UX for managing teams (#1509)
|
||||||
|
|
||||||
|
- Fixed: Timeline's delete-then-tag display bug (#1824)
|
||||||
|
- Fixed: Add .well-known endpoint for Quay (#1790)
|
||||||
|
- Fixed: .tar.gz does not work when building from archive via web UI (#1832)
|
||||||
|
- Fixed: Delete empty Swift chunks (#1844)
|
||||||
|
- Fixed: Handling of custom LDAP cert (#1846)
|
||||||
|
|
||||||
|
### v1.17.0
|
||||||
|
|
||||||
|
- Added: Added Labels API (#1631)
|
||||||
|
- Added: Kubernetes namespace existence check (#1771)
|
||||||
|
- Added: New UI and permissions handling for robots and teams (#1754, #1815)
|
||||||
|
- Added: Retry attempts to the S3-like storages (#1748, #1801, #1802)
|
||||||
|
- Added: Improved messaging when changing email addresses (#1735)
|
||||||
|
- Added: Emails now include logos (#1691)
|
||||||
|
- Added: Improved messaging around expired builds (#1681)
|
||||||
|
|
||||||
|
- Fixed: Logs inside the container failing to rotate (#1812)
|
||||||
|
- Fixed: Filtering of repositories only visible to organization admins (#1795)
|
||||||
|
- Fixed: Invalid HTTP response when creating a duplicate tag (#1780)
|
||||||
|
- Fixed: Asynchronous Worker robustness (#1778, #1781)
|
||||||
|
- Fixed: Manual build failure when using Bitbucket triggers (#1767)
|
||||||
|
- Fixed: Missing "Sign Out" link on mobile UI (#1765)
|
||||||
|
- Fixed: Miscellaneous changes to title usage (#1763)
|
||||||
|
- Fixed: Repository star appearing when not logged in (#1758)
|
||||||
|
- Fixed: Invalid AppC manifests generated when missing an ENV (#1753)
|
||||||
|
- Fixed: Timezones now incorporated into audit logs (#1747)
|
||||||
|
- Fixed: Fixed redirection to specific tags using short URLs (#1743)
|
||||||
|
- Fixed: Broken pagination over only public repositories (#1724, #1726, #1730)
|
||||||
|
- Fixed: Invisible glyph icons on date selectors (#1717)
|
||||||
|
- Fixed: Possibility storage of duplicate images (#1706)
|
||||||
|
- Fixed: Broken "Your Account" links in emails (#1694)
|
||||||
|
- Fixed: Non-admin users no longer default to organization-wide read (#1685)
|
||||||
|
- Fixed: Database performance (#1680, #1688, #1690, #1722, #1744, #1772)
|
||||||
|
|
||||||
|
### v1.16.6
|
||||||
|
|
||||||
|
- Added: Ability to override secure cookie setting when using HTTPS protocol (#1712)
|
||||||
|
|
||||||
|
### v1.16.5
|
||||||
|
|
||||||
|
- Added: Better logging for delete issues in Swift (#1676)
|
||||||
|
- Added: Storage validation on /status endpoint (#1660)
|
||||||
|
- Added: Better logging for upload issues (#1639, #1670)
|
||||||
|
- Added: Support for Swift retries (#1638)
|
||||||
|
- Added: Support for Swift timeouts (#1634)
|
||||||
|
- Fixed: Pagination off-by-one issue in repository tags API (#1672)
|
||||||
|
- Fixed: Missing requires_cors on archived build logs URL (#1673)
|
||||||
|
- Fixed: Tutorial disconnect UI (#1657)
|
||||||
|
- Fixed: Enter key in password dialogs in Firefox (#1655)
|
||||||
|
- Fixed: Custom trigger links in UI (#1652)
|
||||||
|
- Fixed: GC database query optimizations (#1645, 1662)
|
||||||
|
- Fixed: Multipart refs on builds (#1651)
|
||||||
|
- Fixed: Invalid tags on builds (#1648)
|
||||||
|
- Fixed: Fresh login check failure (#1646)
|
||||||
|
- Fixed: Support for empty RDN in LDAP configuration (#1644)
|
||||||
|
- Fixed: Error raised on duplicate placements when replicating (#1633)
|
||||||
|
|
||||||
|
### v1.16.4
|
||||||
|
|
||||||
|
- Added: Configuration of multiple RDNs for LDAP login (#1601)
|
||||||
|
- Added: Key Server health check (#1598)
|
||||||
|
- Added: Promtheus endpoint (#1596)
|
||||||
|
- Added: Upgrade to latest upstream PyGitHub (#1592)
|
||||||
|
- Fixed: Race condition around starting builds (#1621)
|
||||||
|
- Fixed: Geo-replication for CAS objects (#1608)
|
||||||
|
- Fixed: Popularity metrics on list repositories API endpoint (#1599)
|
||||||
|
- Fixed: Removed redundant namespaces from repository listings (#1595)
|
||||||
|
- Fixed: Internal error when paginating a PostgreSQL-backed Quay (#1593, #1622)
|
||||||
|
- Fixed: GitHub API URLs are properly stripped of trailing slashes (#1590)
|
||||||
|
- Fixed: Tutorial fails gracefully without Redis (#1587)
|
||||||
|
|
||||||
|
### v1.16.3
|
||||||
|
|
||||||
|
- Added: Repository Activity Heatmap (#1569, #1571)
|
||||||
|
- Added: Restyled Robots View (#1568)
|
||||||
|
- Added: LDAP certificates specified by name (#1549)
|
||||||
|
- Added: Multiselect toggles for permissions (#1562)
|
||||||
|
- Added: Dynamically generated sitemap.txt (#1552)
|
||||||
|
- Fixed: Fixed URLs missing ports in setup process (#1583)
|
||||||
|
- Fixed: OAuth key not found error when setting up Dex (#1583)
|
||||||
|
- Fixed: Timestamps in syslog now display the proper time (#1579)
|
||||||
|
- Fixed: Added offset for clock skew in JWT expiration (#1578)
|
||||||
|
- Fixed: Replacement of illegal characters in usernames (#1565)
|
||||||
|
- Fixed: Differentiate between different tags on generated ACIs (#1523)
|
||||||
|
- Fixed: Decreased lifetime of various redis keys (#1561)
|
||||||
|
- Fixed: Build pages now robust to redis outage (#1560)
|
||||||
|
- Fixed: Validation of build arguments before contacting a build worker (#1557)
|
||||||
|
- Fixed: Removed hosted Quay.io status from Enterprise 500 page (#1548)
|
||||||
|
- Fixed: Performance of database queries (#1512)
|
||||||
|
|
||||||
|
### v1.16.2
|
||||||
|
|
||||||
|
- Added: Ability for admins to "Take Ownership" of a namespace (#1526)
|
||||||
|
- Fixed: Encrypted Password Dialog can use External Auth Usernames (#1541)
|
||||||
|
- Fixed: Logging race condition in container startup (#1537)
|
||||||
|
- Fixed: Improved database performance on various pages (#1511, #1514)
|
||||||
|
- Fixed: The 'Return' key now works in password dialogs (#1533)
|
||||||
|
- Fixed: Repository descriptions breaking log page styles (#1532)
|
||||||
|
- Fixed: Styles on Privacy and Terms of Service pages (#1531)
|
||||||
|
|
||||||
|
### v1.16.1
|
||||||
|
|
||||||
|
- Added: Registry JWT now uses Quay's Service Keys (#1498, #1527)
|
||||||
|
- Added: Upgrade to Ubuntu 16.04 LTS base image (#1496)
|
||||||
|
- Added: Storage Replication for Registry v2 images (#1502)
|
||||||
|
- Added: Better error messaging for build logs (#1500)
|
||||||
|
- Added: Granting of OAuth tokens for users via xAuth (#1457)
|
||||||
|
- Added: Random generation of key configuration values (#1485)
|
||||||
|
- Added: Upgrade to AngularJS v1.5 (#1473)
|
||||||
|
- Added: Swift API v3 storage support (#1472)
|
||||||
|
- Added: Clarification on various tool tip dialogs (#1468)
|
||||||
|
- Added: Various backend performance increases (#1459, #1493, #1510, #950)
|
||||||
|
- Added: New Credentials, Team, Robot Dialogs (#1421, #1455)
|
||||||
|
- Fixed: Pagination keys must be url-safe base64 encoded (#1485)
|
||||||
|
- Fixed: Sign In to work with more password managers (#1508)
|
||||||
|
- Fixed: Role deletion UI (#1491)
|
||||||
|
- Fixed: UI expansion when large HTML "pre" tags are used in markdown (#1489)
|
||||||
|
- Fixed: Usernames not properly linking with external auth providers (#1483)
|
||||||
|
- Fixed: Display of dates in action logs UI (#1486)
|
||||||
|
- Fixed: Selection bug with checkboxes in the setup process (#1458)
|
||||||
|
- Fixed: Display error with Sign In (#1466)
|
||||||
|
- Fixed: Race condition in ACI generation (#1463, #1490)
|
||||||
|
- Fixed: Incorrect calculation of the actions log archiver
|
||||||
|
- Fixed: Displaying many image tracks on the Repository tags page (#1451)
|
||||||
|
- Fixed: Handling of admin OAuth Scope (#1447)
|
||||||
|
|
||||||
|
### v1.16.0
|
||||||
|
|
||||||
|
- Added: Unified dashboard for viewing vulnerabilities and packages (#268)
|
||||||
|
- Added: Expose createOrganization API endpoint (#1246)
|
||||||
|
- Added: ACI key setup to the setup tool (#1211)
|
||||||
|
- Added: JWT Key Server (#1332)
|
||||||
|
- Added: New Login Screen UI (#1346)
|
||||||
|
- Added: API errors return application/problem+json format (#1361)
|
||||||
|
- Added: JWT Proxy for authenticating services (#1380)
|
||||||
|
- Added: New design for user and org settings (#1409)
|
||||||
|
- Added: Sescan configuration to setup tool (#1428)
|
||||||
|
- Added: New credentials dialog (#1421)
|
||||||
|
- Fixed: Remove uses of target="_blank" anchors (#1411)
|
||||||
|
- Fixed: Bulk operations don't allow "shift selection" (#1389)
|
||||||
|
- Fixed: Add tag pushed to usage log (#798)
|
||||||
|
- Fixed: Increase timeout on V2 (#1377)
|
||||||
|
- Fixed: Save rotated logs to storage via userfiles (#1356)
|
||||||
|
- Fixed: Include all possible response codes in Swagger document (#1018)
|
||||||
|
- Fixed: Improve notification lookup performance (#1329)
|
||||||
|
- Fixed: Future-proof uncompressed size calculation for blob store (#1325)
|
||||||
|
- Fixed: Client side chunk paths (#1306)
|
||||||
|
- Fixed: ACI Volume Names (#1308)
|
||||||
|
- Fixed: Issue when linking to a parent with a different blob (#1291)
|
||||||
|
- Fixed: Not all 401s set www-authenticate header (#1254)
|
||||||
|
- Fixed: Key error when updating V1 Ids (#1240)
|
||||||
|
- Fixed: Unicode error when calculating new V1 IDs (#1239)
|
||||||
|
- Fixed: Error when turning on receipt emails (#1209)
|
||||||
|
|
||||||
|
### v1.15.5
|
||||||
|
|
||||||
|
- Fixed: Docker pushes with v2 sha mismatch was breaking v2 functionality (#1236)
|
||||||
|
|
||||||
|
### v1.15.4 (Broken)
|
||||||
|
|
||||||
|
- Added: Check that will fail if Quay tries to mislink V1 layers with Docker 1.10 (#1228)
|
||||||
|
- Fixed: Backfill of V2 checksums (#1229)
|
||||||
|
- Fixed: 'BlobUpload' Migration (2015-12-14) for MySQL 5.5 (#1227)
|
||||||
|
- Fixed: Minor UI error in tag specific image view (#1222)
|
||||||
|
- Fixed: Notification logo (#1223)
|
||||||
|
|
||||||
|
### v1.15.3
|
||||||
|
|
||||||
|
- Added: 502 page (#1198)
|
||||||
|
- Added: Token based pagination (#1196, #1095)
|
||||||
|
- Fixed: Trust upstream QE proxies to specify https scheme in X-Forwarded-Proto (#1201)
|
||||||
|
- Fixed: Refreshed dependencies to address security issues (#1195, #1192, #1186, #1182)
|
||||||
|
- Fixed: Tests (#1190, #1184)
|
||||||
|
- Fixed: Setup tool storage engine validation (#1194)
|
||||||
|
|
||||||
|
### v1.15.2
|
||||||
|
|
||||||
|
- Fixed Content-Type of V2 manifests to match updated Docker V2 spec (#1169)
|
||||||
|
- Fixed scope handling for Docker 1.8.3 (#1162)
|
||||||
|
- Fixed typos in docs (#1163, #1164)
|
||||||
|
- Added formal support for library repositories (#1160)
|
||||||
|
|
||||||
|
### v1.15.1
|
||||||
|
|
||||||
|
- Fixed swift path computations
|
||||||
|
|
||||||
|
### v1.15.0
|
||||||
|
|
||||||
|
- Added migration to backfill V2 checksums and torrent SHAs (#1129)
|
||||||
|
- Fixed migration query (#1140)
|
||||||
|
|
||||||
|
### v1.15.0pre
|
||||||
|
|
||||||
|
- Fixed UI toggle bug (#1133)
|
||||||
|
- Fixed bug that displayed billing info in QE (#1124)
|
||||||
|
- Added support for torrent pulls (#1119, #1126, #1111, #1133, #1134, #1136, #1138)
|
||||||
|
|
||||||
|
### v1.14.1
|
||||||
|
|
||||||
|
- Fixed migration of V1 metadata (#1120)
|
||||||
|
- Added list view of repositories in all displays (#1109)
|
||||||
|
- Removed image diff feature (#1102, #1116)
|
||||||
|
- Fixed log bug around month handling (#1114)
|
||||||
|
- Added better recovery of organizations (#1108)
|
||||||
|
- Fixed Content-Type on errors with JSON bodies (#1107)
|
||||||
|
- Added QE version in footer
|
||||||
|
- Fixed unhandled exceptions in Queue
|
||||||
|
- Improved database query performance (#1068, #1097)
|
||||||
|
- Fixed UI for dismissing notifications (#1094)
|
||||||
|
- Added namespaces in `docker search` results (#1086)
|
||||||
|
|
||||||
|
### v1.14.0
|
||||||
|
|
||||||
|
- Added Docker Registry v2 support (#885)
|
||||||
|
- Added the ability to blacklist v2 for specific versions (#1065)
|
||||||
|
- Added HTTP2 support (#1031)
|
||||||
|
- Added automatic action logs rotation (#618)
|
||||||
|
- Made garbage collection frequency configurable (#1074)
|
||||||
|
- Fixed user, repositories and images under MySQL (#830, #843, #1075)
|
||||||
|
- Added storage preferences configuration (#725, #807)
|
||||||
|
- Fixed ACI volumes (#1007)
|
||||||
|
- Fixed date display in Firefox (#937)
|
||||||
|
- Fixed page titles (#952)
|
||||||
|
- Added Gitlab, Bitbucket and Github schema support to custom triggers (#525)
|
||||||
|
- Fixed numerous builder failures
|
||||||
|
|
||||||
|
### v1.13.5
|
||||||
|
|
||||||
|
- Fixed 404 page advertising registry v2 protocol support (#790)
|
||||||
|
|
||||||
|
### v1.13.4
|
||||||
|
|
||||||
|
- Fixed incompatibility with Kubernetes 1.1 (#879)
|
||||||
|
|
||||||
|
### v1.13.3
|
||||||
|
|
||||||
|
- Fixed backfill for migration (#846)
|
||||||
|
|
||||||
|
### v1.13.2
|
||||||
|
|
||||||
|
- Fixed 404 API calls redirecting to 404 page (#762)
|
||||||
|
|
||||||
|
### v1.13.1
|
||||||
|
|
||||||
|
- Fixed broken database migration (#759)
|
||||||
|
- Added OpenGraph preview image (#750, #758)
|
||||||
|
|
||||||
|
### v1.13.0
|
||||||
|
|
||||||
|
- Added new Red Hat Quay rebranding (#723, #738, #735, #745, #746, #748, #747, #751)
|
||||||
|
- Added a styled 404 page (#683)
|
||||||
|
- Hid the run button from users that haven't created a trigger (#727)
|
||||||
|
- Added timeouts to calls to GitLab, Bitbucket, GitHub APIs (#636, #633, #631, #722)
|
||||||
|
- Added more fields to responses from user API (#681)
|
||||||
|
- Fixed bug where every repository appeared private in repository listings (#680)
|
||||||
|
- Added an error when geo-replication is enabled with local storage (#667)
|
||||||
|
- Enabled asynchronous garbage collection for all repositories (#665)
|
||||||
|
- Improved UX uploading Dockerfiles (#656)
|
||||||
|
- Improved registry resiliancy to missing image sizes (#643)
|
||||||
|
- Improved Teams UI (#647)
|
||||||
|
- Added a limit to logs pagination API (#603)
|
||||||
|
- Upgrade docker search to use the new search system (#595)
|
||||||
|
- Fixed database hostname validation to include "." and "\" (#579)
|
||||||
|
- Improved build system's resiliancy if operating without redis (#571)
|
||||||
|
- Updated repository name and namespace validation to match new docker behavior (#535, #644)
|
||||||
|
- Refactored and improved Build Trigger validation (#478, #523, #524, #527, #544, #561, #657, #686, #693, #734)
|
||||||
|
- Optimized moving tags (#520)
|
||||||
|
- Optimized database usage (#517, #518, #519, #598, #601, #605, #615, #641, #675)
|
||||||
|
- Migrated all GitHub triggers to use deploy keys (#503)
|
||||||
|
- Added ability to 'RUN cat .git/HEAD' to get git SHAs in builds (#504)
|
||||||
|
- Improved repository count limitations UI (#492, #529)
|
||||||
|
- Added a releases table to database (#495)
|
||||||
|
- Made repository deletion more robust (#497)
|
||||||
|
- Optimized Swift storage to support direct downloads (#484)
|
||||||
|
- Improved build logs UX (#482, #507)
|
||||||
|
- Add basic Kubernetes secret-store support (#272)
|
||||||
|
- Improved internal test suite (#470, #511, #526, #514, #545, #570, #572, #573, #583, #711, #728, #730)
|
||||||
|
- Improved background worker stability (#471)
|
||||||
|
|
||||||
|
### v1.12.0
|
||||||
|
|
||||||
|
- Added experimental Dex login support (#447, #468)
|
||||||
|
- Fixed tag pagination in API (#463)
|
||||||
|
- Improved performance for archiving build logs (#462, #466)
|
||||||
|
- Optimized cloud storage copying (#460)
|
||||||
|
- Fixed bug where LDN directory was given a relative domain not absolute (#458)
|
||||||
|
- Allow robot account names to have underscores (#453)
|
||||||
|
- Added missing SuperUser aggregate logs endpoint (#449)
|
||||||
|
- Made JWT validation more strict (#446, #448)
|
||||||
|
- Added dialog around restarting the container after setup (#441)
|
||||||
|
- Added selection of Swift API version (#444)
|
||||||
|
- Improved UX around organization name validation (#437)
|
||||||
|
- Stopped relying on undocumented behavior for OAuth redirects (#432)
|
||||||
|
- Hardened against S3 upload failures (#434)
|
||||||
|
- Added experimental automatic storage replication (#191)
|
||||||
|
- Deduplicated logging to syslog (#431, #440)
|
||||||
|
- Added list org member permissions back to API (#429)
|
||||||
|
- Fixed bug in parsing unicode Dockerfiles (#426)
|
||||||
|
- Added CloudWatch metrics for multipart uploads (#419)
|
||||||
|
- Updated CloudWatch metrics to send the max metrics per API call (#412)
|
||||||
|
- Limited the items auto-loaded from GitHub in trigger setup to 30 (#382)
|
||||||
|
- Tweaked build UX (#381, #386, #384, #410, #420, #422)
|
||||||
|
- Changed webhook notifications to also send client SSL certs (#374)
|
||||||
|
- Improved internal test suite (#381, #374, #388, #455, #457)
|
||||||
|
|
||||||
|
### v1.11.2
|
||||||
|
|
||||||
|
- Fixed security bug with LDAP login (#376)
|
||||||
|
|
||||||
|
### 1.11.1
|
||||||
|
|
||||||
|
- Loosened the check for mounted volumes bug (#353)
|
||||||
|
- Strengthened HTTPS configuration (#329)
|
||||||
|
- Disabled password change for non-DB auth (#347)
|
||||||
|
- Added support for custom favicon (#343)
|
||||||
|
- Fixed tarfile support for non-unicode pax fields (#328)
|
||||||
|
- Fixed permissions on tag history API requiring READ instead of WRITE tokens (#316)
|
||||||
|
- Added public access to time machine (#334)
|
||||||
|
- Added missing JSON schema for 'refs' and 'branch_name' (#330)
|
||||||
|
- Always create a new connection to Swift (#336)
|
||||||
|
- Minor UI Fixes (#356, #341, #338, #337)
|
||||||
|
- Minor trigger fixes (#357, #349)
|
||||||
|
- Refactored and fixed internal code (#331)
|
||||||
|
|
||||||
|
### 1.11.0
|
||||||
|
|
||||||
|
- Changed user pages to display public repositories (#321)
|
||||||
|
- Changed docs to load via HTTPS instead of HTTP (#314)
|
||||||
|
- Corrected the defaulting of non-existant app configs to the value False (#312)
|
||||||
|
- Fixed a visual bug in repositories for Chrome Canary users (#307)
|
||||||
|
- Fixed Swagger v2 support to be 100% spec compliant (#289)
|
||||||
|
- Added documentation to search (#303)
|
||||||
|
- Improved internal development experience (#297, #299, #301, #302, #311)
|
||||||
|
- Improved UI performance for large repositories and their logs (#296, #294, #318, #319)
|
||||||
|
- Optimized GC and added experimental async GC (#155)
|
||||||
|
- Updated ACI support to ACI 0.6.1 (#280, #284)
|
||||||
|
- Fixed HTTP 500 on logout (#282)
|
||||||
|
- Prevented storage on a non-mounted container volume (#275)
|
||||||
|
- Fixed fetching repositories from GitHub Enterprise (#277)
|
||||||
|
- Increased the size of Quay.io hosted build nodes (#234)
|
||||||
|
- Refactored and fixed internal code (#270, #285, #290, #295, #300, #283, #317)
|
||||||
|
- Migrated triggers to use Bitbucket's new API (#255, #256)
|
||||||
|
- Added a throbber for deleting a repository (#269)
|
||||||
|
- Tweaked numerous UI elements on Repository listing (#268)
|
||||||
|
- Increased SQL query performance for numerous interactions (#264, #281, #308, #309)
|
||||||
|
|
||||||
|
### 1.10.0
|
||||||
|
|
||||||
|
- Fixed GitHub API usage to prevent over-listing users' repos (#260)
|
||||||
|
- Deleted old landing page (#259)
|
||||||
|
- Corrected mistakes in internal logic (#247, #254, #257)
|
||||||
|
- Tweaked UI for List View of Repositories Page (#253, #261)
|
||||||
|
- Added ability to log in with a team invite code (#250)
|
||||||
|
- Optimized various SQL queries (#249, #252, #258)
|
||||||
|
- Refactored internal libraries (#245, #246)
|
||||||
|
- Fixed missing db cert preventing saving configs in super user panel (#244)
|
||||||
|
- Fixed database status in status endpoint (#242)
|
||||||
|
- Added a flash message for various interactions (#226)
|
||||||
|
- Added Keystone (OpenStack auth) support (#197)
|
||||||
|
- Fixed Logs View in SuperUser panel (#136)
|
||||||
|
|
||||||
|
### 1.9.8
|
||||||
|
|
||||||
|
- Implemented file streams for Swift storage engine (#214)
|
||||||
|
- Made script that sets connection limits optional (#208)
|
||||||
|
- Added warning to tag fetching dialog to use robots with permission (#207)
|
||||||
|
- Fixed error when deleting of robot accounts used in builds (#205)
|
||||||
|
- Added encrypted password output in the Superuser API (#203)
|
||||||
|
- Removed HEAD section from Changelog (#202)
|
||||||
|
- Improved error messages on pull failure (#201)
|
||||||
|
- Added pagination support to tag history API (#200)
|
||||||
|
- Deleted all vendored art files (#199)
|
||||||
|
- Deleted all code related to the old UI (#195)
|
||||||
|
- Added ability to configure database SSL cert (#192)
|
||||||
|
- Fixed JWT to use UTC timestamps (#190)
|
||||||
|
- Added delegated Superuser API access (#189)
|
||||||
|
- Fixed JavaScript null pointers & UI tweaks (#188, #224, #217, #224, #233)
|
||||||
|
- Added messaging when archived build logs fail to load (#187)
|
||||||
|
- Replaced Container Usage tab in the Superuser Panel with this Changelog (#186)
|
||||||
|
- Truncated long commit messages in the UI (#185)
|
||||||
|
|
||||||
|
### 1.9.7
|
||||||
|
|
||||||
|
- Changed etcd timeouts in the ephemeral build manager to be 30s (#183)
|
||||||
|
|
||||||
|
### 1.9.6
|
||||||
|
|
||||||
|
- Added fix for etcd-related issues with the ephemeral build manager (#181)
|
||||||
|
|
||||||
|
### 1.9.5
|
||||||
|
|
||||||
|
- Added changelog (#178)
|
||||||
|
- Updated dependencies (#171, #172)
|
||||||
|
- Speed up some queries by using UNION instead of JOIN (#170)
|
||||||
|
- Improved etcd watch logic for ephemeral build system (#168)
|
||||||
|
- Fixed CSS inconsistencies (#167, #160)
|
||||||
|
- Removed dependency on user existance checks for auth implementations (#166)
|
||||||
|
- Fixed issue where noisy build logs caused builds to timeout (#165)
|
||||||
|
- Added scope descriptions to generate token page (#163)
|
||||||
|
- Expose robots API via Swagger (#162)
|
||||||
|
- Improved loading permissions by adding a short circuit (#154)
|
||||||
|
- Improved coverage of handling builds with revoked OAuth credentials (#153)
|
||||||
|
- Added ability to do manual builds of tags (#152)
|
||||||
128
Dockerfile
Normal file
128
Dockerfile
Normal file
|
|
@ -0,0 +1,128 @@
|
||||||
|
FROM centos:7
|
||||||
|
LABEL maintainer "thomasmckay@redhat.com"
|
||||||
|
|
||||||
|
ENV PYTHON_VERSION=2.7 \
|
||||||
|
PATH=$HOME/.local/bin/:$PATH \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONIOENCODING=UTF-8 \
|
||||||
|
LC_ALL=en_US.UTF-8 \
|
||||||
|
LANG=en_US.UTF-8 \
|
||||||
|
PIP_NO_CACHE_DIR=off
|
||||||
|
|
||||||
|
ENV QUAYDIR /quay-registry
|
||||||
|
ENV QUAYCONF /quay-registry/conf
|
||||||
|
ENV QUAYPATH "."
|
||||||
|
|
||||||
|
RUN mkdir $QUAYDIR
|
||||||
|
WORKDIR $QUAYDIR
|
||||||
|
|
||||||
|
RUN INSTALL_PKGS="\
|
||||||
|
python27 \
|
||||||
|
python27-python-pip \
|
||||||
|
rh-nginx112 rh-nginx112-nginx \
|
||||||
|
openldap \
|
||||||
|
scl-utils \
|
||||||
|
gcc-c++ git \
|
||||||
|
openldap-devel \
|
||||||
|
gpgme-devel \
|
||||||
|
dnsmasq \
|
||||||
|
memcached \
|
||||||
|
openssl \
|
||||||
|
skopeo \
|
||||||
|
" && \
|
||||||
|
yum install -y yum-utils && \
|
||||||
|
yum install -y epel-release centos-release-scl && \
|
||||||
|
yum -y --setopt=tsflags=nodocs --setopt=skip_missing_names_on_install=False install $INSTALL_PKGS && \
|
||||||
|
yum -y update && \
|
||||||
|
yum -y clean all
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN scl enable python27 "\
|
||||||
|
pip install --upgrade setuptools pip && \
|
||||||
|
pip install -r requirements.txt --no-cache && \
|
||||||
|
pip install -r requirements-tests.txt --no-cache && \
|
||||||
|
pip freeze && \
|
||||||
|
mkdir -p $QUAYDIR/static/webfonts && \
|
||||||
|
mkdir -p $QUAYDIR/static/fonts && \
|
||||||
|
mkdir -p $QUAYDIR/static/ldn && \
|
||||||
|
PYTHONPATH=$QUAYPATH python -m external_libraries \
|
||||||
|
"
|
||||||
|
|
||||||
|
RUN cp -r $QUAYDIR/static/ldn $QUAYDIR/config_app/static/ldn && \
|
||||||
|
cp -r $QUAYDIR/static/fonts $QUAYDIR/config_app/static/fonts && \
|
||||||
|
cp -r $QUAYDIR/static/webfonts $QUAYDIR/config_app/static/webfonts
|
||||||
|
|
||||||
|
# Check python dependencies for GPL
|
||||||
|
# Due to the following bug, pip results must be piped to a file before grepping:
|
||||||
|
# https://github.com/pypa/pip/pull/3304
|
||||||
|
# 'docutils' is a setup dependency of botocore required by s3transfer. It's under
|
||||||
|
# GPLv3, and so is manually removed.
|
||||||
|
RUN rm -Rf /opt/rh/python27/root/usr/lib/python2.7/site-packages/docutils && \
|
||||||
|
scl enable python27 "pip freeze" | grep -v '^-e' | awk -F == '{print $1}' | grep -v docutils > piplist.txt && \
|
||||||
|
scl enable python27 "xargs -a piplist.txt pip --disable-pip-version-check show" > pipinfo.txt && \
|
||||||
|
test -z "$(cat pipinfo.txt | grep GPL | grep -v LGPL)" && \
|
||||||
|
rm -f piplist.txt pipinfo.txt
|
||||||
|
|
||||||
|
# # Front-end
|
||||||
|
RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \
|
||||||
|
yum install -y nodejs && \
|
||||||
|
curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \
|
||||||
|
rpm --import https://dl.yarnpkg.com/rpm/pubkey.gpg && \
|
||||||
|
yum install -y yarn && \
|
||||||
|
yarn install --ignore-engines && \
|
||||||
|
yarn build && \
|
||||||
|
yarn build-config-app
|
||||||
|
|
||||||
|
# TODO: Build jwtproxy in dist-git
|
||||||
|
# https://jira.coreos.com/browse/QUAY-1315
|
||||||
|
RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \
|
||||||
|
chmod +x /usr/local/bin/jwtproxy
|
||||||
|
|
||||||
|
# TODO: Build prometheus-aggregator in dist-git
|
||||||
|
# https://jira.coreos.com/browse/QUAY-1324
|
||||||
|
RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\
|
||||||
|
chmod +x /usr/local/bin/prometheus-aggregator
|
||||||
|
|
||||||
|
# Update local copy of AWS IP Ranges.
|
||||||
|
RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
|
||||||
|
|
||||||
|
RUN ln -s $QUAYCONF /conf && \
|
||||||
|
mkdir /var/log/nginx && \
|
||||||
|
ln -sf /dev/stdout /var/log/nginx/access.log && \
|
||||||
|
ln -sf /dev/stdout /var/log/nginx/error.log && \
|
||||||
|
chmod -R a+rwx /var/log/nginx
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
RUN UNINSTALL_PKGS="\
|
||||||
|
gcc-c++ \
|
||||||
|
openldap-devel \
|
||||||
|
gpgme-devel \
|
||||||
|
optipng \
|
||||||
|
kernel-headers \
|
||||||
|
" && \
|
||||||
|
yum remove -y $UNINSTALL_PKGS && \
|
||||||
|
yum clean all && \
|
||||||
|
rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache
|
||||||
|
|
||||||
|
EXPOSE 8080 8443 7443
|
||||||
|
|
||||||
|
RUN chgrp -R 0 $QUAYDIR && \
|
||||||
|
chmod -R g=u $QUAYDIR
|
||||||
|
|
||||||
|
RUN mkdir /datastorage && chgrp 0 /datastorage && chmod g=u /datastorage && \
|
||||||
|
mkdir -p /var/log/nginx && chgrp 0 /var/log/nginx && chmod g=u /var/log/nginx && \
|
||||||
|
mkdir -p /conf/stack && chgrp 0 /conf/stack && chmod g=u /conf/stack && \
|
||||||
|
mkdir -p /tmp && chgrp 0 /tmp && chmod g=u /tmp && \
|
||||||
|
chmod g=u /etc/passwd
|
||||||
|
|
||||||
|
RUN chgrp 0 /var/opt/rh/rh-nginx112/log/nginx && chmod g=u /var/opt/rh/rh-nginx112/log/nginx
|
||||||
|
|
||||||
|
VOLUME ["/var/log", "/datastorage", "/tmp", "/conf/stack"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/quay-registry/quay-entrypoint.sh"]
|
||||||
|
CMD ["registry"]
|
||||||
|
|
||||||
|
# root required to create and install certs
|
||||||
|
# https://jira.coreos.com/browse/QUAY-1468
|
||||||
|
# USER 1001
|
||||||
8
Dockerfile.cirun
Normal file
8
Dockerfile.cirun
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
FROM quay-ci-base
|
||||||
|
RUN mkdir -p conf/stack
|
||||||
|
RUN rm -rf test/data/test.db
|
||||||
|
ENV ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE remove-old-fields
|
||||||
|
ADD cirun.config.yaml conf/stack/config.yaml
|
||||||
|
RUN /usr/bin/scl enable python27 rh-nginx112 "LOGGING_LEVEL=INFO python initdb.py"
|
||||||
|
ENTRYPOINT ["/quay-registry/quay-entrypoint.sh"]
|
||||||
|
CMD ["registry"]
|
||||||
19
Dockerfile.dev
Normal file
19
Dockerfile.dev
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# -*- mode: dockerfile -*-
|
||||||
|
# vi: set ft=dockerfile :
|
||||||
|
|
||||||
|
FROM quay.io/quay/quay-base:latest
|
||||||
|
|
||||||
|
WORKDIR $QUAYDIR
|
||||||
|
|
||||||
|
COPY requirements.txt requirements-tests.txt ./
|
||||||
|
|
||||||
|
# Put the virtualenv outside the source directory. This lets us mount
|
||||||
|
# the Quay source as a volume for local development.
|
||||||
|
RUN virtualenv --distribute /venv \
|
||||||
|
&& /venv/bin/pip install -r requirements.txt \
|
||||||
|
&& /venv/bin/pip install -r requirements-tests.txt \
|
||||||
|
&& /venv/bin/pip freeze
|
||||||
|
|
||||||
|
ENV PATH /venv/bin:${PATH}
|
||||||
|
|
||||||
|
RUN ln -s $QUAYCONF /conf
|
||||||
133
Dockerfile.rhel7
Normal file
133
Dockerfile.rhel7
Normal file
|
|
@ -0,0 +1,133 @@
|
||||||
|
FROM registry.redhat.io/rhel7:7.7
|
||||||
|
LABEL maintainer "thomasmckay@redhat.com"
|
||||||
|
|
||||||
|
ENV PYTHON_VERSION=2.7 \
|
||||||
|
PATH=$HOME/.local/bin/:$PATH \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONIOENCODING=UTF-8 \
|
||||||
|
LC_ALL=en_US.UTF-8 \
|
||||||
|
LANG=en_US.UTF-8 \
|
||||||
|
PIP_NO_CACHE_DIR=off
|
||||||
|
|
||||||
|
ENV QUAYDIR /quay-registry
|
||||||
|
ENV QUAYCONF /quay-registry/conf
|
||||||
|
ENV QUAYPATH "."
|
||||||
|
|
||||||
|
RUN mkdir $QUAYDIR
|
||||||
|
WORKDIR $QUAYDIR
|
||||||
|
|
||||||
|
RUN INSTALL_PKGS="\
|
||||||
|
python27 \
|
||||||
|
python27-python-pip \
|
||||||
|
rh-nginx112 rh-nginx112-nginx \
|
||||||
|
openldap \
|
||||||
|
scl-utils \
|
||||||
|
gcc-c++ git \
|
||||||
|
openldap-devel \
|
||||||
|
gpgme-devel \
|
||||||
|
dnsmasq \
|
||||||
|
memcached \
|
||||||
|
openssl \
|
||||||
|
skopeo \
|
||||||
|
" && \
|
||||||
|
yum install -y yum-utils && \
|
||||||
|
yum-config-manager --quiet --disable "*" >/dev/null && \
|
||||||
|
yum-config-manager --quiet --enable \
|
||||||
|
rhel-7-server-rpms \
|
||||||
|
rhel-server-rhscl-7-rpms \
|
||||||
|
rhel-7-server-optional-rpms \
|
||||||
|
rhel-7-server-extras-rpms \
|
||||||
|
--save >/dev/null && \
|
||||||
|
yum -y --setopt=tsflags=nodocs --setopt=skip_missing_names_on_install=False install $INSTALL_PKGS && \
|
||||||
|
yum -y update && \
|
||||||
|
yum -y clean all
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN scl enable python27 "\
|
||||||
|
pip install --upgrade setuptools pip && \
|
||||||
|
pip install -r requirements.txt --no-cache && \
|
||||||
|
pip freeze && \
|
||||||
|
mkdir -p $QUAYDIR/static/webfonts && \
|
||||||
|
mkdir -p $QUAYDIR/static/fonts && \
|
||||||
|
mkdir -p $QUAYDIR/static/ldn && \
|
||||||
|
PYTHONPATH=$QUAYPATH python -m external_libraries \
|
||||||
|
"
|
||||||
|
|
||||||
|
RUN cp -r $QUAYDIR/static/ldn $QUAYDIR/config_app/static/ldn && \
|
||||||
|
cp -r $QUAYDIR/static/fonts $QUAYDIR/config_app/static/fonts && \
|
||||||
|
cp -r $QUAYDIR/static/webfonts $QUAYDIR/config_app/static/webfonts
|
||||||
|
|
||||||
|
# Check python dependencies for GPL
|
||||||
|
# Due to the following bug, pip results must be piped to a file before grepping:
|
||||||
|
# https://github.com/pypa/pip/pull/3304
|
||||||
|
# 'docutils' is a setup dependency of botocore required by s3transfer. It's under
|
||||||
|
# GPLv3, and so is manually removed.
|
||||||
|
RUN rm -Rf /opt/rh/python27/root/usr/lib/python2.7/site-packages/docutils && \
|
||||||
|
scl enable python27 "pip freeze" | grep -v '^-e' | awk -F == '{print $1}' | grep -v docutils > piplist.txt && \
|
||||||
|
scl enable python27 "xargs -a piplist.txt pip --disable-pip-version-check show" > pipinfo.txt && \
|
||||||
|
test -z "$(cat pipinfo.txt | grep GPL | grep -v LGPL)" && \
|
||||||
|
rm -f piplist.txt pipinfo.txt
|
||||||
|
|
||||||
|
# Front-end
|
||||||
|
RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \
|
||||||
|
yum install -y nodejs && \
|
||||||
|
curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \
|
||||||
|
rpm --import https://dl.yarnpkg.com/rpm/pubkey.gpg && \
|
||||||
|
yum install -y yarn && \
|
||||||
|
yarn install --ignore-engines && \
|
||||||
|
yarn build && \
|
||||||
|
yarn build-config-app
|
||||||
|
|
||||||
|
# TODO: Build jwtproxy in dist-git
|
||||||
|
# https://jira.coreos.com/browse/QUAY-1315
|
||||||
|
RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \
|
||||||
|
chmod +x /usr/local/bin/jwtproxy
|
||||||
|
|
||||||
|
# TODO: Build prometheus-aggregator in dist-git
|
||||||
|
# https://jira.coreos.com/browse/QUAY-1324
|
||||||
|
RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\
|
||||||
|
chmod +x /usr/local/bin/prometheus-aggregator
|
||||||
|
|
||||||
|
# Update local copy of AWS IP Ranges.
|
||||||
|
RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
|
||||||
|
|
||||||
|
RUN ln -s $QUAYCONF /conf && \
|
||||||
|
mkdir /var/log/nginx && \
|
||||||
|
ln -sf /dev/stdout /var/log/nginx/access.log && \
|
||||||
|
ln -sf /dev/stdout /var/log/nginx/error.log && \
|
||||||
|
chmod -R a+rwx /var/log/nginx
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
RUN UNINSTALL_PKGS="\
|
||||||
|
gcc-c++ git \
|
||||||
|
openldap-devel \
|
||||||
|
gpgme-devel \
|
||||||
|
optipng \
|
||||||
|
kernel-headers \
|
||||||
|
" && \
|
||||||
|
yum remove -y $UNINSTALL_PKGS && \
|
||||||
|
yum clean all && \
|
||||||
|
rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache
|
||||||
|
|
||||||
|
EXPOSE 8080 8443 7443
|
||||||
|
|
||||||
|
RUN chgrp -R 0 $QUAYDIR && \
|
||||||
|
chmod -R g=u $QUAYDIR
|
||||||
|
|
||||||
|
RUN mkdir /datastorage && chgrp 0 /datastorage && chmod g=u /datastorage && \
|
||||||
|
mkdir -p /var/log/nginx && chgrp 0 /var/log/nginx && chmod g=u /var/log/nginx && \
|
||||||
|
mkdir -p /conf/stack && chgrp 0 /conf/stack && chmod g=u /conf/stack && \
|
||||||
|
mkdir -p /tmp && chgrp 0 /tmp && chmod g=u /tmp && \
|
||||||
|
chmod g=u /etc/passwd
|
||||||
|
|
||||||
|
RUN chgrp 0 /var/opt/rh/rh-nginx112/log/nginx && chmod g=u /var/opt/rh/rh-nginx112/log/nginx
|
||||||
|
|
||||||
|
VOLUME ["/var/log", "/datastorage", "/tmp", "/conf/stack"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/quay-registry/quay-entrypoint.sh"]
|
||||||
|
CMD ["registry"]
|
||||||
|
|
||||||
|
# root required to create and install certs
|
||||||
|
# https://jira.coreos.com/browse/QUAY-1468
|
||||||
|
# USER 1001
|
||||||
66
GOVERNANCE.md
Normal file
66
GOVERNANCE.md
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
# Project Quay Governance
|
||||||
|
|
||||||
|
Project Quay is run according to the guidelines specified below. This is a living document and is expected to evolve along with Project Quay itself.
|
||||||
|
|
||||||
|
## Principles
|
||||||
|
|
||||||
|
Project Quay strives to follow these principles at all times:
|
||||||
|
* Openness - Quay evolves and improves out in the open, with transparent work and decision making that is clear and well understood.
|
||||||
|
* Respectfulness - Quay is a project for a diverse community where different points of view are welcomed. Healthy and respectful discussions help us meet our goals and deliver a better end product.
|
||||||
|
* Meritocracy - In the Quay community all ideas are heard but only the best ideas help drive the project forward. As an open, respectful community we will judge all ideas on their technical merit and alignment with Quay's design principles.
|
||||||
|
* Accountability - The Quay community is accountable
|
||||||
|
* to our users to deliver the best software possible
|
||||||
|
* to the project to ensure each Contributor and Maintainer carries out their duties to the best of their abilities
|
||||||
|
* to itself to ensure the Quay remains a project where indviduals can be passionate about contributing their time and energy
|
||||||
|
|
||||||
|
## Maintainers
|
||||||
|
|
||||||
|
Maintainers play a special role to ensure that contributions align with the expected quality, consistency and long term vision for Project Quay. Each Maintainer is vital to the success of Project Quay and has decided to make the commitment to that cause. Being a Maintainer is difficult work and not for everyone. Therefore Project Quay will have a small group of Maintainers- as many as deemed necessary to handle the pipeline of contributions being made to the project.
|
||||||
|
|
||||||
|
### Becoming a Maintainer
|
||||||
|
|
||||||
|
Each Maintainer must also be a Contributor. Candidates for the Maintainer role are individuals who have made recent, substantial and recurring contributions to the project. The existing Maintainers will periodically identify Contributors and make recommendations to the community that those individuals become Maintainers. The Maintainers will then vote on the candidate and if so agreed the candidate will be invited to raise a PR to add their name into the MAINTAINERS.md file. Approval of that PR signals the Contributor is now a Maintainer.
|
||||||
|
|
||||||
|
### Responsibilities of a Maintainer
|
||||||
|
|
||||||
|
Project Quay's success depends on how well Maintainers perform their duties. Maintainers are responsible to monitor Slack and e-mail lists, help triage issues on the Project Quay JIRA board, review PRs and ensure responses are being provided to Contributors, assist with regular Project Quay releases. If Contributors are the lifeblood of an open source community, the Maintainers act as the heart, hands, eyes and ears, helping to keep the project moving and viable.
|
||||||
|
|
||||||
|
### Stepping Down as a Maintainer
|
||||||
|
|
||||||
|
A Maintainer may decide they are no longer interested in or able to carry out the role. In such a situation the Maintainer should notify the other Maintainers of their intentions to step down and help identify a replacement from existing Contributors. Ideally the outgoing Maintainer will ensure that any outstanding work has been transitioned to another Maintainer. To carry out the actual removal the outgoing Maintainer raises a PR against MAINTAINERS.md file to remove their name.
|
||||||
|
|
||||||
|
## Contributors
|
||||||
|
|
||||||
|
Anyone can be a Contributor to Project Quay. No special approval is required- simply go through our Getting Started guide, fork one of our repositories and submit a PR. All types of conributions will be welcome, whether it is through bug reports via JIRA, code, or documentation.
|
||||||
|
|
||||||
|
## Sub-Projects
|
||||||
|
|
||||||
|
Project Quay will be primarily focused on the delivery of Quay itself but also contains various sub-projects such as Clair and Quay-Builders. Each sub-project must have their own dedicated repositories containing a MAINTAINERS.md file. Each sub-project will abide by this Governance model.
|
||||||
|
|
||||||
|
Requests for new sub-projects under Project Quay should be raised to the Maintainers.
|
||||||
|
|
||||||
|
## Code of Conduct
|
||||||
|
|
||||||
|
Project Quay abides by the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
|
||||||
|
|
||||||
|
## How Decisons Are Made
|
||||||
|
|
||||||
|
Most of the decison making for Project Quay will happen through the regular PR approval process. We stand by the notion that what exists in the Project Quay repositories are the end result of countless community-driven decisions.
|
||||||
|
|
||||||
|
When a more complex decision is required, for example a technical issue related to a PR, it is expected that involved parties will resolve the dispute in a respectful and efficent manner. If the dispute cannot be resolved between the involved parties then the Maintainers will review the dispute and come to an agreement via majority vote amongst themselves. All decision making should be tracked via a JIRA issue and performed transparently via the Project Quay communications channels.
|
||||||
|
|
||||||
|
## Project Quay Releases
|
||||||
|
|
||||||
|
On a regular basis, Project Quay will issue a release. The release cadence will not be strictly defined but should happen approximately every 3 months. Maintainers will be part of a rotating "Release Nanny" role whereby each Maintainer shares the responsibility of creating a Quay release.
|
||||||
|
|
||||||
|
Release duties include:
|
||||||
|
* Creating the Release Notes
|
||||||
|
* Verifying the automated tests have passed
|
||||||
|
* Building the necessary Quay, Clair-JWT, and Quay-Builder container images
|
||||||
|
* Publishing the container images to quay.io
|
||||||
|
* Updating the github release pages
|
||||||
|
* Notifying the community of the new release
|
||||||
|
|
||||||
|
## DCO and Licenses
|
||||||
|
|
||||||
|
Project Quay uses the [Apache 2.0](https://opensource.org/licenses/Apache-2.0) license.
|
||||||
201
LICENSE
Normal file
201
LICENSE
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright {yyyy} {name of copyright owner}
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
171
Makefile
Normal file
171
Makefile
Normal file
|
|
@ -0,0 +1,171 @@
|
||||||
|
SHELL := /bin/bash
|
||||||
|
|
||||||
|
export PATH := ./venv/bin:$(PATH)
|
||||||
|
|
||||||
|
SHA := $(shell git rev-parse --short HEAD )
|
||||||
|
REPO := quay.io/quay/quay
|
||||||
|
TAG := $(REPO):$(SHA)
|
||||||
|
|
||||||
|
MODIFIED_FILES_COUNT = $(shell git diff --name-only origin/master | grep -E .+\.py$ | wc -l)
|
||||||
|
GIT_MERGE_BASED = $(shell git merge-base origin/master HEAD)
|
||||||
|
MODIFIED_FILES = $(shell git diff --name-only $(GIT_MERGE_BASED) | grep -E .+\.py$ | paste -sd ' ')
|
||||||
|
|
||||||
|
show-modified:
|
||||||
|
echo $(MODIFIED_FILES)
|
||||||
|
|
||||||
|
.PHONY: all unit-test registry-test registry-test-old buildman-test test pkgs build run clean
|
||||||
|
|
||||||
|
all: clean pkgs test build
|
||||||
|
|
||||||
|
pkgs: requirements.txt requirements-dev.txt requirements-tests.txt
|
||||||
|
pip install -r $<
|
||||||
|
|
||||||
|
requirements.txt: requirements-nover.txt
|
||||||
|
# Create a new virtualenv and activate it
|
||||||
|
pyenv virtualenv 2.7.12 quay-deps
|
||||||
|
pyenv activate quay-deps
|
||||||
|
|
||||||
|
# Install unversioned dependencies with your changes
|
||||||
|
pip install -r requirements-nover.txt
|
||||||
|
|
||||||
|
# Run the unit test suite
|
||||||
|
$(MAKE) unit
|
||||||
|
|
||||||
|
# Freeze the versions of all of the dependencies
|
||||||
|
pip freeze > requirements.txt
|
||||||
|
|
||||||
|
# Delete the virtualenv
|
||||||
|
pyenv uninstall quay-deps
|
||||||
|
|
||||||
|
QUAY_CONFIG ?= ../quay-config
|
||||||
|
conf/stack/license: $(QUAY_CONFIG)/local/license
|
||||||
|
mkdir -p conf/stack
|
||||||
|
ln -s $(QUAY_CONFIG)/local/license conf/stack/license
|
||||||
|
|
||||||
|
unit-test:
|
||||||
|
ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields TEST=true PYTHONPATH="." py.test \
|
||||||
|
--cov="." --cov-report=html --cov-report=term-missing \
|
||||||
|
--timeout=3600 --verbose -x \
|
||||||
|
./
|
||||||
|
|
||||||
|
registry-test:
|
||||||
|
TEST=true ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields PYTHONPATH="." py.test \
|
||||||
|
--cov="." --cov-report=html --cov-report=term-missing \
|
||||||
|
--timeout=3600 --verbose --show-count -x \
|
||||||
|
test/registry/registry_tests.py
|
||||||
|
|
||||||
|
registry-test-old:
|
||||||
|
TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test \
|
||||||
|
--cov="." --cov-report=html --cov-report=term-missing \
|
||||||
|
--timeout=3600 --verbose --show-count -x \
|
||||||
|
./test/registry_tests.py
|
||||||
|
|
||||||
|
buildman-test:
|
||||||
|
TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test \
|
||||||
|
--cov="." --cov-report=html --cov-report=term-missing \
|
||||||
|
--timeout=3600 --verbose --show-count -x \
|
||||||
|
./buildman/
|
||||||
|
|
||||||
|
certs-test:
|
||||||
|
./test/test_certs_install.sh
|
||||||
|
|
||||||
|
full-db-test: ensure-test-db
|
||||||
|
TEST=true PYTHONPATH=. QUAY_OVERRIDE_CONFIG='{"DATABASE_SECRET_KEY": "anothercrazykey!"}' \
|
||||||
|
ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields alembic upgrade head
|
||||||
|
TEST=true PYTHONPATH=. ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields \
|
||||||
|
SKIP_DB_SCHEMA=true py.test --timeout=7200 \
|
||||||
|
--verbose --show-count -x --ignore=endpoints/appr/test/ \
|
||||||
|
./
|
||||||
|
|
||||||
|
clients-test:
|
||||||
|
cd test/clients; python clients_test.py
|
||||||
|
|
||||||
|
test: unit-test registry-test registry-test-old certs-test
|
||||||
|
|
||||||
|
ensure-test-db:
|
||||||
|
@if [ -z $(TEST_DATABASE_URI) ]; then \
|
||||||
|
echo "TEST_DATABASE_URI is undefined"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
PG_PASSWORD := quay
|
||||||
|
PG_USER := quay
|
||||||
|
PG_HOST := postgresql://$(PG_USER):$(PG_PASSWORD)@localhost/quay
|
||||||
|
|
||||||
|
test_postgres : TEST_ENV := SKIP_DB_SCHEMA=true TEST=true \
|
||||||
|
TEST_DATABASE_URI=$(PG_HOST) PYTHONPATH=.
|
||||||
|
|
||||||
|
test_postgres:
|
||||||
|
docker rm -f postgres-testrunner-postgres || true
|
||||||
|
docker run --name postgres-testrunner-postgres \
|
||||||
|
-e POSTGRES_PASSWORD=$(PG_PASSWORD) -e POSTGRES_USER=${PG_USER} \
|
||||||
|
-p 5432:5432 -d postgres:9.2
|
||||||
|
until pg_isready -d $(PG_HOST); do sleep 1; echo "Waiting for postgres"; done
|
||||||
|
$(TEST_ENV) alembic upgrade head
|
||||||
|
$(TEST_ENV) py.test --timeout=7200 --verbose --show-count ./ --color=no \
|
||||||
|
--ignore=endpoints/appr/test/ -x
|
||||||
|
docker rm -f postgres-testrunner-postgres || true
|
||||||
|
|
||||||
|
WEBPACK := node_modules/.bin/webpack
|
||||||
|
$(WEBPACK): package.json
|
||||||
|
npm install webpack
|
||||||
|
npm install
|
||||||
|
|
||||||
|
BUNDLE := static/js/build/bundle.js
|
||||||
|
$(BUNDLE): $(WEBPACK) tsconfig.json webpack.config.js typings.json
|
||||||
|
$(WEBPACK)
|
||||||
|
|
||||||
|
GRUNT := grunt/node_modules/.bin/grunt
|
||||||
|
$(GRUNT): grunt/package.json
|
||||||
|
cd grunt && npm install
|
||||||
|
|
||||||
|
JS := quay-frontend.js quay-frontend.min.js template-cache.js
|
||||||
|
CSS := quay-frontend.css
|
||||||
|
DIST := $(addprefix static/dist/, $(JS) $(CSS) cachebusters.json)
|
||||||
|
$(DIST): $(GRUNT)
|
||||||
|
cd grunt && ../$(GRUNT)
|
||||||
|
|
||||||
|
build: $(WEBPACK) $(GRUNT)
|
||||||
|
|
||||||
|
docker-build: pkgs build
|
||||||
|
ifneq (0,$(shell git status --porcelain | awk 'BEGIN {print $N}'))
|
||||||
|
echo 'dirty build not supported - run `FORCE=true make clean` to remove'
|
||||||
|
exit 1
|
||||||
|
endif
|
||||||
|
# get named head (ex: branch, tag, etc..)
|
||||||
|
NAME = $(shell git rev-parse --abbrev-ref HEAD)
|
||||||
|
# checkout commit so .git/HEAD points to full sha (used in Dockerfile)
|
||||||
|
git checkout $(SHA)
|
||||||
|
docker build -t $(TAG) .
|
||||||
|
git checkout $(NAME)
|
||||||
|
echo $(TAG)
|
||||||
|
|
||||||
|
run: license
|
||||||
|
goreman start
|
||||||
|
|
||||||
|
|
||||||
|
clean:
|
||||||
|
find . -name "*.pyc" -exec rm -rf {} \;
|
||||||
|
rm -rf node_modules 2> /dev/null
|
||||||
|
rm -rf grunt/node_modules 2> /dev/null
|
||||||
|
rm -rf dest 2> /dev/null
|
||||||
|
rm -rf dist 2> /dev/null
|
||||||
|
rm -rf .cache 2> /dev/null
|
||||||
|
rm -rf static/js/build
|
||||||
|
rm -rf static/build
|
||||||
|
rm -rf static/dist
|
||||||
|
rm -rf build
|
||||||
|
rm -rf conf/stack
|
||||||
|
rm -rf screenshots
|
||||||
|
|
||||||
|
|
||||||
|
yapf-all:
|
||||||
|
yapf -r . -p -i
|
||||||
|
|
||||||
|
|
||||||
|
yapf-diff:
|
||||||
|
if [ $(MODIFIED_FILES_COUNT) -ne 0 ]; then yapf -d -p $(MODIFIED_FILES) ; fi
|
||||||
|
|
||||||
|
|
||||||
|
yapf-test:
|
||||||
|
if [ `yapf -d -p $(MODIFIED_FILES) | wc -l` -gt 0 ] ; then false ; else true ;fi
|
||||||
69
Makefile.ci
Normal file
69
Makefile.ci
Normal file
|
|
@ -0,0 +1,69 @@
|
||||||
|
SHELL := /bin/bash
|
||||||
|
PYTEST_MARK ?= shard_1_of_1
|
||||||
|
|
||||||
|
export PATH := ./venv/bin:$(PATH)
|
||||||
|
|
||||||
|
.PHONY: all unit-test registry-test registry-test-old test
|
||||||
|
|
||||||
|
all: test
|
||||||
|
|
||||||
|
unit-test:
|
||||||
|
TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test \
|
||||||
|
-m $(PYTEST_MARK) \
|
||||||
|
--cov="." --cov-report=html --cov-report=term-missing \
|
||||||
|
--timeout=3600 --verbose -x \
|
||||||
|
./
|
||||||
|
|
||||||
|
registry-test:
|
||||||
|
TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test \
|
||||||
|
-m $(PYTEST_MARK) \
|
||||||
|
--cov="." --cov-report=html --cov-report=term-missing \
|
||||||
|
--timeout=3600 --verbose --show-count -x \
|
||||||
|
test/registry/registry_tests.py
|
||||||
|
|
||||||
|
registry-test-old:
|
||||||
|
TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test \
|
||||||
|
--cov="." --cov-report=html --cov-report=term-missing \
|
||||||
|
--timeout=3600 --verbose --show-count -x \
|
||||||
|
./test/registry_tests.py
|
||||||
|
|
||||||
|
certs-test:
|
||||||
|
./test/test_certs_install.sh
|
||||||
|
|
||||||
|
gunicorn-tests:
|
||||||
|
./test/test_gunicorn_running.sh
|
||||||
|
|
||||||
|
full-db-test: ensure-test-db
|
||||||
|
TEST=true PYTHONPATH=. QUAY_OVERRIDE_CONFIG='{"DATABASE_SECRET_KEY": "anothercrazykey!"}' \
|
||||||
|
ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields alembic upgrade head
|
||||||
|
TEST=true PYTHONPATH=. ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields \
|
||||||
|
SKIP_DB_SCHEMA=true py.test --timeout=7200 \
|
||||||
|
-m $(PYTEST_MARK) \
|
||||||
|
--verbose --show-count -x --ignore=endpoints/appr/test/ \
|
||||||
|
./
|
||||||
|
|
||||||
|
test: unit-test registry-test
|
||||||
|
|
||||||
|
ensure-test-db:
|
||||||
|
@if [ -z $(TEST_DATABASE_URI) ]; then \
|
||||||
|
echo "TEST_DATABASE_URI is undefined"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
PG_PASSWORD := quay
|
||||||
|
PG_USER := quay
|
||||||
|
PG_HOST := postgresql://$(PG_USER):$(PG_PASSWORD)@localhost/quay
|
||||||
|
|
||||||
|
test_postgres : TEST_ENV := SKIP_DB_SCHEMA=true TEST=true \
|
||||||
|
TEST_DATABASE_URI=$(PG_HOST) PYTHONPATH=.
|
||||||
|
|
||||||
|
test_postgres:
|
||||||
|
docker rm -f postgres-testrunner-postgres || true
|
||||||
|
docker run --name postgres-testrunner-postgres \
|
||||||
|
-e POSTGRES_PASSWORD=$(PG_PASSWORD) -e POSTGRES_USER=${PG_USER} \
|
||||||
|
-p 5432:5432 -d postgres:9.2
|
||||||
|
until pg_isready -d $(PG_HOST); do sleep 1; echo "Waiting for postgres"; done
|
||||||
|
$(TEST_ENV) alembic upgrade head
|
||||||
|
$(TEST_ENV) py.test --timeout=7200 --verbose --show-count ./ --color=no \
|
||||||
|
--ignore=endpoints/appr/test/ -x
|
||||||
|
docker rm -f postgres-testrunner-postgres || true
|
||||||
4
Procfile
Normal file
4
Procfile
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
app: gunicorn -c conf/gunicorn_local.py application:application
|
||||||
|
webpack: npm run watch
|
||||||
|
builder: python -m buildman.builder
|
||||||
|
|
||||||
75
README.md
Normal file
75
README.md
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
# Project Quay
|
||||||
|
|
||||||
|
[](https://travis-ci.com/quay/projectquay)
|
||||||
|
|
||||||
|
:warning: The `master` branch may be in an *unstable or even broken state* during development.
|
||||||
|
Please use [releases] instead of the `master` branch in order to get stable software.
|
||||||
|
|
||||||
|
[releases]: https://github.com/quay/projectquay/releases
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Project Quay builds, stores, and distributes your container images.
|
||||||
|
|
||||||
|
High-level features include:
|
||||||
|
|
||||||
|
- Docker Registry Protocol [v1], [v2]
|
||||||
|
- Docker Manifest Schema [v2.1], [v2.2]
|
||||||
|
- [AppC Image Discovery] via on-demand transcoding
|
||||||
|
- Image Squashing via on-demand transcoding
|
||||||
|
- Authentication provided by [LDAP], [Keystone], [OIDC], [Google], [GitHub]
|
||||||
|
- ACLs, team management, and auditability logs
|
||||||
|
- Geo-replicated storage provided by local filesystems, [S3], [GCS], [Swift], [Ceph]
|
||||||
|
- Continuous Integration integrated with [GitHub], [Bitbucket], [GitLab], and [git]
|
||||||
|
- Security Vulnerability Analysis via [Clair]
|
||||||
|
- [Swagger]-compliant HTTP API
|
||||||
|
|
||||||
|
[v1]: https://docs.docker.com/v1.6/reference/api/registry_api/
|
||||||
|
[v2]: https://docs.docker.com/v1.6/registry/
|
||||||
|
[v2.1]: https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md
|
||||||
|
[v2.2]: https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md
|
||||||
|
[AppC Image Discovery]: https://github.com/appc/spec/blob/master/spec/discovery.md
|
||||||
|
[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol
|
||||||
|
[Keystone]: http://docs.openstack.org/developer/keystone
|
||||||
|
[OIDC]: https://en.wikipedia.org/wiki/OpenID_Connect
|
||||||
|
[Google]: https://developers.google.com/identity/sign-in/web/sign-in
|
||||||
|
[GitHub]: https://developer.github.com/v3/oauth
|
||||||
|
[S3]: https://aws.amazon.com/s3
|
||||||
|
[GCS]: https://cloud.google.com/storage
|
||||||
|
[Swift]: http://swift.openstack.org
|
||||||
|
[Ceph]: http://docs.ceph.com/docs/master/radosgw/config
|
||||||
|
[GitHub]: https://github.com
|
||||||
|
[Bitbucket]: https://bitbucket.com
|
||||||
|
[GitLab]: https://gitlab.com
|
||||||
|
[git]: https://git-scm.com
|
||||||
|
[Clair]: https://github.com/quay/clair
|
||||||
|
[Swagger]: http://swagger.io
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
* Explore a live instance of Project Quay hosted at [Quay.io]
|
||||||
|
* Watch [talks] given about Project Quay
|
||||||
|
* Review the [documentation] for Red Hat Quay
|
||||||
|
* Get up and running with a containerized [development environment]
|
||||||
|
|
||||||
|
[Quay.io]: https://quay.io
|
||||||
|
[talks]: /docs/talks.md
|
||||||
|
[documentation]: https://access.redhat.com/documentation/en-us/red_hat_quay
|
||||||
|
[development environment]: /docs/development-container.md
|
||||||
|
|
||||||
|
## Community
|
||||||
|
|
||||||
|
* Mailing List: [quay-dev@googlegroups.com]
|
||||||
|
* IRC: #quay on [freenode.net]
|
||||||
|
* Bug tracking: [GitHub Issues]
|
||||||
|
* Security Issues: [security@redhat.com]
|
||||||
|
|
||||||
|
[quay-dev@googlegroups.com]: https://groups.google.com/forum/#!forum/quay-dev
|
||||||
|
[freenode.net]: https://webchat.freenode.net
|
||||||
|
[GitHub Issues]: https://github.com/quay/projectquay/issues
|
||||||
|
[security@redhat.com]: mailto:security@redhat.com
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
Project Quay is under the Apache 2.0 license.
|
||||||
|
See the LICENSE file for details.
|
||||||
49
TESTING.md
Normal file
49
TESTING.md
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
# Testing quay
|
||||||
|
|
||||||
|
## Unit tests (run in CI automatically)
|
||||||
|
|
||||||
|
Basic unit tests for testing all the functionality of Quay:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make unit-test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Registry tests (run in CI automatically)
|
||||||
|
|
||||||
|
Quay has two sets of registry tests (current and legacy), which simulate Docker clients by executing
|
||||||
|
REST operations against a spanwed Quay.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make registry-test
|
||||||
|
make registry-test-old
|
||||||
|
```
|
||||||
|
|
||||||
|
## Certs tests (run in CI automatically)
|
||||||
|
|
||||||
|
Ensures that custom TLS certificates are correctly loaded into the Quay container on startup.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make certs-test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Full database tests (run in CI automatically)
|
||||||
|
|
||||||
|
The full database tests runs the entire suite of Quay unit tests against a real running database
|
||||||
|
instance.
|
||||||
|
|
||||||
|
NOTE: The database *must be running* on the local machine before this test can be run.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
TEST_DATABASE_URI=database-connection-string make full-db-test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Clients tests (must be manually run)
|
||||||
|
|
||||||
|
The clients test spawns CoreOS virtual machines via Vagrant and VirtualBox and runs real Docker/podman
|
||||||
|
commands against a *running Quay*.
|
||||||
|
|
||||||
|
NOTE: A Quay *must be running* on the local machine before this test can be run.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make clients-test 10.0.2.2:5000 # IP+Port of the Quay on the host machine.
|
||||||
|
```
|
||||||
47
_init.py
Normal file
47
_init.py
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from util.config.provider import get_config_provider
|
||||||
|
|
||||||
|
|
||||||
|
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
CONF_DIR = os.getenv("QUAYCONF", os.path.join(ROOT_DIR, "conf/"))
|
||||||
|
STATIC_DIR = os.path.join(ROOT_DIR, 'static/')
|
||||||
|
STATIC_LDN_DIR = os.path.join(STATIC_DIR, 'ldn/')
|
||||||
|
STATIC_FONTS_DIR = os.path.join(STATIC_DIR, 'fonts/')
|
||||||
|
STATIC_WEBFONTS_DIR = os.path.join(STATIC_DIR, 'webfonts/')
|
||||||
|
TEMPLATE_DIR = os.path.join(ROOT_DIR, 'templates/')
|
||||||
|
|
||||||
|
IS_TESTING = 'TEST' in os.environ
|
||||||
|
IS_BUILDING = 'BUILDING' in os.environ
|
||||||
|
IS_KUBERNETES = 'KUBERNETES_SERVICE_HOST' in os.environ
|
||||||
|
OVERRIDE_CONFIG_DIRECTORY = os.path.join(CONF_DIR, 'stack/')
|
||||||
|
|
||||||
|
|
||||||
|
config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py',
|
||||||
|
testing=IS_TESTING, kubernetes=IS_KUBERNETES)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_version_number_changelog():
|
||||||
|
try:
|
||||||
|
with open(os.path.join(ROOT_DIR, 'CHANGELOG.md')) as f:
|
||||||
|
return re.search(r'(v[0-9]+\.[0-9]+\.[0-9]+)', f.readline()).group(0)
|
||||||
|
except IOError:
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
def _get_git_sha():
|
||||||
|
if os.path.exists("GIT_HEAD"):
|
||||||
|
with open(os.path.join(ROOT_DIR, "GIT_HEAD")) as f:
|
||||||
|
return f.read()
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
return subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()[0:8]
|
||||||
|
except (OSError, subprocess.CalledProcessError, Exception):
|
||||||
|
pass
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
__version__ = _get_version_number_changelog()
|
||||||
|
__gitrev__ = _get_git_sha()
|
||||||
22
active_migration.py
Normal file
22
active_migration.py
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
from enum import Enum, unique
|
||||||
|
from data.migrationutil import DefinedDataMigration, MigrationPhase
|
||||||
|
|
||||||
|
@unique
|
||||||
|
class ERTMigrationFlags(Enum):
|
||||||
|
""" Flags for the encrypted robot token migration. """
|
||||||
|
READ_OLD_FIELDS = 'read-old'
|
||||||
|
WRITE_OLD_FIELDS = 'write-old'
|
||||||
|
|
||||||
|
|
||||||
|
ActiveDataMigration = DefinedDataMigration(
|
||||||
|
'encrypted_robot_tokens',
|
||||||
|
'ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE',
|
||||||
|
[
|
||||||
|
MigrationPhase('add-new-fields', 'c13c8052f7a6', [ERTMigrationFlags.READ_OLD_FIELDS,
|
||||||
|
ERTMigrationFlags.WRITE_OLD_FIELDS]),
|
||||||
|
MigrationPhase('backfill-then-read-only-new',
|
||||||
|
'703298a825c2', [ERTMigrationFlags.WRITE_OLD_FIELDS]),
|
||||||
|
MigrationPhase('stop-writing-both', '703298a825c2', []),
|
||||||
|
MigrationPhase('remove-old-fields', 'c059b952ed76', []),
|
||||||
|
]
|
||||||
|
)
|
||||||
68
alembic.ini
Normal file
68
alembic.ini
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
# A generic, single database configuration.
|
||||||
|
|
||||||
|
[alembic]
|
||||||
|
# path to migration scripts
|
||||||
|
script_location = data/migrations
|
||||||
|
|
||||||
|
# template used to generate migration files
|
||||||
|
# file_template = %%(rev)s_%%(slug)s
|
||||||
|
|
||||||
|
# max length of characters to apply to the
|
||||||
|
# "slug" field
|
||||||
|
#truncate_slug_length = 40
|
||||||
|
|
||||||
|
# set to 'true' to run the environment during
|
||||||
|
# the 'revision' command, regardless of autogenerate
|
||||||
|
# revision_environment = false
|
||||||
|
|
||||||
|
# set to 'true' to allow .pyc and .pyo files without
|
||||||
|
# a source .py file to be detected as revisions in the
|
||||||
|
# versions/ directory
|
||||||
|
# sourceless = false
|
||||||
|
|
||||||
|
sqlalchemy.url = sqlite:///will/be/overridden
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic,peewee,boto
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = DEBUG
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_peewee]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = peewee
|
||||||
|
|
||||||
|
[logger_boto]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = boto
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
||||||
314
app.py
Normal file
314
app.py
Normal file
|
|
@ -0,0 +1,314 @@
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
from Crypto.PublicKey import RSA
|
||||||
|
from flask import Flask, request, Request
|
||||||
|
from flask_login import LoginManager
|
||||||
|
from flask_mail import Mail
|
||||||
|
from flask_principal import Principal
|
||||||
|
from jwkest.jwk import RSAKey
|
||||||
|
from werkzeug.contrib.fixers import ProxyFix
|
||||||
|
from werkzeug.exceptions import HTTPException
|
||||||
|
|
||||||
|
import features
|
||||||
|
|
||||||
|
from _init import (config_provider, CONF_DIR, IS_KUBERNETES, IS_TESTING, OVERRIDE_CONFIG_DIRECTORY,
|
||||||
|
IS_BUILDING)
|
||||||
|
|
||||||
|
from auth.auth_context import get_authenticated_user
|
||||||
|
from avatars.avatars import Avatar
|
||||||
|
from buildman.manager.buildcanceller import BuildCanceller
|
||||||
|
from data import database
|
||||||
|
from data import model
|
||||||
|
from data import logs_model
|
||||||
|
from data.archivedlogs import LogArchive
|
||||||
|
from data.billing import Billing
|
||||||
|
from data.buildlogs import BuildLogs
|
||||||
|
from data.cache import get_model_cache
|
||||||
|
from data.model.user import LoginWrappedDBUser
|
||||||
|
from data.queue import WorkQueue, BuildMetricQueueReporter
|
||||||
|
from data.userevent import UserEventsBuilderModule
|
||||||
|
from data.userfiles import Userfiles
|
||||||
|
from data.users import UserAuthentication
|
||||||
|
from data.registry_model import registry_model
|
||||||
|
from path_converters import RegexConverter, RepositoryPathConverter, APIRepositoryPathConverter
|
||||||
|
from oauth.services.github import GithubOAuthService
|
||||||
|
from oauth.services.gitlab import GitLabOAuthService
|
||||||
|
from oauth.loginmanager import OAuthLoginManager
|
||||||
|
from storage import Storage
|
||||||
|
from util.config import URLSchemeAndHostname
|
||||||
|
from util.log import filter_logs
|
||||||
|
from util import get_app_url
|
||||||
|
from util.secscan.secscan_util import get_blob_download_uri_getter
|
||||||
|
from util.ipresolver import IPResolver
|
||||||
|
from util.saas.analytics import Analytics
|
||||||
|
from util.saas.useranalytics import UserAnalytics
|
||||||
|
from util.saas.exceptionlog import Sentry
|
||||||
|
from util.names import urn_generator
|
||||||
|
from util.config.configutil import generate_secret_key
|
||||||
|
from util.config.superusermanager import SuperUserManager
|
||||||
|
from util.label_validator import LabelValidator
|
||||||
|
from util.metrics.metricqueue import MetricQueue
|
||||||
|
from util.metrics.prometheus import PrometheusPlugin
|
||||||
|
from util.saas.cloudwatch import start_cloudwatch_sender
|
||||||
|
from util.secscan.api import SecurityScannerAPI
|
||||||
|
from util.repomirror.api import RepoMirrorAPI
|
||||||
|
from util.tufmetadata.api import TUFMetadataAPI
|
||||||
|
from util.security.instancekeys import InstanceKeys
|
||||||
|
from util.security.signing import Signer
|
||||||
|
|
||||||
|
|
||||||
|
OVERRIDE_CONFIG_YAML_FILENAME = os.path.join(CONF_DIR, 'stack/config.yaml')
|
||||||
|
OVERRIDE_CONFIG_PY_FILENAME = os.path.join(CONF_DIR, 'stack/config.py')
|
||||||
|
|
||||||
|
OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG'
|
||||||
|
|
||||||
|
DOCKER_V2_SIGNINGKEY_FILENAME = 'docker_v2.pem'
|
||||||
|
INIT_SCRIPTS_LOCATION = '/conf/init/'
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Instantiate the configuration.
|
||||||
|
is_testing = IS_TESTING
|
||||||
|
is_kubernetes = IS_KUBERNETES
|
||||||
|
is_building = IS_BUILDING
|
||||||
|
|
||||||
|
if is_testing:
|
||||||
|
from test.testconfig import TestConfig
|
||||||
|
logger.debug('Loading test config.')
|
||||||
|
app.config.from_object(TestConfig())
|
||||||
|
else:
|
||||||
|
from config import DefaultConfig
|
||||||
|
logger.debug('Loading default config.')
|
||||||
|
app.config.from_object(DefaultConfig())
|
||||||
|
app.teardown_request(database.close_db_filter)
|
||||||
|
|
||||||
|
# Load the override config via the provider.
|
||||||
|
config_provider.update_app_config(app.config)
|
||||||
|
|
||||||
|
# Update any configuration found in the override environment variable.
|
||||||
|
environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}'))
|
||||||
|
app.config.update(environ_config)
|
||||||
|
|
||||||
|
# Fix remote address handling for Flask.
|
||||||
|
if app.config.get('PROXY_COUNT', 1):
|
||||||
|
app.wsgi_app = ProxyFix(app.wsgi_app, num_proxies=app.config.get('PROXY_COUNT', 1))
|
||||||
|
|
||||||
|
# Ensure the V3 upgrade key is specified correctly. If not, simply fail.
|
||||||
|
# TODO: Remove for V3.1.
|
||||||
|
if not is_testing and not is_building and app.config.get('SETUP_COMPLETE', False):
|
||||||
|
v3_upgrade_mode = app.config.get('V3_UPGRADE_MODE')
|
||||||
|
if v3_upgrade_mode is None:
|
||||||
|
raise Exception('Configuration flag `V3_UPGRADE_MODE` must be set. Please check the upgrade docs')
|
||||||
|
|
||||||
|
if (v3_upgrade_mode != 'background'
|
||||||
|
and v3_upgrade_mode != 'complete'
|
||||||
|
and v3_upgrade_mode != 'production-transition'
|
||||||
|
and v3_upgrade_mode != 'post-oci-rollout'
|
||||||
|
and v3_upgrade_mode != 'post-oci-roll-back-compat'):
|
||||||
|
raise Exception('Invalid value for config `V3_UPGRADE_MODE`. Please check the upgrade docs')
|
||||||
|
|
||||||
|
# Split the registry model based on config.
|
||||||
|
# TODO: Remove once we are fully on the OCI data model.
|
||||||
|
registry_model.setup_split(app.config.get('OCI_NAMESPACE_PROPORTION') or 0,
|
||||||
|
app.config.get('OCI_NAMESPACE_WHITELIST') or set(),
|
||||||
|
app.config.get('V22_NAMESPACE_WHITELIST') or set(),
|
||||||
|
app.config.get('V3_UPGRADE_MODE'))
|
||||||
|
|
||||||
|
# Allow user to define a custom storage preference for the local instance.
|
||||||
|
_distributed_storage_preference = os.environ.get('QUAY_DISTRIBUTED_STORAGE_PREFERENCE', '').split()
|
||||||
|
if _distributed_storage_preference:
|
||||||
|
app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = _distributed_storage_preference
|
||||||
|
|
||||||
|
# Generate a secret key if none was specified.
|
||||||
|
if app.config['SECRET_KEY'] is None:
|
||||||
|
logger.debug('Generating in-memory secret key')
|
||||||
|
app.config['SECRET_KEY'] = generate_secret_key()
|
||||||
|
|
||||||
|
# If the "preferred" scheme is https, then http is not allowed. Therefore, ensure we have a secure
|
||||||
|
# session cookie.
|
||||||
|
if (app.config['PREFERRED_URL_SCHEME'] == 'https' and
|
||||||
|
not app.config.get('FORCE_NONSECURE_SESSION_COOKIE', False)):
|
||||||
|
app.config['SESSION_COOKIE_SECURE'] = True
|
||||||
|
|
||||||
|
# Load features from config.
|
||||||
|
features.import_features(app.config)
|
||||||
|
|
||||||
|
CONFIG_DIGEST = hashlib.sha256(json.dumps(app.config, default=str)).hexdigest()[0:8]
|
||||||
|
|
||||||
|
logger.debug("Loaded config", extra={"config": app.config})
|
||||||
|
|
||||||
|
|
||||||
|
class RequestWithId(Request):
|
||||||
|
request_gen = staticmethod(urn_generator(['request']))
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(RequestWithId, self).__init__(*args, **kwargs)
|
||||||
|
self.request_id = self.request_gen()
|
||||||
|
|
||||||
|
|
||||||
|
@app.before_request
|
||||||
|
def _request_start():
|
||||||
|
if os.getenv('PYDEV_DEBUG', None):
|
||||||
|
import pydevd
|
||||||
|
host, port = os.getenv('PYDEV_DEBUG').split(':')
|
||||||
|
pydevd.settrace(host, port=int(port), stdoutToServer=True, stderrToServer=True, suspend=False)
|
||||||
|
|
||||||
|
logger.debug('Starting request: %s (%s)', request.request_id, request.path,
|
||||||
|
extra={"request_id": request.request_id})
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_FILTER = lambda x: '[FILTERED]'
|
||||||
|
FILTERED_VALUES = [
|
||||||
|
{'key': ['password'], 'fn': DEFAULT_FILTER},
|
||||||
|
{'key': ['user', 'password'], 'fn': DEFAULT_FILTER},
|
||||||
|
{'key': ['blob'], 'fn': lambda x: x[0:8]}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@app.after_request
|
||||||
|
def _request_end(resp):
|
||||||
|
try:
|
||||||
|
jsonbody = request.get_json(force=True, silent=True)
|
||||||
|
except HTTPException:
|
||||||
|
jsonbody = None
|
||||||
|
|
||||||
|
values = request.values.to_dict()
|
||||||
|
|
||||||
|
if jsonbody and not isinstance(jsonbody, dict):
|
||||||
|
jsonbody = {'_parsererror': jsonbody}
|
||||||
|
|
||||||
|
if isinstance(values, dict):
|
||||||
|
filter_logs(values, FILTERED_VALUES)
|
||||||
|
|
||||||
|
extra = {
|
||||||
|
"endpoint": request.endpoint,
|
||||||
|
"request_id" : request.request_id,
|
||||||
|
"remote_addr": request.remote_addr,
|
||||||
|
"http_method": request.method,
|
||||||
|
"original_url": request.url,
|
||||||
|
"path": request.path,
|
||||||
|
"parameters": values,
|
||||||
|
"json_body": jsonbody,
|
||||||
|
"confsha": CONFIG_DIGEST,
|
||||||
|
}
|
||||||
|
|
||||||
|
if request.user_agent is not None:
|
||||||
|
extra["user-agent"] = request.user_agent.string
|
||||||
|
|
||||||
|
logger.debug('Ending request: %s (%s)', request.request_id, request.path, extra=extra)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
root_logger = logging.getLogger()
|
||||||
|
|
||||||
|
app.request_class = RequestWithId
|
||||||
|
|
||||||
|
# Register custom converters.
|
||||||
|
app.url_map.converters['regex'] = RegexConverter
|
||||||
|
app.url_map.converters['repopath'] = RepositoryPathConverter
|
||||||
|
app.url_map.converters['apirepopath'] = APIRepositoryPathConverter
|
||||||
|
|
||||||
|
Principal(app, use_sessions=False)
|
||||||
|
|
||||||
|
tf = app.config['DB_TRANSACTION_FACTORY']
|
||||||
|
|
||||||
|
model_cache = get_model_cache(app.config)
|
||||||
|
avatar = Avatar(app)
|
||||||
|
login_manager = LoginManager(app)
|
||||||
|
mail = Mail(app)
|
||||||
|
prometheus = PrometheusPlugin(app)
|
||||||
|
metric_queue = MetricQueue(prometheus)
|
||||||
|
chunk_cleanup_queue = WorkQueue(app.config['CHUNK_CLEANUP_QUEUE_NAME'], tf, metric_queue=metric_queue)
|
||||||
|
instance_keys = InstanceKeys(app)
|
||||||
|
ip_resolver = IPResolver(app)
|
||||||
|
storage = Storage(app, metric_queue, chunk_cleanup_queue, instance_keys, config_provider, ip_resolver)
|
||||||
|
userfiles = Userfiles(app, storage)
|
||||||
|
log_archive = LogArchive(app, storage)
|
||||||
|
analytics = Analytics(app)
|
||||||
|
user_analytics = UserAnalytics(app)
|
||||||
|
billing = Billing(app)
|
||||||
|
sentry = Sentry(app)
|
||||||
|
build_logs = BuildLogs(app)
|
||||||
|
authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECTORY)
|
||||||
|
userevents = UserEventsBuilderModule(app)
|
||||||
|
superusers = SuperUserManager(app)
|
||||||
|
signer = Signer(app, config_provider)
|
||||||
|
instance_keys = InstanceKeys(app)
|
||||||
|
label_validator = LabelValidator(app)
|
||||||
|
build_canceller = BuildCanceller(app)
|
||||||
|
|
||||||
|
start_cloudwatch_sender(metric_queue, app)
|
||||||
|
|
||||||
|
github_trigger = GithubOAuthService(app.config, 'GITHUB_TRIGGER_CONFIG')
|
||||||
|
gitlab_trigger = GitLabOAuthService(app.config, 'GITLAB_TRIGGER_CONFIG')
|
||||||
|
|
||||||
|
oauth_login = OAuthLoginManager(app.config)
|
||||||
|
oauth_apps = [github_trigger, gitlab_trigger]
|
||||||
|
|
||||||
|
image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf,
|
||||||
|
has_namespace=False, metric_queue=metric_queue)
|
||||||
|
dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
|
||||||
|
metric_queue=metric_queue,
|
||||||
|
reporter=BuildMetricQueueReporter(metric_queue),
|
||||||
|
has_namespace=True)
|
||||||
|
notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf, has_namespace=True,
|
||||||
|
metric_queue=metric_queue)
|
||||||
|
secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf,
|
||||||
|
has_namespace=False,
|
||||||
|
metric_queue=metric_queue)
|
||||||
|
export_action_logs_queue = WorkQueue(app.config['EXPORT_ACTION_LOGS_QUEUE_NAME'], tf,
|
||||||
|
has_namespace=True,
|
||||||
|
metric_queue=metric_queue)
|
||||||
|
|
||||||
|
# Note: We set `has_namespace` to `False` here, as we explicitly want this queue to not be emptied
|
||||||
|
# when a namespace is marked for deletion.
|
||||||
|
namespace_gc_queue = WorkQueue(app.config['NAMESPACE_GC_QUEUE_NAME'], tf, has_namespace=False,
|
||||||
|
metric_queue=metric_queue)
|
||||||
|
|
||||||
|
all_queues = [image_replication_queue, dockerfile_build_queue, notification_queue,
|
||||||
|
secscan_notification_queue, chunk_cleanup_queue, namespace_gc_queue]
|
||||||
|
|
||||||
|
url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
|
||||||
|
secscan_api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
|
||||||
|
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
|
||||||
|
instance_keys=instance_keys)
|
||||||
|
|
||||||
|
repo_mirror_api = RepoMirrorAPI(app.config, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
|
||||||
|
instance_keys=instance_keys)
|
||||||
|
|
||||||
|
tuf_metadata_api = TUFMetadataAPI(app, app.config)
|
||||||
|
|
||||||
|
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
|
||||||
|
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)
|
||||||
|
if os.path.exists(_v2_key_path):
|
||||||
|
docker_v2_signing_key = RSAKey().load(_v2_key_path)
|
||||||
|
else:
|
||||||
|
docker_v2_signing_key = RSAKey(key=RSA.generate(2048))
|
||||||
|
|
||||||
|
# Configure the database.
|
||||||
|
if app.config.get('DATABASE_SECRET_KEY') is None and app.config.get('SETUP_COMPLETE', False):
|
||||||
|
raise Exception('Missing DATABASE_SECRET_KEY in config; did you perhaps forget to add it?')
|
||||||
|
|
||||||
|
database.configure(app.config)
|
||||||
|
|
||||||
|
model.config.app_config = app.config
|
||||||
|
model.config.store = storage
|
||||||
|
model.config.register_image_cleanup_callback(secscan_api.cleanup_layers)
|
||||||
|
model.config.register_repo_cleanup_callback(tuf_metadata_api.delete_metadata)
|
||||||
|
|
||||||
|
|
||||||
|
@login_manager.user_loader
|
||||||
|
def load_user(user_uuid):
|
||||||
|
logger.debug('User loader loading deferred user with uuid: %s', user_uuid)
|
||||||
|
return LoginWrappedDBUser(user_uuid)
|
||||||
|
|
||||||
|
logs_model.configure(app.config)
|
||||||
|
|
||||||
|
get_app_url = partial(get_app_url, app.config)
|
||||||
22
application.py
Normal file
22
application.py
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
# NOTE: Must be before we import or call anything that may be synchronous.
|
||||||
|
from gevent import monkey
|
||||||
|
monkey.patch_all()
|
||||||
|
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
import logging.config
|
||||||
|
|
||||||
|
from util.log import logfile_path
|
||||||
|
from app import app as application
|
||||||
|
|
||||||
|
|
||||||
|
# Bind all of the blueprints
|
||||||
|
import web
|
||||||
|
import verbs
|
||||||
|
import registry
|
||||||
|
import secscan
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
logging.config.fileConfig(logfile_path(debug=True), disable_existing_loggers=False)
|
||||||
|
application.run(port=5000, debug=True, threaded=True, host='0.0.0.0')
|
||||||
0
auth/__init__.py
Normal file
0
auth/__init__.py
Normal file
21
auth/auth_context.py
Normal file
21
auth/auth_context.py
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
from flask import _request_ctx_stack
|
||||||
|
|
||||||
|
def get_authenticated_context():
|
||||||
|
""" Returns the auth context for the current request context, if any. """
|
||||||
|
return getattr(_request_ctx_stack.top, 'authenticated_context', None)
|
||||||
|
|
||||||
|
def get_authenticated_user():
|
||||||
|
""" Returns the authenticated user, if any, or None if none. """
|
||||||
|
context = get_authenticated_context()
|
||||||
|
return context.authed_user if context else None
|
||||||
|
|
||||||
|
def get_validated_oauth_token():
|
||||||
|
""" Returns the authenticated and validated OAuth access token, if any, or None if none. """
|
||||||
|
context = get_authenticated_context()
|
||||||
|
return context.authed_oauth_token if context else None
|
||||||
|
|
||||||
|
def set_authenticated_context(auth_context):
|
||||||
|
""" Sets the auth context for the current request context to that given. """
|
||||||
|
ctx = _request_ctx_stack.top
|
||||||
|
ctx.authenticated_context = auth_context
|
||||||
|
return auth_context
|
||||||
437
auth/auth_context_type.py
Normal file
437
auth/auth_context_type.py
Normal file
|
|
@ -0,0 +1,437 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from abc import ABCMeta, abstractmethod
|
||||||
|
from cachetools.func import lru_cache
|
||||||
|
from six import add_metaclass
|
||||||
|
|
||||||
|
from app import app
|
||||||
|
from data import model
|
||||||
|
|
||||||
|
from flask_principal import Identity, identity_changed
|
||||||
|
|
||||||
|
from auth.auth_context import set_authenticated_context
|
||||||
|
from auth.context_entity import ContextEntityKind, CONTEXT_ENTITY_HANDLERS
|
||||||
|
from auth.permissions import QuayDeferredPermissionUser
|
||||||
|
from auth.scopes import scopes_from_scope_string
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@add_metaclass(ABCMeta)
|
||||||
|
class AuthContext(object):
|
||||||
|
"""
|
||||||
|
Interface that represents the current context of authentication.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def entity_kind(self):
|
||||||
|
""" Returns the kind of the entity in this auth context. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def is_anonymous(self):
|
||||||
|
""" Returns true if this is an anonymous context. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def authed_oauth_token(self):
|
||||||
|
""" Returns the authenticated OAuth token, if any. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def authed_user(self):
|
||||||
|
""" Returns the authenticated user, whether directly, or via an OAuth or access token. Note that
|
||||||
|
this property will also return robot accounts.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def has_nonrobot_user(self):
|
||||||
|
""" Returns whether a user (not a robot) was authenticated successfully. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def identity(self):
|
||||||
|
""" Returns the identity for the auth context. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def description(self):
|
||||||
|
""" Returns a human-readable and *public* description of the current auth context. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def credential_username(self):
|
||||||
|
""" Returns the username to create credentials for this context's entity, if any. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def analytics_id_and_public_metadata(self):
|
||||||
|
""" Returns the analytics ID and public log metadata for this auth context. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def apply_to_request_context(self):
|
||||||
|
""" Applies this auth result to the auth context and Flask-Principal. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def to_signed_dict(self):
|
||||||
|
""" Serializes the auth context into a dictionary suitable for inclusion in a JWT or other
|
||||||
|
form of signed serialization.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def unique_key(self):
|
||||||
|
""" Returns a key that is unique to this auth context type and its data. For example, an
|
||||||
|
instance of the auth context type for the user might be a string of the form
|
||||||
|
`user-{user-uuid}`. Callers should treat this key as opaque and not rely on the contents
|
||||||
|
for anything besides uniqueness. This is typically used by callers when they'd like to
|
||||||
|
check cache but not hit the database to get a fully validated auth context.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ValidatedAuthContext(AuthContext):
|
||||||
|
""" ValidatedAuthContext represents the loaded, authenticated and validated auth information
|
||||||
|
for the current request context.
|
||||||
|
"""
|
||||||
|
def __init__(self, user=None, token=None, oauthtoken=None, robot=None, appspecifictoken=None,
|
||||||
|
signed_data=None):
|
||||||
|
# Note: These field names *MUST* match the string values of the kinds defined in
|
||||||
|
# ContextEntityKind.
|
||||||
|
self.user = user
|
||||||
|
self.robot = robot
|
||||||
|
self.token = token
|
||||||
|
self.oauthtoken = oauthtoken
|
||||||
|
self.appspecifictoken = appspecifictoken
|
||||||
|
self.signed_data = signed_data
|
||||||
|
|
||||||
|
def tuple(self):
|
||||||
|
return vars(self).values()
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return self.tuple() == other.tuple()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def entity_kind(self):
|
||||||
|
""" Returns the kind of the entity in this auth context. """
|
||||||
|
for kind in ContextEntityKind:
|
||||||
|
if hasattr(self, kind.value) and getattr(self, kind.value):
|
||||||
|
return kind
|
||||||
|
|
||||||
|
return ContextEntityKind.anonymous
|
||||||
|
|
||||||
|
@property
|
||||||
|
def authed_user(self):
|
||||||
|
""" Returns the authenticated user, whether directly, or via an OAuth token. Note that this
|
||||||
|
will also return robot accounts.
|
||||||
|
"""
|
||||||
|
authed_user = self._authed_user()
|
||||||
|
if authed_user is not None and not authed_user.enabled:
|
||||||
|
logger.warning('Attempt to reference a disabled user/robot: %s', authed_user.username)
|
||||||
|
return None
|
||||||
|
|
||||||
|
return authed_user
|
||||||
|
|
||||||
|
@property
|
||||||
|
def authed_oauth_token(self):
|
||||||
|
return self.oauthtoken
|
||||||
|
|
||||||
|
def _authed_user(self):
|
||||||
|
if self.oauthtoken:
|
||||||
|
return self.oauthtoken.authorized_user
|
||||||
|
|
||||||
|
if self.appspecifictoken:
|
||||||
|
return self.appspecifictoken.user
|
||||||
|
|
||||||
|
if self.signed_data:
|
||||||
|
return model.user.get_user(self.signed_data['user_context'])
|
||||||
|
|
||||||
|
return self.user if self.user else self.robot
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_anonymous(self):
|
||||||
|
""" Returns true if this is an anonymous context. """
|
||||||
|
return not self.authed_user and not self.token and not self.signed_data
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_nonrobot_user(self):
|
||||||
|
""" Returns whether a user (not a robot) was authenticated successfully. """
|
||||||
|
return bool(self.authed_user and not self.robot)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def identity(self):
|
||||||
|
""" Returns the identity for the auth context. """
|
||||||
|
if self.oauthtoken:
|
||||||
|
scope_set = scopes_from_scope_string(self.oauthtoken.scope)
|
||||||
|
return QuayDeferredPermissionUser.for_user(self.oauthtoken.authorized_user, scope_set)
|
||||||
|
|
||||||
|
if self.authed_user:
|
||||||
|
return QuayDeferredPermissionUser.for_user(self.authed_user)
|
||||||
|
|
||||||
|
if self.token:
|
||||||
|
return Identity(self.token.get_code(), 'token')
|
||||||
|
|
||||||
|
if self.signed_data:
|
||||||
|
identity = Identity(None, 'signed_grant')
|
||||||
|
identity.provides.update(self.signed_data['grants'])
|
||||||
|
return identity
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def entity_reference(self):
|
||||||
|
""" Returns the DB object reference for this context's entity. """
|
||||||
|
if self.entity_kind == ContextEntityKind.anonymous:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return getattr(self, self.entity_kind.value)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def description(self):
|
||||||
|
""" Returns a human-readable and *public* description of the current auth context. """
|
||||||
|
handler = CONTEXT_ENTITY_HANDLERS[self.entity_kind]()
|
||||||
|
return handler.description(self.entity_reference)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def credential_username(self):
|
||||||
|
""" Returns the username to create credentials for this context's entity, if any. """
|
||||||
|
handler = CONTEXT_ENTITY_HANDLERS[self.entity_kind]()
|
||||||
|
return handler.credential_username(self.entity_reference)
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self):
|
||||||
|
""" Returns the analytics ID and public log metadata for this auth context. """
|
||||||
|
handler = CONTEXT_ENTITY_HANDLERS[self.entity_kind]()
|
||||||
|
return handler.analytics_id_and_public_metadata(self.entity_reference)
|
||||||
|
|
||||||
|
def apply_to_request_context(self):
|
||||||
|
""" Applies this auth result to the auth context and Flask-Principal. """
|
||||||
|
# Save to the request context.
|
||||||
|
set_authenticated_context(self)
|
||||||
|
|
||||||
|
# Set the identity for Flask-Principal.
|
||||||
|
if self.identity:
|
||||||
|
identity_changed.send(app, identity=self.identity)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def unique_key(self):
|
||||||
|
signed_dict = self.to_signed_dict()
|
||||||
|
return '%s-%s' % (signed_dict['entity_kind'], signed_dict.get('entity_reference', '(anon)'))
|
||||||
|
|
||||||
|
def to_signed_dict(self):
|
||||||
|
""" Serializes the auth context into a dictionary suitable for inclusion in a JWT or other
|
||||||
|
form of signed serialization.
|
||||||
|
"""
|
||||||
|
dict_data = {
|
||||||
|
'version': 2,
|
||||||
|
'entity_kind': self.entity_kind.value,
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.entity_kind != ContextEntityKind.anonymous:
|
||||||
|
handler = CONTEXT_ENTITY_HANDLERS[self.entity_kind]()
|
||||||
|
dict_data.update({
|
||||||
|
'entity_reference': handler.get_serialized_entity_reference(self.entity_reference),
|
||||||
|
})
|
||||||
|
|
||||||
|
# Add legacy information.
|
||||||
|
# TODO: Remove this all once the new code is fully deployed.
|
||||||
|
if self.token:
|
||||||
|
dict_data.update({
|
||||||
|
'kind': 'token',
|
||||||
|
'token': self.token.code,
|
||||||
|
})
|
||||||
|
|
||||||
|
if self.oauthtoken:
|
||||||
|
dict_data.update({
|
||||||
|
'kind': 'oauth',
|
||||||
|
'oauth': self.oauthtoken.uuid,
|
||||||
|
'user': self.authed_user.username,
|
||||||
|
})
|
||||||
|
|
||||||
|
if self.user or self.robot:
|
||||||
|
dict_data.update({
|
||||||
|
'kind': 'user',
|
||||||
|
'user': self.authed_user.username,
|
||||||
|
})
|
||||||
|
|
||||||
|
if self.appspecifictoken:
|
||||||
|
dict_data.update({
|
||||||
|
'kind': 'user',
|
||||||
|
'user': self.authed_user.username,
|
||||||
|
})
|
||||||
|
|
||||||
|
if self.is_anonymous:
|
||||||
|
dict_data.update({
|
||||||
|
'kind': 'anonymous',
|
||||||
|
})
|
||||||
|
|
||||||
|
# End of legacy information.
|
||||||
|
return dict_data
|
||||||
|
|
||||||
|
class SignedAuthContext(AuthContext):
|
||||||
|
""" SignedAuthContext represents an auth context loaded from a signed token of some kind,
|
||||||
|
such as a JWT. Unlike ValidatedAuthContext, SignedAuthContext operates lazily, only loading
|
||||||
|
the actual {user, robot, token, etc} when requested. This allows registry operations that
|
||||||
|
only need to check if *some* entity is present to do so, without hitting the database.
|
||||||
|
"""
|
||||||
|
def __init__(self, kind, signed_data, v1_dict_format):
|
||||||
|
self.kind = kind
|
||||||
|
self.signed_data = signed_data
|
||||||
|
self.v1_dict_format = v1_dict_format
|
||||||
|
|
||||||
|
@property
|
||||||
|
def unique_key(self):
|
||||||
|
if self.v1_dict_format:
|
||||||
|
# Since V1 data format is verbose, just use the validated version to get the key.
|
||||||
|
return self._get_validated().unique_key
|
||||||
|
|
||||||
|
signed_dict = self.signed_data
|
||||||
|
return '%s-%s' % (signed_dict['entity_kind'], signed_dict.get('entity_reference', '(anon)'))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def build_from_signed_dict(cls, dict_data, v1_dict_format=False):
|
||||||
|
if not v1_dict_format:
|
||||||
|
entity_kind = ContextEntityKind(dict_data.get('entity_kind', 'anonymous'))
|
||||||
|
return SignedAuthContext(entity_kind, dict_data, v1_dict_format)
|
||||||
|
|
||||||
|
# Legacy handling.
|
||||||
|
# TODO: Remove this all once the new code is fully deployed.
|
||||||
|
kind_string = dict_data.get('kind', 'anonymous')
|
||||||
|
if kind_string == 'oauth':
|
||||||
|
kind_string = 'oauthtoken'
|
||||||
|
|
||||||
|
kind = ContextEntityKind(kind_string)
|
||||||
|
return SignedAuthContext(kind, dict_data, v1_dict_format)
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _get_validated(self):
|
||||||
|
""" Returns a ValidatedAuthContext for this signed context, resolving all the necessary
|
||||||
|
references.
|
||||||
|
"""
|
||||||
|
if not self.v1_dict_format:
|
||||||
|
if self.kind == ContextEntityKind.anonymous:
|
||||||
|
return ValidatedAuthContext()
|
||||||
|
|
||||||
|
serialized_entity_reference = self.signed_data['entity_reference']
|
||||||
|
handler = CONTEXT_ENTITY_HANDLERS[self.kind]()
|
||||||
|
entity_reference = handler.deserialize_entity_reference(serialized_entity_reference)
|
||||||
|
if entity_reference is None:
|
||||||
|
logger.debug('Could not deserialize entity reference `%s` under kind `%s`',
|
||||||
|
serialized_entity_reference, self.kind)
|
||||||
|
return ValidatedAuthContext()
|
||||||
|
|
||||||
|
return ValidatedAuthContext(**{self.kind.value: entity_reference})
|
||||||
|
|
||||||
|
# Legacy handling.
|
||||||
|
# TODO: Remove this all once the new code is fully deployed.
|
||||||
|
kind_string = self.signed_data.get('kind', 'anonymous')
|
||||||
|
if kind_string == 'oauth':
|
||||||
|
kind_string = 'oauthtoken'
|
||||||
|
|
||||||
|
kind = ContextEntityKind(kind_string)
|
||||||
|
if kind == ContextEntityKind.anonymous:
|
||||||
|
return ValidatedAuthContext()
|
||||||
|
|
||||||
|
if kind == ContextEntityKind.user or kind == ContextEntityKind.robot:
|
||||||
|
user = model.user.get_user(self.signed_data.get('user', ''))
|
||||||
|
if not user:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return ValidatedAuthContext(robot=user) if user.robot else ValidatedAuthContext(user=user)
|
||||||
|
|
||||||
|
if kind == ContextEntityKind.token:
|
||||||
|
token = model.token.load_token_data(self.signed_data.get('token'))
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return ValidatedAuthContext(token=token)
|
||||||
|
|
||||||
|
if kind == ContextEntityKind.oauthtoken:
|
||||||
|
user = model.user.get_user(self.signed_data.get('user', ''))
|
||||||
|
if not user:
|
||||||
|
return None
|
||||||
|
|
||||||
|
token_uuid = self.signed_data.get('oauth', '')
|
||||||
|
oauthtoken = model.oauth.lookup_access_token_for_user(user, token_uuid)
|
||||||
|
if not oauthtoken:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return ValidatedAuthContext(oauthtoken=oauthtoken)
|
||||||
|
|
||||||
|
raise Exception('Unknown auth context kind `%s` when deserializing %s' % (kind,
|
||||||
|
self.signed_data))
|
||||||
|
# End of legacy handling.
|
||||||
|
|
||||||
|
@property
|
||||||
|
def entity_kind(self):
|
||||||
|
""" Returns the kind of the entity in this auth context. """
|
||||||
|
return self.kind
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_anonymous(self):
|
||||||
|
""" Returns true if this is an anonymous context. """
|
||||||
|
return self.kind == ContextEntityKind.anonymous
|
||||||
|
|
||||||
|
@property
|
||||||
|
def authed_user(self):
|
||||||
|
""" Returns the authenticated user, whether directly, or via an OAuth or access token. Note that
|
||||||
|
this property will also return robot accounts.
|
||||||
|
"""
|
||||||
|
if self.kind == ContextEntityKind.anonymous:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return self._get_validated().authed_user
|
||||||
|
|
||||||
|
@property
|
||||||
|
def authed_oauth_token(self):
|
||||||
|
if self.kind == ContextEntityKind.anonymous:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return self._get_validated().authed_oauth_token
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_nonrobot_user(self):
|
||||||
|
""" Returns whether a user (not a robot) was authenticated successfully. """
|
||||||
|
if self.kind == ContextEntityKind.anonymous:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return self._get_validated().has_nonrobot_user
|
||||||
|
|
||||||
|
@property
|
||||||
|
def identity(self):
|
||||||
|
""" Returns the identity for the auth context. """
|
||||||
|
return self._get_validated().identity
|
||||||
|
|
||||||
|
@property
|
||||||
|
def description(self):
|
||||||
|
""" Returns a human-readable and *public* description of the current auth context. """
|
||||||
|
return self._get_validated().description
|
||||||
|
|
||||||
|
@property
|
||||||
|
def credential_username(self):
|
||||||
|
""" Returns the username to create credentials for this context's entity, if any. """
|
||||||
|
return self._get_validated().credential_username
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self):
|
||||||
|
""" Returns the analytics ID and public log metadata for this auth context. """
|
||||||
|
return self._get_validated().analytics_id_and_public_metadata()
|
||||||
|
|
||||||
|
def apply_to_request_context(self):
|
||||||
|
""" Applies this auth result to the auth context and Flask-Principal. """
|
||||||
|
return self._get_validated().apply_to_request_context()
|
||||||
|
|
||||||
|
def to_signed_dict(self):
|
||||||
|
""" Serializes the auth context into a dictionary suitable for inclusion in a JWT or other
|
||||||
|
form of signed serialization.
|
||||||
|
"""
|
||||||
|
return self.signed_data
|
||||||
58
auth/basic.py
Normal file
58
auth/basic.py
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from base64 import b64decode
|
||||||
|
from flask import request
|
||||||
|
|
||||||
|
from auth.credentials import validate_credentials
|
||||||
|
from auth.validateresult import ValidateResult, AuthKind
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def has_basic_auth(username):
|
||||||
|
""" Returns true if a basic auth header exists with a username and password pair that validates
|
||||||
|
against the internal authentication system. Returns True on full success and False on any
|
||||||
|
failure (missing header, invalid header, invalid credentials, etc).
|
||||||
|
"""
|
||||||
|
auth_header = request.headers.get('authorization', '')
|
||||||
|
result = validate_basic_auth(auth_header)
|
||||||
|
return result.has_nonrobot_user and result.context.user.username == username
|
||||||
|
|
||||||
|
|
||||||
|
def validate_basic_auth(auth_header):
|
||||||
|
""" Validates the specified basic auth header, returning whether its credentials point
|
||||||
|
to a valid user or token.
|
||||||
|
"""
|
||||||
|
if not auth_header:
|
||||||
|
return ValidateResult(AuthKind.basic, missing=True)
|
||||||
|
|
||||||
|
logger.debug('Attempt to process basic auth header')
|
||||||
|
|
||||||
|
# Parse the basic auth header.
|
||||||
|
assert isinstance(auth_header, basestring)
|
||||||
|
credentials, err = _parse_basic_auth_header(auth_header)
|
||||||
|
if err is not None:
|
||||||
|
logger.debug('Got invalid basic auth header: %s', auth_header)
|
||||||
|
return ValidateResult(AuthKind.basic, missing=True)
|
||||||
|
|
||||||
|
auth_username, auth_password_or_token = credentials
|
||||||
|
result, _ = validate_credentials(auth_username, auth_password_or_token)
|
||||||
|
return result.with_kind(AuthKind.basic)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_basic_auth_header(auth):
|
||||||
|
""" Parses the given basic auth header, returning the credentials found inside.
|
||||||
|
"""
|
||||||
|
normalized = [part.strip() for part in auth.split(' ') if part]
|
||||||
|
if normalized[0].lower() != 'basic' or len(normalized) != 2:
|
||||||
|
return None, 'Invalid basic auth header'
|
||||||
|
|
||||||
|
try:
|
||||||
|
credentials = [part.decode('utf-8') for part in b64decode(normalized[1]).split(':', 1)]
|
||||||
|
except (TypeError, UnicodeDecodeError, ValueError):
|
||||||
|
logger.exception('Exception when parsing basic auth header: %s', auth)
|
||||||
|
return None, 'Could not parse basic auth header'
|
||||||
|
|
||||||
|
if len(credentials) != 2:
|
||||||
|
return None, 'Unexpected number of credentials found in basic auth header'
|
||||||
|
|
||||||
|
return credentials, None
|
||||||
203
auth/context_entity.py
Normal file
203
auth/context_entity.py
Normal file
|
|
@ -0,0 +1,203 @@
|
||||||
|
from abc import ABCMeta, abstractmethod
|
||||||
|
from six import add_metaclass
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
from data import model
|
||||||
|
|
||||||
|
from auth.credential_consts import (ACCESS_TOKEN_USERNAME, OAUTH_TOKEN_USERNAME,
|
||||||
|
APP_SPECIFIC_TOKEN_USERNAME)
|
||||||
|
|
||||||
|
class ContextEntityKind(Enum):
|
||||||
|
""" Defines the various kinds of entities in an auth context. Note that the string values of
|
||||||
|
these fields *must* match the names of the fields in the ValidatedAuthContext class, as
|
||||||
|
we fill them in directly based on the string names here.
|
||||||
|
"""
|
||||||
|
anonymous = 'anonymous'
|
||||||
|
user = 'user'
|
||||||
|
robot = 'robot'
|
||||||
|
token = 'token'
|
||||||
|
oauthtoken = 'oauthtoken'
|
||||||
|
appspecifictoken = 'appspecifictoken'
|
||||||
|
signed_data = 'signed_data'
|
||||||
|
|
||||||
|
|
||||||
|
@add_metaclass(ABCMeta)
|
||||||
|
class ContextEntityHandler(object):
|
||||||
|
"""
|
||||||
|
Interface that represents handling specific kinds of entities under an auth context.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
""" Returns the username to create credentials for this entity, if any. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
""" Returns the entity reference for this kind of auth context, serialized into a form that can
|
||||||
|
be placed into a JSON object and put into a JWT. This is typically a DB UUID or another
|
||||||
|
unique identifier for the object in the DB.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
""" Returns the deserialized reference to the entity in the database, or None if none. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def description(self, entity_reference):
|
||||||
|
""" Returns a human-readable and *public* description of the current entity. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
""" Returns the analyitics ID and a dict of public metadata for the current entity. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class AnonymousEntityHandler(ContextEntityHandler):
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def description(self, entity_reference):
|
||||||
|
return "anonymous"
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
return "anonymous", {}
|
||||||
|
|
||||||
|
|
||||||
|
class UserEntityHandler(ContextEntityHandler):
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
return entity_reference.username
|
||||||
|
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
return entity_reference.uuid
|
||||||
|
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
return model.user.get_user_by_uuid(serialized_entity_reference)
|
||||||
|
|
||||||
|
def description(self, entity_reference):
|
||||||
|
return "user %s" % entity_reference.username
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
return entity_reference.username, {
|
||||||
|
'username': entity_reference.username,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class RobotEntityHandler(ContextEntityHandler):
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
return entity_reference.username
|
||||||
|
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
return entity_reference.username
|
||||||
|
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
return model.user.lookup_robot(serialized_entity_reference)
|
||||||
|
|
||||||
|
def description(self, entity_reference):
|
||||||
|
return "robot %s" % entity_reference.username
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
return entity_reference.username, {
|
||||||
|
'username': entity_reference.username,
|
||||||
|
'is_robot': True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TokenEntityHandler(ContextEntityHandler):
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
return ACCESS_TOKEN_USERNAME
|
||||||
|
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
return entity_reference.get_code()
|
||||||
|
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
return model.token.load_token_data(serialized_entity_reference)
|
||||||
|
|
||||||
|
def description(self, entity_reference):
|
||||||
|
return "token %s" % entity_reference.friendly_name
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
return 'token:%s' % entity_reference.id, {
|
||||||
|
'token': entity_reference.friendly_name,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class OAuthTokenEntityHandler(ContextEntityHandler):
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
return OAUTH_TOKEN_USERNAME
|
||||||
|
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
return entity_reference.uuid
|
||||||
|
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
return model.oauth.lookup_access_token_by_uuid(serialized_entity_reference)
|
||||||
|
|
||||||
|
def description(self, entity_reference):
|
||||||
|
return "oauthtoken for user %s" % entity_reference.authorized_user.username
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
return 'oauthtoken:%s' % entity_reference.id, {
|
||||||
|
'oauth_token_id': entity_reference.id,
|
||||||
|
'oauth_token_application_id': entity_reference.application.client_id,
|
||||||
|
'oauth_token_application': entity_reference.application.name,
|
||||||
|
'username': entity_reference.authorized_user.username,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class AppSpecificTokenEntityHandler(ContextEntityHandler):
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
return APP_SPECIFIC_TOKEN_USERNAME
|
||||||
|
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
return entity_reference.uuid
|
||||||
|
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
return model.appspecifictoken.get_token_by_uuid(serialized_entity_reference)
|
||||||
|
|
||||||
|
def description(self, entity_reference):
|
||||||
|
tpl = (entity_reference.title, entity_reference.user.username)
|
||||||
|
return "app specific token %s for user %s" % tpl
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
return 'appspecifictoken:%s' % entity_reference.id, {
|
||||||
|
'app_specific_token': entity_reference.uuid,
|
||||||
|
'app_specific_token_title': entity_reference.title,
|
||||||
|
'username': entity_reference.user.username,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class SignedDataEntityHandler(ContextEntityHandler):
|
||||||
|
def credential_username(self, entity_reference):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_serialized_entity_reference(self, entity_reference):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def deserialize_entity_reference(self, serialized_entity_reference):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def description(self, entity_reference):
|
||||||
|
return "signed"
|
||||||
|
|
||||||
|
def analytics_id_and_public_metadata(self, entity_reference):
|
||||||
|
return 'signed', {'signed': entity_reference}
|
||||||
|
|
||||||
|
|
||||||
|
CONTEXT_ENTITY_HANDLERS = {
|
||||||
|
ContextEntityKind.anonymous: AnonymousEntityHandler,
|
||||||
|
ContextEntityKind.user: UserEntityHandler,
|
||||||
|
ContextEntityKind.robot: RobotEntityHandler,
|
||||||
|
ContextEntityKind.token: TokenEntityHandler,
|
||||||
|
ContextEntityKind.oauthtoken: OAuthTokenEntityHandler,
|
||||||
|
ContextEntityKind.appspecifictoken: AppSpecificTokenEntityHandler,
|
||||||
|
ContextEntityKind.signed_data: SignedDataEntityHandler,
|
||||||
|
}
|
||||||
37
auth/cookie.py
Normal file
37
auth/cookie.py
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
from flask_login import current_user
|
||||||
|
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def validate_session_cookie(auth_header_unusued=None):
|
||||||
|
""" Attempts to load a user from a session cookie. """
|
||||||
|
if current_user.is_anonymous:
|
||||||
|
return ValidateResult(AuthKind.cookie, missing=True)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Attempt to parse the user uuid to make sure the cookie has the right value type
|
||||||
|
UUID(current_user.get_id())
|
||||||
|
except ValueError:
|
||||||
|
logger.debug('Got non-UUID for session cookie user: %s', current_user.get_id())
|
||||||
|
return ValidateResult(AuthKind.cookie, error_message='Invalid session cookie format')
|
||||||
|
|
||||||
|
logger.debug('Loading user from cookie: %s', current_user.get_id())
|
||||||
|
db_user = current_user.db_user()
|
||||||
|
if db_user is None:
|
||||||
|
return ValidateResult(AuthKind.cookie, error_message='Could not find matching user')
|
||||||
|
|
||||||
|
# Don't allow disabled users to login.
|
||||||
|
if not db_user.enabled:
|
||||||
|
logger.debug('User %s in session cookie is disabled', db_user.username)
|
||||||
|
return ValidateResult(AuthKind.cookie, error_message='User account is disabled')
|
||||||
|
|
||||||
|
# Don't allow organizations to "login".
|
||||||
|
if db_user.organization:
|
||||||
|
logger.debug('User %s in session cookie is in-fact organization', db_user.username)
|
||||||
|
return ValidateResult(AuthKind.cookie, error_message='Cannot login to organization')
|
||||||
|
|
||||||
|
return ValidateResult(AuthKind.cookie, user=db_user)
|
||||||
3
auth/credential_consts.py
Normal file
3
auth/credential_consts.py
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
ACCESS_TOKEN_USERNAME = '$token'
|
||||||
|
OAUTH_TOKEN_USERNAME = '$oauthtoken'
|
||||||
|
APP_SPECIFIC_TOKEN_USERNAME = '$app'
|
||||||
85
auth/credentials.py
Normal file
85
auth/credentials.py
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
import features
|
||||||
|
|
||||||
|
from app import authentication
|
||||||
|
from auth.oauth import validate_oauth_token
|
||||||
|
from auth.validateresult import ValidateResult, AuthKind
|
||||||
|
from auth.credential_consts import (ACCESS_TOKEN_USERNAME, OAUTH_TOKEN_USERNAME,
|
||||||
|
APP_SPECIFIC_TOKEN_USERNAME)
|
||||||
|
from data import model
|
||||||
|
from util.names import parse_robot_username
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class CredentialKind(Enum):
|
||||||
|
user = 'user'
|
||||||
|
robot = 'robot'
|
||||||
|
token = ACCESS_TOKEN_USERNAME
|
||||||
|
oauth_token = OAUTH_TOKEN_USERNAME
|
||||||
|
app_specific_token = APP_SPECIFIC_TOKEN_USERNAME
|
||||||
|
|
||||||
|
|
||||||
|
def validate_credentials(auth_username, auth_password_or_token):
|
||||||
|
""" Validates a pair of auth username and password/token credentials. """
|
||||||
|
# Check for access tokens.
|
||||||
|
if auth_username == ACCESS_TOKEN_USERNAME:
|
||||||
|
logger.debug('Found credentials for access token')
|
||||||
|
try:
|
||||||
|
token = model.token.load_token_data(auth_password_or_token)
|
||||||
|
logger.debug('Successfully validated credentials for access token %s', token.id)
|
||||||
|
return ValidateResult(AuthKind.credentials, token=token), CredentialKind.token
|
||||||
|
except model.DataModelException:
|
||||||
|
logger.warning('Failed to validate credentials for access token %s', auth_password_or_token)
|
||||||
|
return (ValidateResult(AuthKind.credentials, error_message='Invalid access token'),
|
||||||
|
CredentialKind.token)
|
||||||
|
|
||||||
|
# Check for App Specific tokens.
|
||||||
|
if features.APP_SPECIFIC_TOKENS and auth_username == APP_SPECIFIC_TOKEN_USERNAME:
|
||||||
|
logger.debug('Found credentials for app specific auth token')
|
||||||
|
token = model.appspecifictoken.access_valid_token(auth_password_or_token)
|
||||||
|
if token is None:
|
||||||
|
logger.debug('Failed to validate credentials for app specific token: %s',
|
||||||
|
auth_password_or_token)
|
||||||
|
return (ValidateResult(AuthKind.credentials, error_message='Invalid token'),
|
||||||
|
CredentialKind.app_specific_token)
|
||||||
|
|
||||||
|
if not token.user.enabled:
|
||||||
|
logger.debug('Tried to use an app specific token for a disabled user: %s',
|
||||||
|
token.uuid)
|
||||||
|
return (ValidateResult(AuthKind.credentials,
|
||||||
|
error_message='This user has been disabled. Please contact your administrator.'),
|
||||||
|
CredentialKind.app_specific_token)
|
||||||
|
|
||||||
|
logger.debug('Successfully validated credentials for app specific token %s', token.id)
|
||||||
|
return (ValidateResult(AuthKind.credentials, appspecifictoken=token),
|
||||||
|
CredentialKind.app_specific_token)
|
||||||
|
|
||||||
|
# Check for OAuth tokens.
|
||||||
|
if auth_username == OAUTH_TOKEN_USERNAME:
|
||||||
|
return validate_oauth_token(auth_password_or_token), CredentialKind.oauth_token
|
||||||
|
|
||||||
|
# Check for robots and users.
|
||||||
|
is_robot = parse_robot_username(auth_username)
|
||||||
|
if is_robot:
|
||||||
|
logger.debug('Found credentials header for robot %s', auth_username)
|
||||||
|
try:
|
||||||
|
robot = model.user.verify_robot(auth_username, auth_password_or_token)
|
||||||
|
logger.debug('Successfully validated credentials for robot %s', auth_username)
|
||||||
|
return ValidateResult(AuthKind.credentials, robot=robot), CredentialKind.robot
|
||||||
|
except model.InvalidRobotException as ire:
|
||||||
|
logger.warning('Failed to validate credentials for robot %s: %s', auth_username, ire)
|
||||||
|
return ValidateResult(AuthKind.credentials, error_message=str(ire)), CredentialKind.robot
|
||||||
|
|
||||||
|
# Otherwise, treat as a standard user.
|
||||||
|
(authenticated, err) = authentication.verify_and_link_user(auth_username, auth_password_or_token,
|
||||||
|
basic_auth=True)
|
||||||
|
if authenticated:
|
||||||
|
logger.debug('Successfully validated credentials for user %s', authenticated.username)
|
||||||
|
return ValidateResult(AuthKind.credentials, user=authenticated), CredentialKind.user
|
||||||
|
else:
|
||||||
|
logger.warning('Failed to validate credentials for user %s: %s', auth_username, err)
|
||||||
|
return ValidateResult(AuthKind.credentials, error_message=err), CredentialKind.user
|
||||||
96
auth/decorators.py
Normal file
96
auth/decorators.py
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from functools import wraps
|
||||||
|
from flask import request, session
|
||||||
|
|
||||||
|
from app import metric_queue
|
||||||
|
from auth.basic import validate_basic_auth
|
||||||
|
from auth.oauth import validate_bearer_auth
|
||||||
|
from auth.cookie import validate_session_cookie
|
||||||
|
from auth.signedgrant import validate_signed_grant
|
||||||
|
|
||||||
|
from util.http import abort
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def _auth_decorator(pass_result=False, handlers=None):
|
||||||
|
""" Builds an auth decorator that runs the given handlers and, if any return successfully,
|
||||||
|
sets up the auth context. The wrapped function will be invoked *regardless of success or
|
||||||
|
failure of the auth handler(s)*
|
||||||
|
"""
|
||||||
|
def processor(func):
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
auth_header = request.headers.get('authorization', '')
|
||||||
|
result = None
|
||||||
|
|
||||||
|
for handler in handlers:
|
||||||
|
result = handler(auth_header)
|
||||||
|
# If the handler was missing the necessary information, skip it and try the next one.
|
||||||
|
if result.missing:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check for a valid result.
|
||||||
|
if result.auth_valid:
|
||||||
|
logger.debug('Found valid auth result: %s', result.tuple())
|
||||||
|
|
||||||
|
# Set the various pieces of the auth context.
|
||||||
|
result.apply_to_context()
|
||||||
|
|
||||||
|
# Log the metric.
|
||||||
|
metric_queue.authentication_count.Inc(labelvalues=[result.kind, True])
|
||||||
|
break
|
||||||
|
|
||||||
|
# Otherwise, report the error.
|
||||||
|
if result.error_message is not None:
|
||||||
|
# Log the failure.
|
||||||
|
metric_queue.authentication_count.Inc(labelvalues=[result.kind, False])
|
||||||
|
break
|
||||||
|
|
||||||
|
if pass_result:
|
||||||
|
kwargs['auth_result'] = result
|
||||||
|
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
return wrapper
|
||||||
|
return processor
|
||||||
|
|
||||||
|
|
||||||
|
process_oauth = _auth_decorator(handlers=[validate_bearer_auth, validate_session_cookie])
|
||||||
|
process_auth = _auth_decorator(handlers=[validate_signed_grant, validate_basic_auth])
|
||||||
|
process_auth_or_cookie = _auth_decorator(handlers=[validate_basic_auth, validate_session_cookie])
|
||||||
|
process_basic_auth = _auth_decorator(handlers=[validate_basic_auth], pass_result=True)
|
||||||
|
process_basic_auth_no_pass = _auth_decorator(handlers=[validate_basic_auth])
|
||||||
|
|
||||||
|
|
||||||
|
def require_session_login(func):
|
||||||
|
""" Decorates a function and ensures that a valid session cookie exists or a 401 is raised. If
|
||||||
|
a valid session cookie does exist, the authenticated user and identity are also set.
|
||||||
|
"""
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
result = validate_session_cookie()
|
||||||
|
if result.has_nonrobot_user:
|
||||||
|
result.apply_to_context()
|
||||||
|
metric_queue.authentication_count.Inc(labelvalues=[result.kind, True])
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
elif not result.missing:
|
||||||
|
metric_queue.authentication_count.Inc(labelvalues=[result.kind, False])
|
||||||
|
|
||||||
|
abort(401, message='Method requires login and no valid login could be loaded.')
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def extract_namespace_repo_from_session(func):
|
||||||
|
""" Extracts the namespace and repository name from the current session (which must exist)
|
||||||
|
and passes them into the decorated function as the first and second arguments. If the
|
||||||
|
session doesn't exist or does not contain these arugments, a 400 error is raised.
|
||||||
|
"""
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
if 'namespace' not in session or 'repository' not in session:
|
||||||
|
logger.error('Unable to load namespace or repository from session: %s', session)
|
||||||
|
abort(400, message='Missing namespace in request')
|
||||||
|
|
||||||
|
return func(session['namespace'], session['repository'], *args, **kwargs)
|
||||||
|
return wrapper
|
||||||
48
auth/oauth.py
Normal file
48
auth/oauth.py
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from auth.scopes import scopes_from_scope_string
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
from data import model
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def validate_bearer_auth(auth_header):
|
||||||
|
""" Validates an OAuth token found inside a basic auth `Bearer` token, returning whether it
|
||||||
|
points to a valid OAuth token.
|
||||||
|
"""
|
||||||
|
if not auth_header:
|
||||||
|
return ValidateResult(AuthKind.oauth, missing=True)
|
||||||
|
|
||||||
|
normalized = [part.strip() for part in auth_header.split(' ') if part]
|
||||||
|
if normalized[0].lower() != 'bearer' or len(normalized) != 2:
|
||||||
|
logger.debug('Got invalid bearer token format: %s', auth_header)
|
||||||
|
return ValidateResult(AuthKind.oauth, missing=True)
|
||||||
|
|
||||||
|
(_, oauth_token) = normalized
|
||||||
|
return validate_oauth_token(oauth_token)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_oauth_token(token):
|
||||||
|
""" Validates the specified OAuth token, returning whether it points to a valid OAuth token.
|
||||||
|
"""
|
||||||
|
validated = model.oauth.validate_access_token(token)
|
||||||
|
if not validated:
|
||||||
|
logger.warning('OAuth access token could not be validated: %s', token)
|
||||||
|
return ValidateResult(AuthKind.oauth,
|
||||||
|
error_message='OAuth access token could not be validated')
|
||||||
|
|
||||||
|
if validated.expires_at <= datetime.utcnow():
|
||||||
|
logger.warning('OAuth access with an expired token: %s', token)
|
||||||
|
return ValidateResult(AuthKind.oauth, error_message='OAuth access token has expired')
|
||||||
|
|
||||||
|
# Don't allow disabled users to login.
|
||||||
|
if not validated.authorized_user.enabled:
|
||||||
|
return ValidateResult(AuthKind.oauth,
|
||||||
|
error_message='Granter of the oauth access token is disabled')
|
||||||
|
|
||||||
|
# We have a valid token
|
||||||
|
scope_set = scopes_from_scope_string(validated.scope)
|
||||||
|
logger.debug('Successfully validated oauth access token with scope: %s', scope_set)
|
||||||
|
return ValidateResult(AuthKind.oauth, oauthtoken=validated)
|
||||||
364
auth/permissions.py
Normal file
364
auth/permissions.py
Normal file
|
|
@ -0,0 +1,364 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from collections import namedtuple, defaultdict
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
from flask_principal import identity_loaded, Permission, Identity, identity_changed
|
||||||
|
|
||||||
|
|
||||||
|
from app import app, superusers
|
||||||
|
from auth import scopes
|
||||||
|
from data import model
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
_ResourceNeed = namedtuple('resource', ['type', 'namespace', 'name', 'role'])
|
||||||
|
_RepositoryNeed = partial(_ResourceNeed, 'repository')
|
||||||
|
_NamespaceWideNeed = namedtuple('namespacewide', ['type', 'namespace', 'role'])
|
||||||
|
_OrganizationNeed = partial(_NamespaceWideNeed, 'organization')
|
||||||
|
_OrganizationRepoNeed = partial(_NamespaceWideNeed, 'organizationrepo')
|
||||||
|
_TeamTypeNeed = namedtuple('teamwideneed', ['type', 'orgname', 'teamname', 'role'])
|
||||||
|
_TeamNeed = partial(_TeamTypeNeed, 'orgteam')
|
||||||
|
_UserTypeNeed = namedtuple('userspecificneed', ['type', 'username', 'role'])
|
||||||
|
_UserNeed = partial(_UserTypeNeed, 'user')
|
||||||
|
_SuperUserNeed = partial(namedtuple('superuserneed', ['type']), 'superuser')
|
||||||
|
|
||||||
|
|
||||||
|
REPO_ROLES = [None, 'read', 'write', 'admin']
|
||||||
|
TEAM_ROLES = [None, 'member', 'creator', 'admin']
|
||||||
|
USER_ROLES = [None, 'read', 'admin']
|
||||||
|
|
||||||
|
TEAM_ORGWIDE_REPO_ROLES = {
|
||||||
|
'admin': 'admin',
|
||||||
|
'creator': None,
|
||||||
|
'member': None,
|
||||||
|
}
|
||||||
|
|
||||||
|
SCOPE_MAX_REPO_ROLES = defaultdict(lambda: None)
|
||||||
|
SCOPE_MAX_REPO_ROLES.update({
|
||||||
|
scopes.READ_REPO: 'read',
|
||||||
|
scopes.WRITE_REPO: 'write',
|
||||||
|
scopes.ADMIN_REPO: 'admin',
|
||||||
|
scopes.DIRECT_LOGIN: 'admin',
|
||||||
|
})
|
||||||
|
|
||||||
|
SCOPE_MAX_TEAM_ROLES = defaultdict(lambda: None)
|
||||||
|
SCOPE_MAX_TEAM_ROLES.update({
|
||||||
|
scopes.CREATE_REPO: 'creator',
|
||||||
|
scopes.DIRECT_LOGIN: 'admin',
|
||||||
|
scopes.ORG_ADMIN: 'admin',
|
||||||
|
})
|
||||||
|
|
||||||
|
SCOPE_MAX_USER_ROLES = defaultdict(lambda: None)
|
||||||
|
SCOPE_MAX_USER_ROLES.update({
|
||||||
|
scopes.READ_USER: 'read',
|
||||||
|
scopes.DIRECT_LOGIN: 'admin',
|
||||||
|
scopes.ADMIN_USER: 'admin',
|
||||||
|
})
|
||||||
|
|
||||||
|
def repository_read_grant(namespace, repository):
|
||||||
|
return _RepositoryNeed(namespace, repository, 'read')
|
||||||
|
|
||||||
|
|
||||||
|
def repository_write_grant(namespace, repository):
|
||||||
|
return _RepositoryNeed(namespace, repository, 'write')
|
||||||
|
|
||||||
|
|
||||||
|
def repository_admin_grant(namespace, repository):
|
||||||
|
return _RepositoryNeed(namespace, repository, 'admin')
|
||||||
|
|
||||||
|
|
||||||
|
class QuayDeferredPermissionUser(Identity):
|
||||||
|
def __init__(self, uuid, auth_type, auth_scopes, user=None):
|
||||||
|
super(QuayDeferredPermissionUser, self).__init__(uuid, auth_type)
|
||||||
|
|
||||||
|
self._namespace_wide_loaded = set()
|
||||||
|
self._repositories_loaded = set()
|
||||||
|
self._personal_loaded = False
|
||||||
|
|
||||||
|
self._scope_set = auth_scopes
|
||||||
|
self._user_object = user
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def for_id(uuid, auth_scopes=None):
|
||||||
|
auth_scopes = auth_scopes if auth_scopes is not None else {scopes.DIRECT_LOGIN}
|
||||||
|
return QuayDeferredPermissionUser(uuid, 'user_uuid', auth_scopes)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def for_user(user, auth_scopes=None):
|
||||||
|
auth_scopes = auth_scopes if auth_scopes is not None else {scopes.DIRECT_LOGIN}
|
||||||
|
return QuayDeferredPermissionUser(user.uuid, 'user_uuid', auth_scopes, user=user)
|
||||||
|
|
||||||
|
def _translate_role_for_scopes(self, cardinality, max_roles, role):
|
||||||
|
if self._scope_set is None:
|
||||||
|
return role
|
||||||
|
|
||||||
|
max_for_scopes = max({cardinality.index(max_roles[scope]) for scope in self._scope_set})
|
||||||
|
|
||||||
|
if max_for_scopes < cardinality.index(role):
|
||||||
|
logger.debug('Translated permission %s -> %s', role, cardinality[max_for_scopes])
|
||||||
|
return cardinality[max_for_scopes]
|
||||||
|
else:
|
||||||
|
return role
|
||||||
|
|
||||||
|
def _team_role_for_scopes(self, role):
|
||||||
|
return self._translate_role_for_scopes(TEAM_ROLES, SCOPE_MAX_TEAM_ROLES, role)
|
||||||
|
|
||||||
|
def _repo_role_for_scopes(self, role):
|
||||||
|
return self._translate_role_for_scopes(REPO_ROLES, SCOPE_MAX_REPO_ROLES, role)
|
||||||
|
|
||||||
|
def _user_role_for_scopes(self, role):
|
||||||
|
return self._translate_role_for_scopes(USER_ROLES, SCOPE_MAX_USER_ROLES, role)
|
||||||
|
|
||||||
|
def _populate_user_provides(self, user_object):
|
||||||
|
""" Populates the provides that naturally apply to a user, such as being the admin of
|
||||||
|
their own namespace.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Add the user specific permissions, only for non-oauth permission
|
||||||
|
user_grant = _UserNeed(user_object.username, self._user_role_for_scopes('admin'))
|
||||||
|
logger.debug('User permission: {0}'.format(user_grant))
|
||||||
|
self.provides.add(user_grant)
|
||||||
|
|
||||||
|
# Every user is the admin of their own 'org'
|
||||||
|
user_namespace = _OrganizationNeed(user_object.username, self._team_role_for_scopes('admin'))
|
||||||
|
logger.debug('User namespace permission: {0}'.format(user_namespace))
|
||||||
|
self.provides.add(user_namespace)
|
||||||
|
|
||||||
|
# Org repo roles can differ for scopes
|
||||||
|
user_repos = _OrganizationRepoNeed(user_object.username, self._repo_role_for_scopes('admin'))
|
||||||
|
logger.debug('User namespace repo permission: {0}'.format(user_repos))
|
||||||
|
self.provides.add(user_repos)
|
||||||
|
|
||||||
|
if ((scopes.SUPERUSER in self._scope_set or scopes.DIRECT_LOGIN in self._scope_set) and
|
||||||
|
superusers.is_superuser(user_object.username)):
|
||||||
|
logger.debug('Adding superuser to user: %s', user_object.username)
|
||||||
|
self.provides.add(_SuperUserNeed())
|
||||||
|
|
||||||
|
def _populate_namespace_wide_provides(self, user_object, namespace_filter):
|
||||||
|
""" Populates the namespace-wide provides for a particular user under a particular namespace.
|
||||||
|
This method does *not* add any provides for specific repositories.
|
||||||
|
"""
|
||||||
|
|
||||||
|
for team in model.permission.get_org_wide_permissions(user_object, org_filter=namespace_filter):
|
||||||
|
team_org_grant = _OrganizationNeed(team.organization.username,
|
||||||
|
self._team_role_for_scopes(team.role.name))
|
||||||
|
logger.debug('Organization team added permission: {0}'.format(team_org_grant))
|
||||||
|
self.provides.add(team_org_grant)
|
||||||
|
|
||||||
|
team_repo_role = TEAM_ORGWIDE_REPO_ROLES[team.role.name]
|
||||||
|
org_repo_grant = _OrganizationRepoNeed(team.organization.username,
|
||||||
|
self._repo_role_for_scopes(team_repo_role))
|
||||||
|
logger.debug('Organization team added repo permission: {0}'.format(org_repo_grant))
|
||||||
|
self.provides.add(org_repo_grant)
|
||||||
|
|
||||||
|
team_grant = _TeamNeed(team.organization.username, team.name,
|
||||||
|
self._team_role_for_scopes(team.role.name))
|
||||||
|
logger.debug('Team added permission: {0}'.format(team_grant))
|
||||||
|
self.provides.add(team_grant)
|
||||||
|
|
||||||
|
def _populate_repository_provides(self, user_object, namespace_filter, repository_name):
|
||||||
|
""" Populates the repository-specific provides for a particular user and repository. """
|
||||||
|
|
||||||
|
if namespace_filter and repository_name:
|
||||||
|
permissions = model.permission.get_user_repository_permissions(user_object, namespace_filter,
|
||||||
|
repository_name)
|
||||||
|
else:
|
||||||
|
permissions = model.permission.get_all_user_repository_permissions(user_object)
|
||||||
|
|
||||||
|
for perm in permissions:
|
||||||
|
repo_grant = _RepositoryNeed(perm.repository.namespace_user.username, perm.repository.name,
|
||||||
|
self._repo_role_for_scopes(perm.role.name))
|
||||||
|
logger.debug('User added permission: {0}'.format(repo_grant))
|
||||||
|
self.provides.add(repo_grant)
|
||||||
|
|
||||||
|
def can(self, permission):
|
||||||
|
logger.debug('Loading user permissions after deferring for: %s', self.id)
|
||||||
|
user_object = self._user_object or model.user.get_user_by_uuid(self.id)
|
||||||
|
if user_object is None:
|
||||||
|
return super(QuayDeferredPermissionUser, self).can(permission)
|
||||||
|
|
||||||
|
# Add the user-specific provides.
|
||||||
|
if not self._personal_loaded:
|
||||||
|
self._populate_user_provides(user_object)
|
||||||
|
self._personal_loaded = True
|
||||||
|
|
||||||
|
# If we now have permission, no need to load any more permissions.
|
||||||
|
if super(QuayDeferredPermissionUser, self).can(permission):
|
||||||
|
return super(QuayDeferredPermissionUser, self).can(permission)
|
||||||
|
|
||||||
|
# Check for namespace and/or repository permissions.
|
||||||
|
perm_namespace = permission.namespace
|
||||||
|
perm_repo_name = permission.repo_name
|
||||||
|
perm_repository = None
|
||||||
|
|
||||||
|
if perm_namespace and perm_repo_name:
|
||||||
|
perm_repository = '%s/%s' % (perm_namespace, perm_repo_name)
|
||||||
|
|
||||||
|
if not perm_namespace and not perm_repo_name:
|
||||||
|
# Nothing more to load, so just check directly.
|
||||||
|
return super(QuayDeferredPermissionUser, self).can(permission)
|
||||||
|
|
||||||
|
# Lazy-load the repository-specific permissions.
|
||||||
|
if perm_repository and perm_repository not in self._repositories_loaded:
|
||||||
|
self._populate_repository_provides(user_object, perm_namespace, perm_repo_name)
|
||||||
|
self._repositories_loaded.add(perm_repository)
|
||||||
|
|
||||||
|
# If we now have permission, no need to load any more permissions.
|
||||||
|
if super(QuayDeferredPermissionUser, self).can(permission):
|
||||||
|
return super(QuayDeferredPermissionUser, self).can(permission)
|
||||||
|
|
||||||
|
# Lazy-load the namespace-wide-only permissions.
|
||||||
|
if perm_namespace and perm_namespace not in self._namespace_wide_loaded:
|
||||||
|
self._populate_namespace_wide_provides(user_object, perm_namespace)
|
||||||
|
self._namespace_wide_loaded.add(perm_namespace)
|
||||||
|
|
||||||
|
return super(QuayDeferredPermissionUser, self).can(permission)
|
||||||
|
|
||||||
|
|
||||||
|
class QuayPermission(Permission):
|
||||||
|
""" Base for all permissions in Quay. """
|
||||||
|
namespace = None
|
||||||
|
repo_name = None
|
||||||
|
|
||||||
|
|
||||||
|
class ModifyRepositoryPermission(QuayPermission):
|
||||||
|
def __init__(self, namespace, name):
|
||||||
|
admin_need = _RepositoryNeed(namespace, name, 'admin')
|
||||||
|
write_need = _RepositoryNeed(namespace, name, 'write')
|
||||||
|
org_admin_need = _OrganizationRepoNeed(namespace, 'admin')
|
||||||
|
org_write_need = _OrganizationRepoNeed(namespace, 'write')
|
||||||
|
|
||||||
|
self.namespace = namespace
|
||||||
|
self.repo_name = name
|
||||||
|
|
||||||
|
super(ModifyRepositoryPermission, self).__init__(admin_need, write_need, org_admin_need,
|
||||||
|
org_write_need)
|
||||||
|
|
||||||
|
|
||||||
|
class ReadRepositoryPermission(QuayPermission):
|
||||||
|
def __init__(self, namespace, name):
|
||||||
|
admin_need = _RepositoryNeed(namespace, name, 'admin')
|
||||||
|
write_need = _RepositoryNeed(namespace, name, 'write')
|
||||||
|
read_need = _RepositoryNeed(namespace, name, 'read')
|
||||||
|
org_admin_need = _OrganizationRepoNeed(namespace, 'admin')
|
||||||
|
org_write_need = _OrganizationRepoNeed(namespace, 'write')
|
||||||
|
org_read_need = _OrganizationRepoNeed(namespace, 'read')
|
||||||
|
|
||||||
|
self.namespace = namespace
|
||||||
|
self.repo_name = name
|
||||||
|
|
||||||
|
super(ReadRepositoryPermission, self).__init__(admin_need, write_need, read_need,
|
||||||
|
org_admin_need, org_read_need, org_write_need)
|
||||||
|
|
||||||
|
|
||||||
|
class AdministerRepositoryPermission(QuayPermission):
|
||||||
|
def __init__(self, namespace, name):
|
||||||
|
admin_need = _RepositoryNeed(namespace, name, 'admin')
|
||||||
|
org_admin_need = _OrganizationRepoNeed(namespace, 'admin')
|
||||||
|
|
||||||
|
self.namespace = namespace
|
||||||
|
self.repo_name = name
|
||||||
|
|
||||||
|
super(AdministerRepositoryPermission, self).__init__(admin_need,
|
||||||
|
org_admin_need)
|
||||||
|
|
||||||
|
|
||||||
|
class CreateRepositoryPermission(QuayPermission):
|
||||||
|
def __init__(self, namespace):
|
||||||
|
admin_org = _OrganizationNeed(namespace, 'admin')
|
||||||
|
create_repo_org = _OrganizationNeed(namespace, 'creator')
|
||||||
|
|
||||||
|
self.namespace = namespace
|
||||||
|
|
||||||
|
super(CreateRepositoryPermission, self).__init__(admin_org,
|
||||||
|
create_repo_org)
|
||||||
|
|
||||||
|
class SuperUserPermission(QuayPermission):
|
||||||
|
def __init__(self):
|
||||||
|
need = _SuperUserNeed()
|
||||||
|
super(SuperUserPermission, self).__init__(need)
|
||||||
|
|
||||||
|
|
||||||
|
class UserAdminPermission(QuayPermission):
|
||||||
|
def __init__(self, username):
|
||||||
|
user_admin = _UserNeed(username, 'admin')
|
||||||
|
super(UserAdminPermission, self).__init__(user_admin)
|
||||||
|
|
||||||
|
|
||||||
|
class UserReadPermission(QuayPermission):
|
||||||
|
def __init__(self, username):
|
||||||
|
user_admin = _UserNeed(username, 'admin')
|
||||||
|
user_read = _UserNeed(username, 'read')
|
||||||
|
super(UserReadPermission, self).__init__(user_read, user_admin)
|
||||||
|
|
||||||
|
|
||||||
|
class AdministerOrganizationPermission(QuayPermission):
|
||||||
|
def __init__(self, org_name):
|
||||||
|
admin_org = _OrganizationNeed(org_name, 'admin')
|
||||||
|
|
||||||
|
self.namespace = org_name
|
||||||
|
|
||||||
|
super(AdministerOrganizationPermission, self).__init__(admin_org)
|
||||||
|
|
||||||
|
|
||||||
|
class OrganizationMemberPermission(QuayPermission):
|
||||||
|
def __init__(self, org_name):
|
||||||
|
admin_org = _OrganizationNeed(org_name, 'admin')
|
||||||
|
repo_creator_org = _OrganizationNeed(org_name, 'creator')
|
||||||
|
org_member = _OrganizationNeed(org_name, 'member')
|
||||||
|
|
||||||
|
self.namespace = org_name
|
||||||
|
|
||||||
|
super(OrganizationMemberPermission, self).__init__(admin_org, org_member,
|
||||||
|
repo_creator_org)
|
||||||
|
|
||||||
|
|
||||||
|
class ViewTeamPermission(QuayPermission):
|
||||||
|
def __init__(self, org_name, team_name):
|
||||||
|
team_admin = _TeamNeed(org_name, team_name, 'admin')
|
||||||
|
team_creator = _TeamNeed(org_name, team_name, 'creator')
|
||||||
|
team_member = _TeamNeed(org_name, team_name, 'member')
|
||||||
|
admin_org = _OrganizationNeed(org_name, 'admin')
|
||||||
|
|
||||||
|
self.namespace = org_name
|
||||||
|
|
||||||
|
super(ViewTeamPermission, self).__init__(team_admin, team_creator,
|
||||||
|
team_member, admin_org)
|
||||||
|
|
||||||
|
|
||||||
|
class AlwaysFailPermission(QuayPermission):
|
||||||
|
def can(self):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
@identity_loaded.connect_via(app)
|
||||||
|
def on_identity_loaded(sender, identity):
|
||||||
|
logger.debug('Identity loaded: %s' % identity)
|
||||||
|
# We have verified an identity, load in all of the permissions
|
||||||
|
|
||||||
|
if isinstance(identity, QuayDeferredPermissionUser):
|
||||||
|
logger.debug('Deferring permissions for user with uuid: %s', identity.id)
|
||||||
|
|
||||||
|
elif identity.auth_type == 'user_uuid':
|
||||||
|
logger.debug('Switching username permission to deferred object with uuid: %s', identity.id)
|
||||||
|
switch_to_deferred = QuayDeferredPermissionUser.for_id(identity.id)
|
||||||
|
identity_changed.send(app, identity=switch_to_deferred)
|
||||||
|
|
||||||
|
elif identity.auth_type == 'token':
|
||||||
|
logger.debug('Loading permissions for token: %s', identity.id)
|
||||||
|
token_data = model.token.load_token_data(identity.id)
|
||||||
|
|
||||||
|
repo_grant = _RepositoryNeed(token_data.repository.namespace_user.username,
|
||||||
|
token_data.repository.name,
|
||||||
|
token_data.role.name)
|
||||||
|
logger.debug('Delegate token added permission: %s', repo_grant)
|
||||||
|
identity.provides.add(repo_grant)
|
||||||
|
|
||||||
|
elif identity.auth_type == 'signed_grant' or identity.auth_type == 'signed_jwt':
|
||||||
|
logger.debug('Loaded %s identity for: %s', identity.auth_type, identity.id)
|
||||||
|
|
||||||
|
else:
|
||||||
|
logger.error('Unknown identity auth type: %s', identity.auth_type)
|
||||||
164
auth/registry_jwt_auth.py
Normal file
164
auth/registry_jwt_auth.py
Normal file
|
|
@ -0,0 +1,164 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from functools import wraps
|
||||||
|
|
||||||
|
from jsonschema import validate, ValidationError
|
||||||
|
from flask import request, url_for
|
||||||
|
from flask_principal import identity_changed, Identity
|
||||||
|
|
||||||
|
from app import app, get_app_url, instance_keys, metric_queue
|
||||||
|
from auth.auth_context import set_authenticated_context
|
||||||
|
from auth.auth_context_type import SignedAuthContext
|
||||||
|
from auth.permissions import repository_read_grant, repository_write_grant, repository_admin_grant
|
||||||
|
from util.http import abort
|
||||||
|
from util.names import parse_namespace_repository
|
||||||
|
from util.security.registry_jwt import (ANONYMOUS_SUB, decode_bearer_header,
|
||||||
|
InvalidBearerTokenException)
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
ACCESS_SCHEMA = {
|
||||||
|
'type': 'array',
|
||||||
|
'description': 'List of access granted to the subject',
|
||||||
|
'items': {
|
||||||
|
'type': 'object',
|
||||||
|
'required': [
|
||||||
|
'type',
|
||||||
|
'name',
|
||||||
|
'actions',
|
||||||
|
],
|
||||||
|
'properties': {
|
||||||
|
'type': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'We only allow repository permissions',
|
||||||
|
'enum': [
|
||||||
|
'repository',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
'name': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'The name of the repository for which we are receiving access'
|
||||||
|
},
|
||||||
|
'actions': {
|
||||||
|
'type': 'array',
|
||||||
|
'description': 'List of specific verbs which can be performed against repository',
|
||||||
|
'items': {
|
||||||
|
'type': 'string',
|
||||||
|
'enum': [
|
||||||
|
'push',
|
||||||
|
'pull',
|
||||||
|
'*',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidJWTException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def get_auth_headers(repository=None, scopes=None):
|
||||||
|
""" Returns a dictionary of headers for auth responses. """
|
||||||
|
headers = {}
|
||||||
|
realm_auth_path = url_for('v2.generate_registry_jwt')
|
||||||
|
authenticate = 'Bearer realm="{0}{1}",service="{2}"'.format(get_app_url(),
|
||||||
|
realm_auth_path,
|
||||||
|
app.config['SERVER_HOSTNAME'])
|
||||||
|
if repository:
|
||||||
|
scopes_string = "repository:{0}".format(repository)
|
||||||
|
if scopes:
|
||||||
|
scopes_string += ':' + ','.join(scopes)
|
||||||
|
|
||||||
|
authenticate += ',scope="{0}"'.format(scopes_string)
|
||||||
|
|
||||||
|
headers['WWW-Authenticate'] = authenticate
|
||||||
|
headers['Docker-Distribution-API-Version'] = 'registry/2.0'
|
||||||
|
return headers
|
||||||
|
|
||||||
|
|
||||||
|
def identity_from_bearer_token(bearer_header):
|
||||||
|
""" Process a bearer header and return the loaded identity, or raise InvalidJWTException if an
|
||||||
|
identity could not be loaded. Expects tokens and grants in the format of the Docker registry
|
||||||
|
v2 auth spec: https://docs.docker.com/registry/spec/auth/token/
|
||||||
|
"""
|
||||||
|
logger.debug('Validating auth header: %s', bearer_header)
|
||||||
|
|
||||||
|
try:
|
||||||
|
payload = decode_bearer_header(bearer_header, instance_keys, app.config,
|
||||||
|
metric_queue=metric_queue)
|
||||||
|
except InvalidBearerTokenException as bte:
|
||||||
|
logger.exception('Invalid bearer token: %s', bte)
|
||||||
|
raise InvalidJWTException(bte)
|
||||||
|
|
||||||
|
loaded_identity = Identity(payload['sub'], 'signed_jwt')
|
||||||
|
|
||||||
|
# Process the grants from the payload
|
||||||
|
if 'access' in payload:
|
||||||
|
try:
|
||||||
|
validate(payload['access'], ACCESS_SCHEMA)
|
||||||
|
except ValidationError:
|
||||||
|
logger.exception('We should not be minting invalid credentials')
|
||||||
|
raise InvalidJWTException('Token contained invalid or malformed access grants')
|
||||||
|
|
||||||
|
lib_namespace = app.config['LIBRARY_NAMESPACE']
|
||||||
|
for grant in payload['access']:
|
||||||
|
namespace, repo_name = parse_namespace_repository(grant['name'], lib_namespace)
|
||||||
|
|
||||||
|
if '*' in grant['actions']:
|
||||||
|
loaded_identity.provides.add(repository_admin_grant(namespace, repo_name))
|
||||||
|
elif 'push' in grant['actions']:
|
||||||
|
loaded_identity.provides.add(repository_write_grant(namespace, repo_name))
|
||||||
|
elif 'pull' in grant['actions']:
|
||||||
|
loaded_identity.provides.add(repository_read_grant(namespace, repo_name))
|
||||||
|
|
||||||
|
default_context = {
|
||||||
|
'kind': 'anonymous'
|
||||||
|
}
|
||||||
|
|
||||||
|
if payload['sub'] != ANONYMOUS_SUB:
|
||||||
|
default_context = {
|
||||||
|
'kind': 'user',
|
||||||
|
'user': payload['sub'],
|
||||||
|
}
|
||||||
|
|
||||||
|
return loaded_identity, payload.get('context', default_context)
|
||||||
|
|
||||||
|
|
||||||
|
def process_registry_jwt_auth(scopes=None):
|
||||||
|
""" Processes the registry JWT auth token found in the authorization header. If none found,
|
||||||
|
no error is returned. If an invalid token is found, raises a 401.
|
||||||
|
"""
|
||||||
|
def inner(func):
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
logger.debug('Called with params: %s, %s', args, kwargs)
|
||||||
|
auth = request.headers.get('authorization', '').strip()
|
||||||
|
if auth:
|
||||||
|
try:
|
||||||
|
extracted_identity, context_dict = identity_from_bearer_token(auth)
|
||||||
|
identity_changed.send(app, identity=extracted_identity)
|
||||||
|
logger.debug('Identity changed to %s', extracted_identity.id)
|
||||||
|
|
||||||
|
auth_context = SignedAuthContext.build_from_signed_dict(context_dict)
|
||||||
|
if auth_context is not None:
|
||||||
|
logger.debug('Auth context set to %s', auth_context.signed_data)
|
||||||
|
set_authenticated_context(auth_context)
|
||||||
|
|
||||||
|
except InvalidJWTException as ije:
|
||||||
|
repository = None
|
||||||
|
if 'namespace_name' in kwargs and 'repo_name' in kwargs:
|
||||||
|
repository = kwargs['namespace_name'] + '/' + kwargs['repo_name']
|
||||||
|
|
||||||
|
abort(401, message=ije.message, headers=get_auth_headers(repository=repository,
|
||||||
|
scopes=scopes))
|
||||||
|
else:
|
||||||
|
logger.debug('No auth header.')
|
||||||
|
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
return wrapper
|
||||||
|
return inner
|
||||||
146
auth/scopes.py
Normal file
146
auth/scopes.py
Normal file
|
|
@ -0,0 +1,146 @@
|
||||||
|
from collections import namedtuple
|
||||||
|
import features
|
||||||
|
import re
|
||||||
|
|
||||||
|
Scope = namedtuple('scope', ['scope', 'icon', 'dangerous', 'title', 'description'])
|
||||||
|
|
||||||
|
|
||||||
|
READ_REPO = Scope(scope='repo:read',
|
||||||
|
icon='fa-hdd-o',
|
||||||
|
dangerous=False,
|
||||||
|
title='View all visible repositories',
|
||||||
|
description=('This application will be able to view and pull all repositories '
|
||||||
|
'visible to the granting user or robot account'))
|
||||||
|
|
||||||
|
WRITE_REPO = Scope(scope='repo:write',
|
||||||
|
icon='fa-hdd-o',
|
||||||
|
dangerous=False,
|
||||||
|
title='Read/Write to any accessible repositories',
|
||||||
|
description=('This application will be able to view, push and pull to all '
|
||||||
|
'repositories to which the granting user or robot account has '
|
||||||
|
'write access'))
|
||||||
|
|
||||||
|
ADMIN_REPO = Scope(scope='repo:admin',
|
||||||
|
icon='fa-hdd-o',
|
||||||
|
dangerous=False,
|
||||||
|
title='Administer Repositories',
|
||||||
|
description=('This application will have administrator access to all '
|
||||||
|
'repositories to which the granting user or robot account has '
|
||||||
|
'access'))
|
||||||
|
|
||||||
|
CREATE_REPO = Scope(scope='repo:create',
|
||||||
|
icon='fa-plus',
|
||||||
|
dangerous=False,
|
||||||
|
title='Create Repositories',
|
||||||
|
description=('This application will be able to create repositories in to any '
|
||||||
|
'namespaces that the granting user or robot account is allowed '
|
||||||
|
'to create repositories'))
|
||||||
|
|
||||||
|
READ_USER = Scope(scope= 'user:read',
|
||||||
|
icon='fa-user',
|
||||||
|
dangerous=False,
|
||||||
|
title='Read User Information',
|
||||||
|
description=('This application will be able to read user information such as '
|
||||||
|
'username and email address.'))
|
||||||
|
|
||||||
|
ADMIN_USER = Scope(scope= 'user:admin',
|
||||||
|
icon='fa-gear',
|
||||||
|
dangerous=True,
|
||||||
|
title='Administer User',
|
||||||
|
description=('This application will be able to administer your account '
|
||||||
|
'including creating robots and granting them permissions '
|
||||||
|
'to your repositories. You should have absolute trust in the '
|
||||||
|
'requesting application before granting this permission.'))
|
||||||
|
|
||||||
|
ORG_ADMIN = Scope(scope='org:admin',
|
||||||
|
icon='fa-gear',
|
||||||
|
dangerous=True,
|
||||||
|
title='Administer Organization',
|
||||||
|
description=('This application will be able to administer your organizations '
|
||||||
|
'including creating robots, creating teams, adjusting team '
|
||||||
|
'membership, and changing billing settings. You should have '
|
||||||
|
'absolute trust in the requesting application before granting this '
|
||||||
|
'permission.'))
|
||||||
|
|
||||||
|
DIRECT_LOGIN = Scope(scope='direct_user_login',
|
||||||
|
icon='fa-exclamation-triangle',
|
||||||
|
dangerous=True,
|
||||||
|
title='Full Access',
|
||||||
|
description=('This scope should not be available to OAuth applications. '
|
||||||
|
'Never approve a request for this scope!'))
|
||||||
|
|
||||||
|
SUPERUSER = Scope(scope='super:user',
|
||||||
|
icon='fa-street-view',
|
||||||
|
dangerous=True,
|
||||||
|
title='Super User Access',
|
||||||
|
description=('This application will be able to administer your installation '
|
||||||
|
'including managing users, managing organizations and other '
|
||||||
|
'features found in the superuser panel. You should have '
|
||||||
|
'absolute trust in the requesting application before granting this '
|
||||||
|
'permission.'))
|
||||||
|
|
||||||
|
ALL_SCOPES = {scope.scope: scope for scope in (READ_REPO, WRITE_REPO, ADMIN_REPO, CREATE_REPO,
|
||||||
|
READ_USER, ORG_ADMIN, SUPERUSER, ADMIN_USER)}
|
||||||
|
|
||||||
|
IMPLIED_SCOPES = {
|
||||||
|
ADMIN_REPO: {ADMIN_REPO, WRITE_REPO, READ_REPO},
|
||||||
|
WRITE_REPO: {WRITE_REPO, READ_REPO},
|
||||||
|
READ_REPO: {READ_REPO},
|
||||||
|
CREATE_REPO: {CREATE_REPO},
|
||||||
|
READ_USER: {READ_USER},
|
||||||
|
ORG_ADMIN: {ORG_ADMIN},
|
||||||
|
SUPERUSER: {SUPERUSER},
|
||||||
|
ADMIN_USER: {ADMIN_USER},
|
||||||
|
None: set(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def app_scopes(app_config):
|
||||||
|
scopes_from_config = dict(ALL_SCOPES)
|
||||||
|
if not app_config.get('FEATURE_SUPER_USERS', False):
|
||||||
|
del scopes_from_config[SUPERUSER.scope]
|
||||||
|
return scopes_from_config
|
||||||
|
|
||||||
|
|
||||||
|
def scopes_from_scope_string(scopes):
|
||||||
|
if not scopes:
|
||||||
|
scopes = ''
|
||||||
|
|
||||||
|
# Note: The scopes string should be space seperated according to the spec:
|
||||||
|
# https://tools.ietf.org/html/rfc6749#section-3.3
|
||||||
|
# However, we also support commas for backwards compatibility with existing callers to our code.
|
||||||
|
scope_set = {ALL_SCOPES.get(scope, None) for scope in re.split(' |,', scopes)}
|
||||||
|
return scope_set if not None in scope_set else set()
|
||||||
|
|
||||||
|
|
||||||
|
def validate_scope_string(scopes):
|
||||||
|
decoded = scopes_from_scope_string(scopes)
|
||||||
|
return len(decoded) > 0
|
||||||
|
|
||||||
|
|
||||||
|
def is_subset_string(full_string, expected_string):
|
||||||
|
""" Returns true if the scopes found in expected_string are also found
|
||||||
|
in full_string.
|
||||||
|
"""
|
||||||
|
full_scopes = scopes_from_scope_string(full_string)
|
||||||
|
if not full_scopes:
|
||||||
|
return False
|
||||||
|
|
||||||
|
full_implied_scopes = set.union(*[IMPLIED_SCOPES[scope] for scope in full_scopes])
|
||||||
|
expected_scopes = scopes_from_scope_string(expected_string)
|
||||||
|
return expected_scopes.issubset(full_implied_scopes)
|
||||||
|
|
||||||
|
|
||||||
|
def get_scope_information(scopes_string):
|
||||||
|
scopes = scopes_from_scope_string(scopes_string)
|
||||||
|
scope_info = []
|
||||||
|
for scope in scopes:
|
||||||
|
scope_info.append({
|
||||||
|
'title': scope.title,
|
||||||
|
'scope': scope.scope,
|
||||||
|
'description': scope.description,
|
||||||
|
'icon': scope.icon,
|
||||||
|
'dangerous': scope.dangerous,
|
||||||
|
})
|
||||||
|
|
||||||
|
return scope_info
|
||||||
55
auth/signedgrant.py
Normal file
55
auth/signedgrant.py
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from flask.sessions import SecureCookieSessionInterface, BadSignature
|
||||||
|
|
||||||
|
from app import app
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# The prefix for all signatures of signed granted.
|
||||||
|
SIGNATURE_PREFIX = 'sigv2='
|
||||||
|
|
||||||
|
def generate_signed_token(grants, user_context):
|
||||||
|
""" Generates a signed session token with the given grants and user context. """
|
||||||
|
ser = SecureCookieSessionInterface().get_signing_serializer(app)
|
||||||
|
data_to_sign = {
|
||||||
|
'grants': grants,
|
||||||
|
'user_context': user_context,
|
||||||
|
}
|
||||||
|
|
||||||
|
encrypted = ser.dumps(data_to_sign)
|
||||||
|
return '{0}{1}'.format(SIGNATURE_PREFIX, encrypted)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_signed_grant(auth_header):
|
||||||
|
""" Validates a signed grant as found inside an auth header and returns whether it points to
|
||||||
|
a valid grant.
|
||||||
|
"""
|
||||||
|
if not auth_header:
|
||||||
|
return ValidateResult(AuthKind.signed_grant, missing=True)
|
||||||
|
|
||||||
|
# Try to parse the token from the header.
|
||||||
|
normalized = [part.strip() for part in auth_header.split(' ') if part]
|
||||||
|
if normalized[0].lower() != 'token' or len(normalized) != 2:
|
||||||
|
logger.debug('Not a token: %s', auth_header)
|
||||||
|
return ValidateResult(AuthKind.signed_grant, missing=True)
|
||||||
|
|
||||||
|
# Check that it starts with the expected prefix.
|
||||||
|
if not normalized[1].startswith(SIGNATURE_PREFIX):
|
||||||
|
logger.debug('Not a signed grant token: %s', auth_header)
|
||||||
|
return ValidateResult(AuthKind.signed_grant, missing=True)
|
||||||
|
|
||||||
|
# Decrypt the grant.
|
||||||
|
encrypted = normalized[1][len(SIGNATURE_PREFIX):]
|
||||||
|
ser = SecureCookieSessionInterface().get_signing_serializer(app)
|
||||||
|
|
||||||
|
try:
|
||||||
|
token_data = ser.loads(encrypted, max_age=app.config['SIGNED_GRANT_EXPIRATION_SEC'])
|
||||||
|
except BadSignature:
|
||||||
|
logger.warning('Signed grant could not be validated: %s', encrypted)
|
||||||
|
return ValidateResult(AuthKind.signed_grant,
|
||||||
|
error_message='Signed grant could not be validated')
|
||||||
|
|
||||||
|
logger.debug('Successfully validated signed grant with data: %s', token_data)
|
||||||
|
return ValidateResult(AuthKind.signed_grant, signed_data=token_data)
|
||||||
51
auth/test/test_auth_context_type.py
Normal file
51
auth/test/test_auth_context_type.py
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from auth.auth_context_type import SignedAuthContext, ValidatedAuthContext, ContextEntityKind
|
||||||
|
from data import model, database
|
||||||
|
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
def get_oauth_token(_):
|
||||||
|
return database.OAuthAccessToken.get()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('kind, entity_reference, loader', [
|
||||||
|
(ContextEntityKind.anonymous, None, None),
|
||||||
|
(ContextEntityKind.appspecifictoken, '%s%s' % ('a' * 60, 'b' * 60),
|
||||||
|
model.appspecifictoken.access_valid_token),
|
||||||
|
(ContextEntityKind.oauthtoken, None, get_oauth_token),
|
||||||
|
(ContextEntityKind.robot, 'devtable+dtrobot', model.user.lookup_robot),
|
||||||
|
(ContextEntityKind.user, 'devtable', model.user.get_user),
|
||||||
|
])
|
||||||
|
@pytest.mark.parametrize('v1_dict_format', [
|
||||||
|
(True),
|
||||||
|
(False),
|
||||||
|
])
|
||||||
|
def test_signed_auth_context(kind, entity_reference, loader, v1_dict_format, initialized_db):
|
||||||
|
if kind == ContextEntityKind.anonymous:
|
||||||
|
validated = ValidatedAuthContext()
|
||||||
|
assert validated.is_anonymous
|
||||||
|
else:
|
||||||
|
ref = loader(entity_reference)
|
||||||
|
validated = ValidatedAuthContext(**{kind.value: ref})
|
||||||
|
assert not validated.is_anonymous
|
||||||
|
|
||||||
|
assert validated.entity_kind == kind
|
||||||
|
assert validated.unique_key
|
||||||
|
|
||||||
|
signed = SignedAuthContext.build_from_signed_dict(validated.to_signed_dict(),
|
||||||
|
v1_dict_format=v1_dict_format)
|
||||||
|
|
||||||
|
if not v1_dict_format:
|
||||||
|
# Under legacy V1 format, we don't track the app specific token, merely its associated user.
|
||||||
|
assert signed.entity_kind == kind
|
||||||
|
assert signed.description == validated.description
|
||||||
|
assert signed.credential_username == validated.credential_username
|
||||||
|
assert signed.analytics_id_and_public_metadata() == validated.analytics_id_and_public_metadata()
|
||||||
|
assert signed.unique_key == validated.unique_key
|
||||||
|
|
||||||
|
assert signed.is_anonymous == validated.is_anonymous
|
||||||
|
assert signed.authed_user == validated.authed_user
|
||||||
|
assert signed.has_nonrobot_user == validated.has_nonrobot_user
|
||||||
|
|
||||||
|
assert signed.to_signed_dict() == validated.to_signed_dict()
|
||||||
98
auth/test/test_basic.py
Normal file
98
auth/test/test_basic.py
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from base64 import b64encode
|
||||||
|
|
||||||
|
from auth.basic import validate_basic_auth
|
||||||
|
from auth.credentials import (ACCESS_TOKEN_USERNAME, OAUTH_TOKEN_USERNAME,
|
||||||
|
APP_SPECIFIC_TOKEN_USERNAME)
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
from data import model
|
||||||
|
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
|
||||||
|
def _token(username, password):
|
||||||
|
assert isinstance(username, basestring)
|
||||||
|
assert isinstance(password, basestring)
|
||||||
|
return 'basic ' + b64encode('%s:%s' % (username, password))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('token, expected_result', [
|
||||||
|
('', ValidateResult(AuthKind.basic, missing=True)),
|
||||||
|
('someinvalidtoken', ValidateResult(AuthKind.basic, missing=True)),
|
||||||
|
('somefoobartoken', ValidateResult(AuthKind.basic, missing=True)),
|
||||||
|
('basic ', ValidateResult(AuthKind.basic, missing=True)),
|
||||||
|
('basic some token', ValidateResult(AuthKind.basic, missing=True)),
|
||||||
|
('basic sometoken', ValidateResult(AuthKind.basic, missing=True)),
|
||||||
|
(_token(APP_SPECIFIC_TOKEN_USERNAME, 'invalid'), ValidateResult(AuthKind.basic,
|
||||||
|
error_message='Invalid token')),
|
||||||
|
(_token(ACCESS_TOKEN_USERNAME, 'invalid'), ValidateResult(AuthKind.basic,
|
||||||
|
error_message='Invalid access token')),
|
||||||
|
(_token(OAUTH_TOKEN_USERNAME, 'invalid'),
|
||||||
|
ValidateResult(AuthKind.basic, error_message='OAuth access token could not be validated')),
|
||||||
|
(_token('devtable', 'invalid'), ValidateResult(AuthKind.basic,
|
||||||
|
error_message='Invalid Username or Password')),
|
||||||
|
(_token('devtable+somebot', 'invalid'), ValidateResult(
|
||||||
|
AuthKind.basic, error_message='Could not find robot with username: devtable+somebot')),
|
||||||
|
(_token('disabled', 'password'), ValidateResult(
|
||||||
|
AuthKind.basic,
|
||||||
|
error_message='This user has been disabled. Please contact your administrator.')),])
|
||||||
|
def test_validate_basic_auth_token(token, expected_result, app):
|
||||||
|
result = validate_basic_auth(token)
|
||||||
|
assert result == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_user(app):
|
||||||
|
token = _token('devtable', 'password')
|
||||||
|
result = validate_basic_auth(token)
|
||||||
|
assert result == ValidateResult(AuthKind.basic, user=model.user.get_user('devtable'))
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_robot(app):
|
||||||
|
robot, password = model.user.create_robot('somerobot', model.user.get_user('devtable'))
|
||||||
|
token = _token(robot.username, password)
|
||||||
|
result = validate_basic_auth(token)
|
||||||
|
assert result == ValidateResult(AuthKind.basic, robot=robot)
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_token(app):
|
||||||
|
access_token = model.token.create_delegate_token('devtable', 'simple', 'sometoken')
|
||||||
|
token = _token(ACCESS_TOKEN_USERNAME, access_token.get_code())
|
||||||
|
result = validate_basic_auth(token)
|
||||||
|
assert result == ValidateResult(AuthKind.basic, token=access_token)
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_oauth(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
app = model.oauth.list_applications_for_org(model.user.get_user_or_org('buynlarge'))[0]
|
||||||
|
oauth_token, code = model.oauth.create_access_token_for_testing(user, app.client_id, 'repo:read')
|
||||||
|
token = _token(OAUTH_TOKEN_USERNAME, code)
|
||||||
|
result = validate_basic_auth(token)
|
||||||
|
assert result == ValidateResult(AuthKind.basic, oauthtoken=oauth_token)
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_app_specific_token(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
app_specific_token = model.appspecifictoken.create_token(user, 'some token')
|
||||||
|
full_token = model.appspecifictoken.get_full_token_string(app_specific_token)
|
||||||
|
token = _token(APP_SPECIFIC_TOKEN_USERNAME, full_token)
|
||||||
|
result = validate_basic_auth(token)
|
||||||
|
assert result == ValidateResult(AuthKind.basic, appspecifictoken=app_specific_token)
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_unicode(app):
|
||||||
|
token = '\xebOH'
|
||||||
|
header = 'basic ' + b64encode(token)
|
||||||
|
result = validate_basic_auth(header)
|
||||||
|
assert result == ValidateResult(AuthKind.basic, missing=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_unicode_2(app):
|
||||||
|
token = '“4JPCOLIVMAY32Q3XGVPHC4CBF8SKII5FWNYMASOFDIVSXTC5I5NBU”'
|
||||||
|
header = 'basic ' + b64encode('devtable+somerobot:%s' % token)
|
||||||
|
result = validate_basic_auth(header)
|
||||||
|
assert result == ValidateResult(
|
||||||
|
AuthKind.basic,
|
||||||
|
error_message='Could not find robot with username: devtable+somerobot and supplied password.')
|
||||||
66
auth/test/test_cookie.py
Normal file
66
auth/test/test_cookie.py
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from flask_login import login_user
|
||||||
|
|
||||||
|
from app import LoginWrappedDBUser
|
||||||
|
from data import model
|
||||||
|
from auth.cookie import validate_session_cookie
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
|
||||||
|
def test_anonymous_cookie(app):
|
||||||
|
assert validate_session_cookie().missing
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalidformatted_cookie(app):
|
||||||
|
# "Login" with a non-UUID reference.
|
||||||
|
someuser = model.user.get_user('devtable')
|
||||||
|
login_user(LoginWrappedDBUser('somenonuuid', someuser))
|
||||||
|
|
||||||
|
# Ensure we get an invalid session cookie format error.
|
||||||
|
result = validate_session_cookie()
|
||||||
|
assert result.authed_user is None
|
||||||
|
assert result.context.identity is None
|
||||||
|
assert not result.has_nonrobot_user
|
||||||
|
assert result.error_message == 'Invalid session cookie format'
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_user(app):
|
||||||
|
# "Login" with a disabled user.
|
||||||
|
someuser = model.user.get_user('disabled')
|
||||||
|
login_user(LoginWrappedDBUser(someuser.uuid, someuser))
|
||||||
|
|
||||||
|
# Ensure we get an invalid session cookie format error.
|
||||||
|
result = validate_session_cookie()
|
||||||
|
assert result.authed_user is None
|
||||||
|
assert result.context.identity is None
|
||||||
|
assert not result.has_nonrobot_user
|
||||||
|
assert result.error_message == 'User account is disabled'
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_user(app):
|
||||||
|
# Login with a valid user.
|
||||||
|
someuser = model.user.get_user('devtable')
|
||||||
|
login_user(LoginWrappedDBUser(someuser.uuid, someuser))
|
||||||
|
|
||||||
|
result = validate_session_cookie()
|
||||||
|
assert result.authed_user == someuser
|
||||||
|
assert result.context.identity is not None
|
||||||
|
assert result.has_nonrobot_user
|
||||||
|
assert result.error_message is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_organization(app):
|
||||||
|
# "Login" with a valid organization.
|
||||||
|
someorg = model.user.get_namespace_user('buynlarge')
|
||||||
|
someorg.uuid = str(uuid.uuid4())
|
||||||
|
someorg.verified = True
|
||||||
|
someorg.save()
|
||||||
|
|
||||||
|
login_user(LoginWrappedDBUser(someorg.uuid, someorg))
|
||||||
|
|
||||||
|
result = validate_session_cookie()
|
||||||
|
assert result.authed_user is None
|
||||||
|
assert result.context.identity is None
|
||||||
|
assert not result.has_nonrobot_user
|
||||||
|
assert result.error_message == 'Cannot login to organization'
|
||||||
147
auth/test/test_credentials.py
Normal file
147
auth/test/test_credentials.py
Normal file
|
|
@ -0,0 +1,147 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
from auth.credentials import validate_credentials, CredentialKind
|
||||||
|
from auth.credential_consts import (ACCESS_TOKEN_USERNAME, OAUTH_TOKEN_USERNAME,
|
||||||
|
APP_SPECIFIC_TOKEN_USERNAME)
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
from data import model
|
||||||
|
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
def test_valid_user(app):
|
||||||
|
result, kind = validate_credentials('devtable', 'password')
|
||||||
|
assert kind == CredentialKind.user
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, user=model.user.get_user('devtable'))
|
||||||
|
|
||||||
|
def test_valid_robot(app):
|
||||||
|
robot, password = model.user.create_robot('somerobot', model.user.get_user('devtable'))
|
||||||
|
result, kind = validate_credentials(robot.username, password)
|
||||||
|
assert kind == CredentialKind.robot
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, robot=robot)
|
||||||
|
|
||||||
|
def test_valid_robot_for_disabled_user(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
user.enabled = False
|
||||||
|
user.save()
|
||||||
|
|
||||||
|
robot, password = model.user.create_robot('somerobot', user)
|
||||||
|
result, kind = validate_credentials(robot.username, password)
|
||||||
|
assert kind == CredentialKind.robot
|
||||||
|
|
||||||
|
err = 'This user has been disabled. Please contact your administrator.'
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, error_message=err)
|
||||||
|
|
||||||
|
def test_valid_token(app):
|
||||||
|
access_token = model.token.create_delegate_token('devtable', 'simple', 'sometoken')
|
||||||
|
result, kind = validate_credentials(ACCESS_TOKEN_USERNAME, access_token.get_code())
|
||||||
|
assert kind == CredentialKind.token
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, token=access_token)
|
||||||
|
|
||||||
|
def test_valid_oauth(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
app = model.oauth.list_applications_for_org(model.user.get_user_or_org('buynlarge'))[0]
|
||||||
|
oauth_token, code = model.oauth.create_access_token_for_testing(user, app.client_id, 'repo:read')
|
||||||
|
result, kind = validate_credentials(OAUTH_TOKEN_USERNAME, code)
|
||||||
|
assert kind == CredentialKind.oauth_token
|
||||||
|
assert result == ValidateResult(AuthKind.oauth, oauthtoken=oauth_token)
|
||||||
|
|
||||||
|
def test_invalid_user(app):
|
||||||
|
result, kind = validate_credentials('devtable', 'somepassword')
|
||||||
|
assert kind == CredentialKind.user
|
||||||
|
assert result == ValidateResult(AuthKind.credentials,
|
||||||
|
error_message='Invalid Username or Password')
|
||||||
|
|
||||||
|
def test_valid_app_specific_token(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
app_specific_token = model.appspecifictoken.create_token(user, 'some token')
|
||||||
|
full_token = model.appspecifictoken.get_full_token_string(app_specific_token)
|
||||||
|
result, kind = validate_credentials(APP_SPECIFIC_TOKEN_USERNAME, full_token)
|
||||||
|
assert kind == CredentialKind.app_specific_token
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, appspecifictoken=app_specific_token)
|
||||||
|
|
||||||
|
def test_valid_app_specific_token_for_disabled_user(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
user.enabled = False
|
||||||
|
user.save()
|
||||||
|
|
||||||
|
app_specific_token = model.appspecifictoken.create_token(user, 'some token')
|
||||||
|
full_token = model.appspecifictoken.get_full_token_string(app_specific_token)
|
||||||
|
result, kind = validate_credentials(APP_SPECIFIC_TOKEN_USERNAME, full_token)
|
||||||
|
assert kind == CredentialKind.app_specific_token
|
||||||
|
|
||||||
|
err = 'This user has been disabled. Please contact your administrator.'
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, error_message=err)
|
||||||
|
|
||||||
|
def test_invalid_app_specific_token(app):
|
||||||
|
result, kind = validate_credentials(APP_SPECIFIC_TOKEN_USERNAME, 'somecode')
|
||||||
|
assert kind == CredentialKind.app_specific_token
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, error_message='Invalid token')
|
||||||
|
|
||||||
|
def test_invalid_app_specific_token_code(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
app_specific_token = model.appspecifictoken.create_token(user, 'some token')
|
||||||
|
full_token = app_specific_token.token_name + 'something'
|
||||||
|
result, kind = validate_credentials(APP_SPECIFIC_TOKEN_USERNAME, full_token)
|
||||||
|
assert kind == CredentialKind.app_specific_token
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, error_message='Invalid token')
|
||||||
|
|
||||||
|
def test_unicode(app):
|
||||||
|
result, kind = validate_credentials('someusername', 'some₪code')
|
||||||
|
assert kind == CredentialKind.user
|
||||||
|
assert not result.auth_valid
|
||||||
|
assert result == ValidateResult(AuthKind.credentials,
|
||||||
|
error_message='Invalid Username or Password')
|
||||||
|
|
||||||
|
def test_unicode_robot(app):
|
||||||
|
robot, _ = model.user.create_robot('somerobot', model.user.get_user('devtable'))
|
||||||
|
result, kind = validate_credentials(robot.username, 'some₪code')
|
||||||
|
|
||||||
|
assert kind == CredentialKind.robot
|
||||||
|
assert not result.auth_valid
|
||||||
|
|
||||||
|
msg = 'Could not find robot with username: devtable+somerobot and supplied password.'
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, error_message=msg)
|
||||||
|
|
||||||
|
def test_invalid_user(app):
|
||||||
|
result, kind = validate_credentials('someinvaliduser', 'password')
|
||||||
|
assert kind == CredentialKind.user
|
||||||
|
assert not result.authed_user
|
||||||
|
assert not result.auth_valid
|
||||||
|
|
||||||
|
def test_invalid_user_password(app):
|
||||||
|
result, kind = validate_credentials('devtable', 'somepassword')
|
||||||
|
assert kind == CredentialKind.user
|
||||||
|
assert not result.authed_user
|
||||||
|
assert not result.auth_valid
|
||||||
|
|
||||||
|
def test_invalid_robot(app):
|
||||||
|
result, kind = validate_credentials('devtable+doesnotexist', 'password')
|
||||||
|
assert kind == CredentialKind.robot
|
||||||
|
assert not result.authed_user
|
||||||
|
assert not result.auth_valid
|
||||||
|
|
||||||
|
def test_invalid_robot_token(app):
|
||||||
|
robot, _ = model.user.create_robot('somerobot', model.user.get_user('devtable'))
|
||||||
|
result, kind = validate_credentials(robot.username, 'invalidpassword')
|
||||||
|
assert kind == CredentialKind.robot
|
||||||
|
assert not result.authed_user
|
||||||
|
assert not result.auth_valid
|
||||||
|
|
||||||
|
def test_invalid_unicode_robot(app):
|
||||||
|
token = '“4JPCOLIVMAY32Q3XGVPHC4CBF8SKII5FWNYMASOFDIVSXTC5I5NBU”'
|
||||||
|
result, kind = validate_credentials('devtable+somerobot', token)
|
||||||
|
assert kind == CredentialKind.robot
|
||||||
|
assert not result.auth_valid
|
||||||
|
msg = 'Could not find robot with username: devtable+somerobot'
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, error_message=msg)
|
||||||
|
|
||||||
|
def test_invalid_unicode_robot_2(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
robot, password = model.user.create_robot('somerobot', user)
|
||||||
|
|
||||||
|
token = '“4JPCOLIVMAY32Q3XGVPHC4CBF8SKII5FWNYMASOFDIVSXTC5I5NBU”'
|
||||||
|
result, kind = validate_credentials('devtable+somerobot', token)
|
||||||
|
assert kind == CredentialKind.robot
|
||||||
|
assert not result.auth_valid
|
||||||
|
msg = 'Could not find robot with username: devtable+somerobot and supplied password.'
|
||||||
|
assert result == ValidateResult(AuthKind.credentials, error_message=msg)
|
||||||
105
auth/test/test_decorators.py
Normal file
105
auth/test/test_decorators.py
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from flask import session
|
||||||
|
from flask_login import login_user
|
||||||
|
from werkzeug.exceptions import HTTPException
|
||||||
|
|
||||||
|
from app import LoginWrappedDBUser
|
||||||
|
from auth.auth_context import get_authenticated_user
|
||||||
|
from auth.decorators import (
|
||||||
|
extract_namespace_repo_from_session, require_session_login, process_auth_or_cookie)
|
||||||
|
from data import model
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_namespace_repo_from_session_missing(app):
|
||||||
|
def emptyfunc():
|
||||||
|
pass
|
||||||
|
|
||||||
|
session.clear()
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
extract_namespace_repo_from_session(emptyfunc)()
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_namespace_repo_from_session_present(app):
|
||||||
|
encountered = []
|
||||||
|
|
||||||
|
def somefunc(namespace, repository):
|
||||||
|
encountered.append(namespace)
|
||||||
|
encountered.append(repository)
|
||||||
|
|
||||||
|
# Add the namespace and repository to the session.
|
||||||
|
session.clear()
|
||||||
|
session['namespace'] = 'foo'
|
||||||
|
session['repository'] = 'bar'
|
||||||
|
|
||||||
|
# Call the decorated method.
|
||||||
|
extract_namespace_repo_from_session(somefunc)()
|
||||||
|
|
||||||
|
assert encountered[0] == 'foo'
|
||||||
|
assert encountered[1] == 'bar'
|
||||||
|
|
||||||
|
|
||||||
|
def test_require_session_login_missing(app):
|
||||||
|
def emptyfunc():
|
||||||
|
pass
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
require_session_login(emptyfunc)()
|
||||||
|
|
||||||
|
|
||||||
|
def test_require_session_login_valid_user(app):
|
||||||
|
def emptyfunc():
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Login as a valid user.
|
||||||
|
someuser = model.user.get_user('devtable')
|
||||||
|
login_user(LoginWrappedDBUser(someuser.uuid, someuser))
|
||||||
|
|
||||||
|
# Call the function.
|
||||||
|
require_session_login(emptyfunc)()
|
||||||
|
|
||||||
|
# Ensure the authenticated user was updated.
|
||||||
|
assert get_authenticated_user() == someuser
|
||||||
|
|
||||||
|
|
||||||
|
def test_require_session_login_invalid_user(app):
|
||||||
|
def emptyfunc():
|
||||||
|
pass
|
||||||
|
|
||||||
|
# "Login" as a disabled user.
|
||||||
|
someuser = model.user.get_user('disabled')
|
||||||
|
login_user(LoginWrappedDBUser(someuser.uuid, someuser))
|
||||||
|
|
||||||
|
# Call the function.
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
require_session_login(emptyfunc)()
|
||||||
|
|
||||||
|
# Ensure the authenticated user was not updated.
|
||||||
|
assert get_authenticated_user() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_auth_or_cookie_invalid_user(app):
|
||||||
|
def emptyfunc():
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Call the function.
|
||||||
|
process_auth_or_cookie(emptyfunc)()
|
||||||
|
|
||||||
|
# Ensure the authenticated user was not updated.
|
||||||
|
assert get_authenticated_user() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_auth_or_cookie_valid_user(app):
|
||||||
|
def emptyfunc():
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Login as a valid user.
|
||||||
|
someuser = model.user.get_user('devtable')
|
||||||
|
login_user(LoginWrappedDBUser(someuser.uuid, someuser))
|
||||||
|
|
||||||
|
# Call the function.
|
||||||
|
process_auth_or_cookie(emptyfunc)()
|
||||||
|
|
||||||
|
# Ensure the authenticated user was updated.
|
||||||
|
assert get_authenticated_user() == someuser
|
||||||
55
auth/test/test_oauth.py
Normal file
55
auth/test/test_oauth.py
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from auth.oauth import validate_bearer_auth, validate_oauth_token
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
from data import model
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('header, expected_result', [
|
||||||
|
('', ValidateResult(AuthKind.oauth, missing=True)),
|
||||||
|
('somerandomtoken', ValidateResult(AuthKind.oauth, missing=True)),
|
||||||
|
('bearer some random token', ValidateResult(AuthKind.oauth, missing=True)),
|
||||||
|
('bearer invalidtoken',
|
||||||
|
ValidateResult(AuthKind.oauth, error_message='OAuth access token could not be validated')),])
|
||||||
|
def test_bearer(header, expected_result, app):
|
||||||
|
assert validate_bearer_auth(header) == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_oauth(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
app = model.oauth.list_applications_for_org(model.user.get_user_or_org('buynlarge'))[0]
|
||||||
|
token_string = '%s%s' % ('a' * 20, 'b' * 20)
|
||||||
|
oauth_token, _ = model.oauth.create_access_token_for_testing(user, app.client_id, 'repo:read',
|
||||||
|
access_token=token_string)
|
||||||
|
result = validate_bearer_auth('bearer ' + token_string)
|
||||||
|
assert result.context.oauthtoken == oauth_token
|
||||||
|
assert result.authed_user == user
|
||||||
|
assert result.auth_valid
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_user_oauth(app):
|
||||||
|
user = model.user.get_user('disabled')
|
||||||
|
token_string = '%s%s' % ('a' * 20, 'b' * 20)
|
||||||
|
oauth_token, _ = model.oauth.create_access_token_for_testing(user, 'deadbeef', 'repo:admin',
|
||||||
|
access_token=token_string)
|
||||||
|
|
||||||
|
result = validate_bearer_auth('bearer ' + token_string)
|
||||||
|
assert result.context.oauthtoken is None
|
||||||
|
assert result.authed_user is None
|
||||||
|
assert not result.auth_valid
|
||||||
|
assert result.error_message == 'Granter of the oauth access token is disabled'
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_token(app):
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
token_string = '%s%s' % ('a' * 20, 'b' * 20)
|
||||||
|
oauth_token, _ = model.oauth.create_access_token_for_testing(user, 'deadbeef', 'repo:admin',
|
||||||
|
access_token=token_string,
|
||||||
|
expires_in=-1000)
|
||||||
|
|
||||||
|
result = validate_bearer_auth('bearer ' + token_string)
|
||||||
|
assert result.context.oauthtoken is None
|
||||||
|
assert result.authed_user is None
|
||||||
|
assert not result.auth_valid
|
||||||
|
assert result.error_message == 'OAuth access token has expired'
|
||||||
37
auth/test/test_permissions.py
Normal file
37
auth/test/test_permissions.py
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from auth import scopes
|
||||||
|
from auth.permissions import SuperUserPermission, QuayDeferredPermissionUser
|
||||||
|
from data import model
|
||||||
|
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
SUPER_USERNAME = 'devtable'
|
||||||
|
UNSUPER_USERNAME = 'freshuser'
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def superuser(initialized_db):
|
||||||
|
return model.user.get_user(SUPER_USERNAME)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def normie(initialized_db):
|
||||||
|
return model.user.get_user(UNSUPER_USERNAME)
|
||||||
|
|
||||||
|
|
||||||
|
def test_superuser_matrix(superuser, normie):
|
||||||
|
test_cases = [
|
||||||
|
(superuser, {scopes.SUPERUSER}, True),
|
||||||
|
(superuser, {scopes.DIRECT_LOGIN}, True),
|
||||||
|
(superuser, {scopes.READ_USER, scopes.SUPERUSER}, True),
|
||||||
|
(superuser, {scopes.READ_USER}, False),
|
||||||
|
(normie, {scopes.SUPERUSER}, False),
|
||||||
|
(normie, {scopes.DIRECT_LOGIN}, False),
|
||||||
|
(normie, {scopes.READ_USER, scopes.SUPERUSER}, False),
|
||||||
|
(normie, {scopes.READ_USER}, False),
|
||||||
|
]
|
||||||
|
|
||||||
|
for user_obj, scope_set, expected in test_cases:
|
||||||
|
perm_user = QuayDeferredPermissionUser.for_user(user_obj, scope_set)
|
||||||
|
has_su = perm_user.can(SuperUserPermission())
|
||||||
|
assert has_su == expected
|
||||||
203
auth/test/test_registry_jwt.py
Normal file
203
auth/test/test_registry_jwt.py
Normal file
|
|
@ -0,0 +1,203 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
import jwt
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app import app, instance_keys
|
||||||
|
from auth.auth_context_type import ValidatedAuthContext
|
||||||
|
from auth.registry_jwt_auth import identity_from_bearer_token, InvalidJWTException
|
||||||
|
from data import model # TODO: remove this after service keys are decoupled
|
||||||
|
from data.database import ServiceKeyApprovalType
|
||||||
|
from initdb import setup_database_for_testing, finished_database_for_testing
|
||||||
|
from util.morecollections import AttrDict
|
||||||
|
from util.security.registry_jwt import ANONYMOUS_SUB, build_context_and_subject
|
||||||
|
|
||||||
|
TEST_AUDIENCE = app.config['SERVER_HOSTNAME']
|
||||||
|
TEST_USER = AttrDict({'username': 'joeuser', 'uuid': 'foobar', 'enabled': True})
|
||||||
|
MAX_SIGNED_S = 3660
|
||||||
|
TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour
|
||||||
|
ANONYMOUS_SUB = '(anonymous)'
|
||||||
|
SERVICE_NAME = 'quay'
|
||||||
|
|
||||||
|
# This import has to come below any references to "app".
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
|
||||||
|
def _access(typ='repository', name='somens/somerepo', actions=None):
|
||||||
|
actions = [] if actions is None else actions
|
||||||
|
return [{
|
||||||
|
'type': typ,
|
||||||
|
'name': name,
|
||||||
|
'actions': actions,
|
||||||
|
}]
|
||||||
|
|
||||||
|
|
||||||
|
def _delete_field(token_data, field_name):
|
||||||
|
token_data.pop(field_name)
|
||||||
|
return token_data
|
||||||
|
|
||||||
|
|
||||||
|
def _token_data(access=[], context=None, audience=TEST_AUDIENCE, user=TEST_USER, iat=None,
|
||||||
|
exp=None, nbf=None, iss=None, subject=None):
|
||||||
|
if subject is None:
|
||||||
|
_, subject = build_context_and_subject(ValidatedAuthContext(user=user))
|
||||||
|
return {
|
||||||
|
'iss': iss or instance_keys.service_name,
|
||||||
|
'aud': audience,
|
||||||
|
'nbf': nbf if nbf is not None else int(time.time()),
|
||||||
|
'iat': iat if iat is not None else int(time.time()),
|
||||||
|
'exp': exp if exp is not None else int(time.time() + TOKEN_VALIDITY_LIFETIME_S),
|
||||||
|
'sub': subject,
|
||||||
|
'access': access,
|
||||||
|
'context': context,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _token(token_data, key_id=None, private_key=None, skip_header=False, alg=None):
|
||||||
|
key_id = key_id or instance_keys.local_key_id
|
||||||
|
private_key = private_key or instance_keys.local_private_key
|
||||||
|
|
||||||
|
if alg == "none":
|
||||||
|
private_key = None
|
||||||
|
|
||||||
|
token_headers = {'kid': key_id}
|
||||||
|
|
||||||
|
if skip_header:
|
||||||
|
token_headers = {}
|
||||||
|
|
||||||
|
token_data = jwt.encode(token_data, private_key, alg or 'RS256', headers=token_headers)
|
||||||
|
return 'Bearer {0}'.format(token_data)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_token(token):
|
||||||
|
return identity_from_bearer_token(token)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_accepted_token(initialized_db):
|
||||||
|
token = _token(_token_data())
|
||||||
|
identity = _parse_token(token)
|
||||||
|
assert identity.id == TEST_USER.username, 'should be %s, but was %s' % (TEST_USER.username,
|
||||||
|
identity.id)
|
||||||
|
assert len(identity.provides) == 0
|
||||||
|
|
||||||
|
anon_token = _token(_token_data(user=None))
|
||||||
|
anon_identity = _parse_token(anon_token)
|
||||||
|
assert anon_identity.id == ANONYMOUS_SUB, 'should be %s, but was %s' % (ANONYMOUS_SUB,
|
||||||
|
anon_identity.id)
|
||||||
|
assert len(identity.provides) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('access', [
|
||||||
|
(_access(actions=['pull', 'push'])),
|
||||||
|
(_access(actions=['pull', '*'])),
|
||||||
|
(_access(actions=['*', 'push'])),
|
||||||
|
(_access(actions=['*'])),
|
||||||
|
(_access(actions=['pull', '*', 'push'])),])
|
||||||
|
def test_token_with_access(access, initialized_db):
|
||||||
|
token = _token(_token_data(access=access))
|
||||||
|
identity = _parse_token(token)
|
||||||
|
assert identity.id == TEST_USER.username, 'should be %s, but was %s' % (TEST_USER.username,
|
||||||
|
identity.id)
|
||||||
|
assert len(identity.provides) == 1
|
||||||
|
|
||||||
|
role = list(identity.provides)[0][3]
|
||||||
|
if "*" in access[0]['actions']:
|
||||||
|
assert role == 'admin'
|
||||||
|
elif "push" in access[0]['actions']:
|
||||||
|
assert role == 'write'
|
||||||
|
elif "pull" in access[0]['actions']:
|
||||||
|
assert role == 'read'
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('token', [
|
||||||
|
pytest.param(_token(
|
||||||
|
_token_data(access=[{
|
||||||
|
'toipe': 'repository',
|
||||||
|
'namesies': 'somens/somerepo',
|
||||||
|
'akshuns': ['pull', 'push', '*']}])), id='bad access'),
|
||||||
|
pytest.param(_token(_token_data(audience='someotherapp')), id='bad aud'),
|
||||||
|
pytest.param(_token(_delete_field(_token_data(), 'aud')), id='no aud'),
|
||||||
|
pytest.param(_token(_token_data(nbf=int(time.time()) + 600)), id='future nbf'),
|
||||||
|
pytest.param(_token(_delete_field(_token_data(), 'nbf')), id='no nbf'),
|
||||||
|
pytest.param(_token(_token_data(iat=int(time.time()) + 600)), id='future iat'),
|
||||||
|
pytest.param(_token(_delete_field(_token_data(), 'iat')), id='no iat'),
|
||||||
|
pytest.param(_token(_token_data(exp=int(time.time()) + MAX_SIGNED_S * 2)), id='exp too long'),
|
||||||
|
pytest.param(_token(_token_data(exp=int(time.time()) - 60)), id='expired'),
|
||||||
|
pytest.param(_token(_delete_field(_token_data(), 'exp')), id='no exp'),
|
||||||
|
pytest.param(_token(_delete_field(_token_data(), 'sub')), id='no sub'),
|
||||||
|
pytest.param(_token(_token_data(iss='badissuer')), id='bad iss'),
|
||||||
|
pytest.param(_token(_delete_field(_token_data(), 'iss')), id='no iss'),
|
||||||
|
pytest.param(_token(_token_data(), skip_header=True), id='no header'),
|
||||||
|
pytest.param(_token(_token_data(), key_id='someunknownkey'), id='bad key'),
|
||||||
|
pytest.param(_token(_token_data(), key_id='kid7'), id='bad key :: kid7'),
|
||||||
|
pytest.param(_token(_token_data(), alg='none', private_key=None), id='none alg'),
|
||||||
|
pytest.param('some random token', id='random token'),
|
||||||
|
pytest.param('Bearer: sometokenhere', id='extra bearer'),
|
||||||
|
pytest.param('\nBearer: dGVzdA', id='leading newline'),
|
||||||
|
])
|
||||||
|
def test_invalid_jwt(token, initialized_db):
|
||||||
|
with pytest.raises(InvalidJWTException):
|
||||||
|
_parse_token(token)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mixing_keys_e2e(initialized_db):
|
||||||
|
token_data = _token_data()
|
||||||
|
|
||||||
|
# Create a new key for testing.
|
||||||
|
p, key = model.service_keys.generate_service_key(instance_keys.service_name, None, kid='newkey',
|
||||||
|
name='newkey', metadata={})
|
||||||
|
private_key = p.exportKey('PEM')
|
||||||
|
|
||||||
|
# Test first with the new valid, but unapproved key.
|
||||||
|
unapproved_key_token = _token(token_data, key_id='newkey', private_key=private_key)
|
||||||
|
with pytest.raises(InvalidJWTException):
|
||||||
|
_parse_token(unapproved_key_token)
|
||||||
|
|
||||||
|
# Approve the key and try again.
|
||||||
|
admin_user = model.user.get_user('devtable')
|
||||||
|
model.service_keys.approve_service_key(key.kid, ServiceKeyApprovalType.SUPERUSER, approver=admin_user)
|
||||||
|
|
||||||
|
valid_token = _token(token_data, key_id='newkey', private_key=private_key)
|
||||||
|
|
||||||
|
identity = _parse_token(valid_token)
|
||||||
|
assert identity.id == TEST_USER.username
|
||||||
|
assert len(identity.provides) == 0
|
||||||
|
|
||||||
|
# Try using a different private key with the existing key ID.
|
||||||
|
bad_private_token = _token(token_data, key_id='newkey',
|
||||||
|
private_key=instance_keys.local_private_key)
|
||||||
|
with pytest.raises(InvalidJWTException):
|
||||||
|
_parse_token(bad_private_token)
|
||||||
|
|
||||||
|
# Try using a different key ID with the existing private key.
|
||||||
|
kid_mismatch_token = _token(token_data, key_id=instance_keys.local_key_id,
|
||||||
|
private_key=private_key)
|
||||||
|
with pytest.raises(InvalidJWTException):
|
||||||
|
_parse_token(kid_mismatch_token)
|
||||||
|
|
||||||
|
# Delete the new key.
|
||||||
|
key.delete_instance(recursive=True)
|
||||||
|
|
||||||
|
# Ensure it still works (via the cache.)
|
||||||
|
deleted_key_token = _token(token_data, key_id='newkey', private_key=private_key)
|
||||||
|
identity = _parse_token(deleted_key_token)
|
||||||
|
assert identity.id == TEST_USER.username
|
||||||
|
assert len(identity.provides) == 0
|
||||||
|
|
||||||
|
# Break the cache.
|
||||||
|
instance_keys.clear_cache()
|
||||||
|
|
||||||
|
# Ensure the key no longer works.
|
||||||
|
with pytest.raises(InvalidJWTException):
|
||||||
|
_parse_token(deleted_key_token)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('token', [
|
||||||
|
u'someunicodetoken✡',
|
||||||
|
u'\xc9\xad\xbd',
|
||||||
|
])
|
||||||
|
def test_unicode_token(token):
|
||||||
|
with pytest.raises(InvalidJWTException):
|
||||||
|
_parse_token(token)
|
||||||
50
auth/test/test_scopes.py
Normal file
50
auth/test/test_scopes.py
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from auth.scopes import (
|
||||||
|
scopes_from_scope_string, validate_scope_string, ALL_SCOPES, is_subset_string)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'scopes_string, expected',
|
||||||
|
[
|
||||||
|
# Valid single scopes.
|
||||||
|
('repo:read', ['repo:read']),
|
||||||
|
('repo:admin', ['repo:admin']),
|
||||||
|
|
||||||
|
# Invalid scopes.
|
||||||
|
('not:valid', []),
|
||||||
|
('repo:admins', []),
|
||||||
|
|
||||||
|
# Valid scope strings.
|
||||||
|
('repo:read repo:admin', ['repo:read', 'repo:admin']),
|
||||||
|
('repo:read,repo:admin', ['repo:read', 'repo:admin']),
|
||||||
|
('repo:read,repo:admin repo:write', ['repo:read', 'repo:admin', 'repo:write']),
|
||||||
|
|
||||||
|
# Partially invalid scopes.
|
||||||
|
('repo:read,not:valid', []),
|
||||||
|
('repo:read repo:admins', []),
|
||||||
|
|
||||||
|
# Invalid scope strings.
|
||||||
|
('repo:read|repo:admin', []),
|
||||||
|
|
||||||
|
# Mixture of delimiters.
|
||||||
|
('repo:read, repo:admin', []),])
|
||||||
|
def test_parsing(scopes_string, expected):
|
||||||
|
expected_scope_set = {ALL_SCOPES[scope_name] for scope_name in expected}
|
||||||
|
parsed_scope_set = scopes_from_scope_string(scopes_string)
|
||||||
|
assert parsed_scope_set == expected_scope_set
|
||||||
|
assert validate_scope_string(scopes_string) == bool(expected)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('superset, subset, result', [
|
||||||
|
('repo:read', 'repo:read', True),
|
||||||
|
('repo:read repo:admin', 'repo:read', True),
|
||||||
|
('repo:read,repo:admin', 'repo:read', True),
|
||||||
|
('repo:read,repo:admin', 'repo:admin', True),
|
||||||
|
('repo:read,repo:admin', 'repo:admin repo:read', True),
|
||||||
|
('', 'repo:read', False),
|
||||||
|
('unknown:tag', 'repo:read', False),
|
||||||
|
('repo:read unknown:tag', 'repo:read', False),
|
||||||
|
('repo:read,unknown:tag', 'repo:read', False),])
|
||||||
|
def test_subset_string(superset, subset, result):
|
||||||
|
assert is_subset_string(superset, subset) == result
|
||||||
32
auth/test/test_signedgrant.py
Normal file
32
auth/test/test_signedgrant.py
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from auth.signedgrant import validate_signed_grant, generate_signed_token, SIGNATURE_PREFIX
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('header, expected_result', [
|
||||||
|
pytest.param('', ValidateResult(AuthKind.signed_grant, missing=True), id='Missing'),
|
||||||
|
pytest.param('somerandomtoken', ValidateResult(AuthKind.signed_grant, missing=True),
|
||||||
|
id='Invalid header'),
|
||||||
|
pytest.param('token somerandomtoken', ValidateResult(AuthKind.signed_grant, missing=True),
|
||||||
|
id='Random Token'),
|
||||||
|
pytest.param('token ' + SIGNATURE_PREFIX + 'foo',
|
||||||
|
ValidateResult(AuthKind.signed_grant,
|
||||||
|
error_message='Signed grant could not be validated'),
|
||||||
|
id='Invalid token'),
|
||||||
|
])
|
||||||
|
def test_token(header, expected_result):
|
||||||
|
assert validate_signed_grant(header) == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_grant():
|
||||||
|
header = 'token ' + generate_signed_token({'a': 'b'}, {'c': 'd'})
|
||||||
|
expected = ValidateResult(AuthKind.signed_grant, signed_data={
|
||||||
|
'grants': {
|
||||||
|
'a': 'b',
|
||||||
|
},
|
||||||
|
'user_context': {
|
||||||
|
'c': 'd'
|
||||||
|
},
|
||||||
|
})
|
||||||
|
assert validate_signed_grant(header) == expected
|
||||||
63
auth/test/test_validateresult.py
Normal file
63
auth/test/test_validateresult.py
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from auth.auth_context import get_authenticated_context
|
||||||
|
from auth.validateresult import AuthKind, ValidateResult
|
||||||
|
from data import model
|
||||||
|
from data.database import AppSpecificAuthToken
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
def get_user():
|
||||||
|
return model.user.get_user('devtable')
|
||||||
|
|
||||||
|
def get_app_specific_token():
|
||||||
|
return AppSpecificAuthToken.get()
|
||||||
|
|
||||||
|
def get_robot():
|
||||||
|
robot, _ = model.user.create_robot('somebot', get_user())
|
||||||
|
return robot
|
||||||
|
|
||||||
|
def get_token():
|
||||||
|
return model.token.create_delegate_token('devtable', 'simple', 'sometoken')
|
||||||
|
|
||||||
|
def get_oauthtoken():
|
||||||
|
user = model.user.get_user('devtable')
|
||||||
|
return list(model.oauth.list_access_tokens_for_user(user))[0]
|
||||||
|
|
||||||
|
def get_signeddata():
|
||||||
|
return {'grants': {'a': 'b'}, 'user_context': {'c': 'd'}}
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('get_entity,entity_kind', [
|
||||||
|
(get_user, 'user'),
|
||||||
|
(get_robot, 'robot'),
|
||||||
|
(get_token, 'token'),
|
||||||
|
(get_oauthtoken, 'oauthtoken'),
|
||||||
|
(get_signeddata, 'signed_data'),
|
||||||
|
(get_app_specific_token, 'appspecifictoken'),
|
||||||
|
])
|
||||||
|
def test_apply_context(get_entity, entity_kind, app):
|
||||||
|
assert get_authenticated_context() is None
|
||||||
|
|
||||||
|
entity = get_entity()
|
||||||
|
args = {}
|
||||||
|
args[entity_kind] = entity
|
||||||
|
|
||||||
|
result = ValidateResult(AuthKind.basic, **args)
|
||||||
|
result.apply_to_context()
|
||||||
|
|
||||||
|
expected_user = entity if entity_kind == 'user' or entity_kind == 'robot' else None
|
||||||
|
if entity_kind == 'oauthtoken':
|
||||||
|
expected_user = entity.authorized_user
|
||||||
|
|
||||||
|
if entity_kind == 'appspecifictoken':
|
||||||
|
expected_user = entity.user
|
||||||
|
|
||||||
|
expected_token = entity if entity_kind == 'token' else None
|
||||||
|
expected_oauth = entity if entity_kind == 'oauthtoken' else None
|
||||||
|
expected_appspecifictoken = entity if entity_kind == 'appspecifictoken' else None
|
||||||
|
expected_grant = entity if entity_kind == 'signed_data' else None
|
||||||
|
|
||||||
|
assert get_authenticated_context().authed_user == expected_user
|
||||||
|
assert get_authenticated_context().token == expected_token
|
||||||
|
assert get_authenticated_context().oauthtoken == expected_oauth
|
||||||
|
assert get_authenticated_context().appspecifictoken == expected_appspecifictoken
|
||||||
|
assert get_authenticated_context().signed_data == expected_grant
|
||||||
56
auth/validateresult.py
Normal file
56
auth/validateresult.py
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
from enum import Enum
|
||||||
|
from auth.auth_context_type import ValidatedAuthContext, ContextEntityKind
|
||||||
|
|
||||||
|
|
||||||
|
class AuthKind(Enum):
|
||||||
|
cookie = 'cookie'
|
||||||
|
basic = 'basic'
|
||||||
|
oauth = 'oauth'
|
||||||
|
signed_grant = 'signed_grant'
|
||||||
|
credentials = 'credentials'
|
||||||
|
|
||||||
|
|
||||||
|
class ValidateResult(object):
|
||||||
|
""" A result of validating auth in one form or another. """
|
||||||
|
def __init__(self, kind, missing=False, user=None, token=None, oauthtoken=None,
|
||||||
|
robot=None, appspecifictoken=None, signed_data=None, error_message=None):
|
||||||
|
self.kind = kind
|
||||||
|
self.missing = missing
|
||||||
|
self.error_message = error_message
|
||||||
|
self.context = ValidatedAuthContext(user=user, token=token, oauthtoken=oauthtoken, robot=robot,
|
||||||
|
appspecifictoken=appspecifictoken, signed_data=signed_data)
|
||||||
|
|
||||||
|
def tuple(self):
|
||||||
|
return (self.kind, self.missing, self.error_message, self.context.tuple())
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return self.tuple() == other.tuple()
|
||||||
|
|
||||||
|
def apply_to_context(self):
|
||||||
|
""" Applies this auth result to the auth context and Flask-Principal. """
|
||||||
|
self.context.apply_to_request_context()
|
||||||
|
|
||||||
|
def with_kind(self, kind):
|
||||||
|
""" Returns a copy of this result, but with the kind replaced. """
|
||||||
|
result = ValidateResult(kind, missing=self.missing, error_message=self.error_message)
|
||||||
|
result.context = self.context
|
||||||
|
return result
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return 'ValidateResult: %s (missing: %s, error: %s)' % (self.kind, self.missing,
|
||||||
|
self.error_message)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def authed_user(self):
|
||||||
|
""" Returns the authenticated user, whether directly, or via an OAuth token. """
|
||||||
|
return self.context.authed_user
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_nonrobot_user(self):
|
||||||
|
""" Returns whether a user (not a robot) was authenticated successfully. """
|
||||||
|
return self.context.has_nonrobot_user
|
||||||
|
|
||||||
|
@property
|
||||||
|
def auth_valid(self):
|
||||||
|
""" Returns whether authentication successfully occurred. """
|
||||||
|
return self.context.entity_kind != ContextEntityKind.anonymous
|
||||||
0
avatars/__init__.py
Normal file
0
avatars/__init__.py
Normal file
115
avatars/avatars.py
Normal file
115
avatars/avatars.py
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
import hashlib
|
||||||
|
import math
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from requests.exceptions import RequestException
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class Avatar(object):
|
||||||
|
def __init__(self, app=None):
|
||||||
|
self.app = app
|
||||||
|
self.state = self._init_app(app)
|
||||||
|
|
||||||
|
def _init_app(self, app):
|
||||||
|
return AVATAR_CLASSES[app.config.get('AVATAR_KIND', 'Gravatar')](
|
||||||
|
app.config['PREFERRED_URL_SCHEME'], app.config['AVATAR_COLORS'], app.config['HTTPCLIENT'])
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
return getattr(self.state, name, None)
|
||||||
|
|
||||||
|
|
||||||
|
class BaseAvatar(object):
|
||||||
|
""" Base class for all avatar implementations. """
|
||||||
|
def __init__(self, preferred_url_scheme, colors, http_client):
|
||||||
|
self.preferred_url_scheme = preferred_url_scheme
|
||||||
|
self.colors = colors
|
||||||
|
self.http_client = http_client
|
||||||
|
|
||||||
|
def get_mail_html(self, name, email_or_id, size=16, kind='user'):
|
||||||
|
""" Returns the full HTML and CSS for viewing the avatar of the given name and email address,
|
||||||
|
with an optional size.
|
||||||
|
"""
|
||||||
|
data = self.get_data(name, email_or_id, kind)
|
||||||
|
url = self._get_url(data['hash'], size) if kind != 'team' else None
|
||||||
|
font_size = size - 6
|
||||||
|
|
||||||
|
if url is not None:
|
||||||
|
# Try to load the gravatar. If we get a non-404 response, then we use it in place of
|
||||||
|
# the CSS avatar.
|
||||||
|
try:
|
||||||
|
response = self.http_client.get(url, timeout=5)
|
||||||
|
if response.status_code == 200:
|
||||||
|
return """<img src="%s" width="%s" height="%s" alt="%s"
|
||||||
|
style="vertical-align: middle;">""" % (url, size, size, kind)
|
||||||
|
except RequestException:
|
||||||
|
logger.exception('Could not retrieve avatar for user %s', name)
|
||||||
|
|
||||||
|
radius = '50%' if kind == 'team' else '0%'
|
||||||
|
letter = 'Ω' if kind == 'team' and data['name'] == 'owners' else data['name'].upper()[0]
|
||||||
|
|
||||||
|
return """
|
||||||
|
<span style="width: %spx; height: %spx; background-color: %s; font-size: %spx;
|
||||||
|
line-height: %spx; margin-left: 2px; margin-right: 2px; display: inline-block;
|
||||||
|
vertical-align: middle; text-align: center; color: white; border-radius: %s">
|
||||||
|
%s
|
||||||
|
</span>
|
||||||
|
""" % (size, size, data['color'], font_size, size, radius, letter)
|
||||||
|
|
||||||
|
def get_data_for_user(self, user):
|
||||||
|
return self.get_data(user.username, user.email, 'robot' if user.robot else 'user')
|
||||||
|
|
||||||
|
def get_data_for_team(self, team):
|
||||||
|
return self.get_data(team.name, team.name, 'team')
|
||||||
|
|
||||||
|
def get_data_for_org(self, org):
|
||||||
|
return self.get_data(org.username, org.email, 'org')
|
||||||
|
|
||||||
|
def get_data_for_external_user(self, external_user):
|
||||||
|
return self.get_data(external_user.username, external_user.email, 'user')
|
||||||
|
|
||||||
|
def get_data(self, name, email_or_id, kind='user'):
|
||||||
|
""" Computes and returns the full data block for the avatar:
|
||||||
|
{
|
||||||
|
'name': name,
|
||||||
|
'hash': The gravatar hash, if any.
|
||||||
|
'color': The color for the avatar
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
colors = self.colors
|
||||||
|
|
||||||
|
# Note: email_or_id may be None if gotten from external auth when email is disabled,
|
||||||
|
# so use the username in that case.
|
||||||
|
username_email_or_id = email_or_id or name
|
||||||
|
hash_value = hashlib.md5(username_email_or_id.strip().lower()).hexdigest()
|
||||||
|
|
||||||
|
byte_count = int(math.ceil(math.log(len(colors), 16)))
|
||||||
|
byte_data = hash_value[0:byte_count]
|
||||||
|
hash_color = colors[int(byte_data, 16) % len(colors)]
|
||||||
|
|
||||||
|
return {
|
||||||
|
'name': name,
|
||||||
|
'hash': hash_value,
|
||||||
|
'color': hash_color,
|
||||||
|
'kind': kind
|
||||||
|
}
|
||||||
|
|
||||||
|
def _get_url(self, hash_value, size):
|
||||||
|
""" Returns the URL for displaying the overlay avatar. """
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class GravatarAvatar(BaseAvatar):
|
||||||
|
""" Avatar system that uses gravatar for generating avatars. """
|
||||||
|
def _get_url(self, hash_value, size=16):
|
||||||
|
return '%s://www.gravatar.com/avatar/%s?d=404&size=%s' % (self.preferred_url_scheme,
|
||||||
|
hash_value, size)
|
||||||
|
|
||||||
|
class LocalAvatar(BaseAvatar):
|
||||||
|
""" Avatar system that uses the local system for generating avatars. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
AVATAR_CLASSES = {
|
||||||
|
'gravatar': GravatarAvatar,
|
||||||
|
'local': LocalAvatar
|
||||||
|
}
|
||||||
3857
bill-of-materials.json
Normal file
3857
bill-of-materials.json
Normal file
File diff suppressed because it is too large
Load diff
131
boot.py
Executable file
131
boot.py
Executable file
|
|
@ -0,0 +1,131 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from urlparse import urlunparse
|
||||||
|
|
||||||
|
from jinja2 import Template
|
||||||
|
from cachetools.func import lru_cache
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import release
|
||||||
|
import os.path
|
||||||
|
|
||||||
|
from app import app
|
||||||
|
from data.model import ServiceKeyDoesNotExist
|
||||||
|
from data.model.release import set_region_release
|
||||||
|
from data.model.service_keys import get_service_key
|
||||||
|
from util.config.database import sync_database_with_config
|
||||||
|
from util.generatepresharedkey import generate_key
|
||||||
|
from _init import CONF_DIR
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def get_audience():
|
||||||
|
audience = app.config.get('JWTPROXY_AUDIENCE')
|
||||||
|
|
||||||
|
if audience:
|
||||||
|
return audience
|
||||||
|
|
||||||
|
scheme = app.config.get('PREFERRED_URL_SCHEME')
|
||||||
|
hostname = app.config.get('SERVER_HOSTNAME')
|
||||||
|
|
||||||
|
# hostname includes port, use that
|
||||||
|
if ':' in hostname:
|
||||||
|
return urlunparse((scheme, hostname, '', '', '', ''))
|
||||||
|
|
||||||
|
# no port, guess based on scheme
|
||||||
|
if scheme == 'https':
|
||||||
|
port = '443'
|
||||||
|
else:
|
||||||
|
port = '80'
|
||||||
|
|
||||||
|
return urlunparse((scheme, hostname + ':' + port, '', '', '', ''))
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_service_key():
|
||||||
|
try:
|
||||||
|
with open(app.config['INSTANCE_SERVICE_KEY_KID_LOCATION']) as f:
|
||||||
|
quay_key_id = f.read()
|
||||||
|
|
||||||
|
try:
|
||||||
|
get_service_key(quay_key_id, approved_only=False)
|
||||||
|
assert os.path.exists(app.config['INSTANCE_SERVICE_KEY_LOCATION'])
|
||||||
|
return quay_key_id
|
||||||
|
except ServiceKeyDoesNotExist:
|
||||||
|
logger.exception('Could not find non-expired existing service key %s; creating a new one',
|
||||||
|
quay_key_id)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Found a valid service key, so exiting.
|
||||||
|
except IOError:
|
||||||
|
logger.exception('Could not load existing service key; creating a new one')
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def setup_jwt_proxy():
|
||||||
|
"""
|
||||||
|
Creates a service key for quay to use in the jwtproxy and generates the JWT proxy configuration.
|
||||||
|
"""
|
||||||
|
if os.path.exists(os.path.join(CONF_DIR, 'jwtproxy_conf.yaml')):
|
||||||
|
# Proxy is already setup. Make sure the service key is still valid.
|
||||||
|
quay_key_id = _verify_service_key()
|
||||||
|
if quay_key_id is not None:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Ensure we have an existing key if in read-only mode.
|
||||||
|
if app.config.get('REGISTRY_STATE', 'normal') == 'readonly':
|
||||||
|
quay_key_id = _verify_service_key()
|
||||||
|
if quay_key_id is None:
|
||||||
|
raise Exception('No valid service key found for read-only registry.')
|
||||||
|
else:
|
||||||
|
# Generate the key for this Quay instance to use.
|
||||||
|
minutes_until_expiration = app.config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120)
|
||||||
|
expiration = datetime.now() + timedelta(minutes=minutes_until_expiration)
|
||||||
|
quay_key, quay_key_id = generate_key(app.config['INSTANCE_SERVICE_KEY_SERVICE'],
|
||||||
|
get_audience(), expiration_date=expiration)
|
||||||
|
|
||||||
|
with open(app.config['INSTANCE_SERVICE_KEY_KID_LOCATION'], mode='w') as f:
|
||||||
|
f.truncate(0)
|
||||||
|
f.write(quay_key_id)
|
||||||
|
|
||||||
|
with open(app.config['INSTANCE_SERVICE_KEY_LOCATION'], mode='w') as f:
|
||||||
|
f.truncate(0)
|
||||||
|
f.write(quay_key.exportKey())
|
||||||
|
|
||||||
|
# Generate the JWT proxy configuration.
|
||||||
|
audience = get_audience()
|
||||||
|
registry = audience + '/keys'
|
||||||
|
security_issuer = app.config.get('SECURITY_SCANNER_ISSUER_NAME', 'security_scanner')
|
||||||
|
|
||||||
|
with open(os.path.join(CONF_DIR, 'jwtproxy_conf.yaml.jnj')) as f:
|
||||||
|
template = Template(f.read())
|
||||||
|
rendered = template.render(
|
||||||
|
conf_dir=CONF_DIR,
|
||||||
|
audience=audience,
|
||||||
|
registry=registry,
|
||||||
|
key_id=quay_key_id,
|
||||||
|
security_issuer=security_issuer,
|
||||||
|
service_key_location=app.config['INSTANCE_SERVICE_KEY_LOCATION'],
|
||||||
|
)
|
||||||
|
|
||||||
|
with open(os.path.join(CONF_DIR, 'jwtproxy_conf.yaml'), 'w') as f:
|
||||||
|
f.write(rendered)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not app.config.get('SETUP_COMPLETE', False):
|
||||||
|
raise Exception('Your configuration bundle is either not mounted or setup has not been completed')
|
||||||
|
|
||||||
|
sync_database_with_config(app.config)
|
||||||
|
setup_jwt_proxy()
|
||||||
|
|
||||||
|
# Record deploy
|
||||||
|
if release.REGION and release.GIT_HEAD:
|
||||||
|
set_region_release(release.SERVICE, release.REGION, release.GIT_HEAD)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
23
build.sh
Executable file
23
build.sh
Executable file
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
if [[ -n "$(git status --porcelain)" ]]; then
|
||||||
|
echo 'dirty build not supported' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# get named head (ex: branch, tag, etc..)
|
||||||
|
NAME="$( git rev-parse --abbrev-ref HEAD )"
|
||||||
|
|
||||||
|
# get 7-character sha
|
||||||
|
SHA=$( git rev-parse --short HEAD )
|
||||||
|
|
||||||
|
# checkout commit so .git/HEAD points to full sha (used in Dockerfile)
|
||||||
|
git checkout $SHA
|
||||||
|
|
||||||
|
REPO=quay.io/quay/quay:$SHA
|
||||||
|
docker build -t $REPO .
|
||||||
|
echo $REPO
|
||||||
|
|
||||||
|
git checkout "$NAME"
|
||||||
2
buildman/MAINTAINERS
Normal file
2
buildman/MAINTAINERS
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
Charlton Austin <charlton.austin@coreos.com> (@charltonaustin)
|
||||||
|
Joseph Schorr <joseph.schorr@coreos.com> (@josephschorr)
|
||||||
0
buildman/__init__.py
Normal file
0
buildman/__init__.py
Normal file
42
buildman/asyncutil.py
Normal file
42
buildman/asyncutil.py
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
from trollius import get_event_loop, coroutine
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_with_threadpool(obj, worker_threads=1):
|
||||||
|
"""
|
||||||
|
Wraps a class in an async executor so that it can be safely used in an event loop like trollius.
|
||||||
|
"""
|
||||||
|
async_executor = ThreadPoolExecutor(worker_threads)
|
||||||
|
return AsyncWrapper(obj, executor=async_executor), async_executor
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncWrapper(object):
|
||||||
|
""" Wrapper class which will transform a syncronous library to one that can be used with
|
||||||
|
trollius coroutines.
|
||||||
|
"""
|
||||||
|
def __init__(self, delegate, loop=None, executor=None):
|
||||||
|
self._loop = loop if loop is not None else get_event_loop()
|
||||||
|
self._delegate = delegate
|
||||||
|
self._executor = executor
|
||||||
|
|
||||||
|
def __getattr__(self, attrib):
|
||||||
|
delegate_attr = getattr(self._delegate, attrib)
|
||||||
|
|
||||||
|
if not callable(delegate_attr):
|
||||||
|
return delegate_attr
|
||||||
|
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
""" Wraps the delegate_attr with primitives that will transform sync calls to ones shelled
|
||||||
|
out to a thread pool.
|
||||||
|
"""
|
||||||
|
callable_delegate_attr = partial(delegate_attr, *args, **kwargs)
|
||||||
|
return self._loop.run_in_executor(self._executor, callable_delegate_attr)
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def __call__(self, *args, **kwargs):
|
||||||
|
callable_delegate_attr = partial(self._delegate, *args, **kwargs)
|
||||||
|
return self._loop.run_in_executor(self._executor, callable_delegate_attr)
|
||||||
97
buildman/builder.py
Normal file
97
buildman/builder.py
Normal file
|
|
@ -0,0 +1,97 @@
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import socket
|
||||||
|
|
||||||
|
import features
|
||||||
|
|
||||||
|
from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
|
||||||
|
from util.log import logfile_path
|
||||||
|
|
||||||
|
from buildman.manager.enterprise import EnterpriseManager
|
||||||
|
from buildman.manager.ephemeral import EphemeralBuilderManager
|
||||||
|
from buildman.server import BuilderServer
|
||||||
|
|
||||||
|
from trollius import SSLContext
|
||||||
|
from raven.handlers.logging import SentryHandler
|
||||||
|
from raven.conf import setup_logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
BUILD_MANAGERS = {
|
||||||
|
'enterprise': EnterpriseManager,
|
||||||
|
'ephemeral': EphemeralBuilderManager,
|
||||||
|
}
|
||||||
|
|
||||||
|
EXTERNALLY_MANAGED = 'external'
|
||||||
|
|
||||||
|
DEFAULT_WEBSOCKET_PORT = 8787
|
||||||
|
DEFAULT_CONTROLLER_PORT = 8686
|
||||||
|
|
||||||
|
LOG_FORMAT = "%(asctime)s [%(process)d] [%(levelname)s] [%(name)s] %(message)s"
|
||||||
|
|
||||||
|
def run_build_manager():
|
||||||
|
if not features.BUILD_SUPPORT:
|
||||||
|
logger.debug('Building is disabled. Please enable the feature flag')
|
||||||
|
while True:
|
||||||
|
time.sleep(1000)
|
||||||
|
return
|
||||||
|
|
||||||
|
if app.config.get('REGISTRY_STATE', 'normal') == 'readonly':
|
||||||
|
logger.debug('Building is disabled while in read-only mode.')
|
||||||
|
while True:
|
||||||
|
time.sleep(1000)
|
||||||
|
return
|
||||||
|
|
||||||
|
build_manager_config = app.config.get('BUILD_MANAGER')
|
||||||
|
if build_manager_config is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
# If the build system is externally managed, then we just sleep this process.
|
||||||
|
if build_manager_config[0] == EXTERNALLY_MANAGED:
|
||||||
|
logger.debug('Builds are externally managed.')
|
||||||
|
while True:
|
||||||
|
time.sleep(1000)
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.debug('Asking to start build manager with lifecycle "%s"', build_manager_config[0])
|
||||||
|
manager_klass = BUILD_MANAGERS.get(build_manager_config[0])
|
||||||
|
if manager_klass is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
manager_hostname = os.environ.get('BUILDMAN_HOSTNAME',
|
||||||
|
app.config.get('BUILDMAN_HOSTNAME',
|
||||||
|
app.config['SERVER_HOSTNAME']))
|
||||||
|
websocket_port = int(os.environ.get('BUILDMAN_WEBSOCKET_PORT',
|
||||||
|
app.config.get('BUILDMAN_WEBSOCKET_PORT',
|
||||||
|
DEFAULT_WEBSOCKET_PORT)))
|
||||||
|
controller_port = int(os.environ.get('BUILDMAN_CONTROLLER_PORT',
|
||||||
|
app.config.get('BUILDMAN_CONTROLLER_PORT',
|
||||||
|
DEFAULT_CONTROLLER_PORT)))
|
||||||
|
|
||||||
|
logger.debug('Will pass buildman hostname %s to builders for websocket connection',
|
||||||
|
manager_hostname)
|
||||||
|
|
||||||
|
logger.debug('Starting build manager with lifecycle "%s"', build_manager_config[0])
|
||||||
|
ssl_context = None
|
||||||
|
if os.environ.get('SSL_CONFIG'):
|
||||||
|
logger.debug('Loading SSL cert and key')
|
||||||
|
ssl_context = SSLContext()
|
||||||
|
ssl_context.load_cert_chain(os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.cert'),
|
||||||
|
os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.key'))
|
||||||
|
|
||||||
|
server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
|
||||||
|
user_files, manager_klass, build_manager_config[1], manager_hostname)
|
||||||
|
server.run('0.0.0.0', websocket_port, controller_port, ssl=ssl_context)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
logging.config.fileConfig(logfile_path(debug=True), disable_existing_loggers=False)
|
||||||
|
logging.getLogger('peewee').setLevel(logging.WARN)
|
||||||
|
logging.getLogger('boto').setLevel(logging.WARN)
|
||||||
|
|
||||||
|
if app.config.get('EXCEPTION_LOG_TYPE', 'FakeSentry') == 'Sentry':
|
||||||
|
buildman_name = '%s:buildman' % socket.gethostname()
|
||||||
|
setup_logging(SentryHandler(app.config.get('SENTRY_DSN', ''), name=buildman_name,
|
||||||
|
level=logging.ERROR))
|
||||||
|
|
||||||
|
run_build_manager()
|
||||||
0
buildman/component/__init__.py
Normal file
0
buildman/component/__init__.py
Normal file
13
buildman/component/basecomponent.py
Normal file
13
buildman/component/basecomponent.py
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
from autobahn.asyncio.wamp import ApplicationSession
|
||||||
|
|
||||||
|
class BaseComponent(ApplicationSession):
|
||||||
|
""" Base class for all registered component sessions in the server. """
|
||||||
|
def __init__(self, config, **kwargs):
|
||||||
|
ApplicationSession.__init__(self, config)
|
||||||
|
self.server = None
|
||||||
|
self.parent_manager = None
|
||||||
|
self.build_logs = None
|
||||||
|
self.user_files = None
|
||||||
|
|
||||||
|
def kind(self):
|
||||||
|
raise NotImplementedError
|
||||||
539
buildman/component/buildcomponent.py
Normal file
539
buildman/component/buildcomponent.py
Normal file
|
|
@ -0,0 +1,539 @@
|
||||||
|
import datetime
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
import json
|
||||||
|
import trollius
|
||||||
|
|
||||||
|
from autobahn.wamp.exception import ApplicationError
|
||||||
|
from trollius import From, Return
|
||||||
|
|
||||||
|
from active_migration import ActiveDataMigration, ERTMigrationFlags
|
||||||
|
from buildman.server import BuildJobResult
|
||||||
|
from buildman.component.basecomponent import BaseComponent
|
||||||
|
from buildman.component.buildparse import extract_current_step
|
||||||
|
from buildman.jobutil.buildjob import BuildJobLoadException
|
||||||
|
from buildman.jobutil.buildstatus import StatusHandler
|
||||||
|
from buildman.jobutil.workererror import WorkerError
|
||||||
|
|
||||||
|
from app import app
|
||||||
|
from data.database import BUILD_PHASE, UseThenDisconnect
|
||||||
|
from data.model import InvalidRepositoryBuildException
|
||||||
|
from data.registry_model import registry_model
|
||||||
|
from util import slash_join
|
||||||
|
|
||||||
|
HEARTBEAT_DELTA = datetime.timedelta(seconds=60)
|
||||||
|
BUILD_HEARTBEAT_DELAY = datetime.timedelta(seconds=30)
|
||||||
|
HEARTBEAT_TIMEOUT = 10
|
||||||
|
INITIAL_TIMEOUT = 25
|
||||||
|
|
||||||
|
SUPPORTED_WORKER_VERSIONS = ['0.3']
|
||||||
|
|
||||||
|
# Label which marks a manifest with its source build ID.
|
||||||
|
INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid'
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class ComponentStatus(object):
|
||||||
|
""" ComponentStatus represents the possible states of a component. """
|
||||||
|
JOINING = 'joining'
|
||||||
|
WAITING = 'waiting'
|
||||||
|
RUNNING = 'running'
|
||||||
|
BUILDING = 'building'
|
||||||
|
TIMED_OUT = 'timeout'
|
||||||
|
|
||||||
|
class BuildComponent(BaseComponent):
|
||||||
|
""" An application session component which conducts one (or more) builds. """
|
||||||
|
def __init__(self, config, realm=None, token=None, **kwargs):
|
||||||
|
self.expected_token = token
|
||||||
|
self.builder_realm = realm
|
||||||
|
|
||||||
|
self.parent_manager = None
|
||||||
|
self.registry_hostname = None
|
||||||
|
|
||||||
|
self._component_status = ComponentStatus.JOINING
|
||||||
|
self._last_heartbeat = None
|
||||||
|
self._current_job = None
|
||||||
|
self._build_status = None
|
||||||
|
self._image_info = None
|
||||||
|
self._worker_version = None
|
||||||
|
|
||||||
|
BaseComponent.__init__(self, config, **kwargs)
|
||||||
|
|
||||||
|
def kind(self):
|
||||||
|
return 'builder'
|
||||||
|
|
||||||
|
def onConnect(self):
|
||||||
|
self.join(self.builder_realm)
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def onJoin(self, details):
|
||||||
|
logger.debug('Registering methods and listeners for component %s', self.builder_realm)
|
||||||
|
yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
|
||||||
|
yield From(self.register(self._determine_cache_tag, u'io.quay.buildworker.determinecachetag'))
|
||||||
|
yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
|
||||||
|
yield From(self.register(self._on_log_message, u'io.quay.builder.logmessagesynchronously'))
|
||||||
|
|
||||||
|
yield From(self.subscribe(self._on_heartbeat, u'io.quay.builder.heartbeat'))
|
||||||
|
|
||||||
|
yield From(self._set_status(ComponentStatus.WAITING))
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def start_build(self, build_job):
|
||||||
|
""" Starts a build. """
|
||||||
|
if self._component_status not in (ComponentStatus.WAITING, ComponentStatus.RUNNING):
|
||||||
|
logger.debug('Could not start build for component %s (build %s, worker version: %s): %s',
|
||||||
|
self.builder_realm, build_job.repo_build.uuid, self._worker_version,
|
||||||
|
self._component_status)
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
logger.debug('Starting build for component %s (build %s, worker version: %s)',
|
||||||
|
self.builder_realm, build_job.repo_build.uuid, self._worker_version)
|
||||||
|
|
||||||
|
self._current_job = build_job
|
||||||
|
self._build_status = StatusHandler(self.build_logs, build_job.repo_build.uuid)
|
||||||
|
self._image_info = {}
|
||||||
|
|
||||||
|
yield From(self._set_status(ComponentStatus.BUILDING))
|
||||||
|
|
||||||
|
# Send the notification that the build has started.
|
||||||
|
build_job.send_notification('build_start')
|
||||||
|
|
||||||
|
# Parse the build configuration.
|
||||||
|
try:
|
||||||
|
build_config = build_job.build_config
|
||||||
|
except BuildJobLoadException as irbe:
|
||||||
|
yield From(self._build_failure('Could not load build job information', irbe))
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
base_image_information = {}
|
||||||
|
|
||||||
|
# Add the pull robot information, if any.
|
||||||
|
if build_job.pull_credentials:
|
||||||
|
base_image_information['username'] = build_job.pull_credentials.get('username', '')
|
||||||
|
base_image_information['password'] = build_job.pull_credentials.get('password', '')
|
||||||
|
|
||||||
|
# Retrieve the repository's fully qualified name.
|
||||||
|
repo = build_job.repo_build.repository
|
||||||
|
repository_name = repo.namespace_user.username + '/' + repo.name
|
||||||
|
|
||||||
|
# Parse the build queue item into build arguments.
|
||||||
|
# build_package: URL to the build package to download and untar/unzip.
|
||||||
|
# defaults to empty string to avoid requiring a pointer on the builder.
|
||||||
|
# sub_directory: The location within the build package of the Dockerfile and the build context.
|
||||||
|
# repository: The repository for which this build is occurring.
|
||||||
|
# registry: The registry for which this build is occuring (e.g. 'quay.io').
|
||||||
|
# pull_token: The token to use when pulling the cache for building.
|
||||||
|
# push_token: The token to use to push the built image.
|
||||||
|
# tag_names: The name(s) of the tag(s) for the newly built image.
|
||||||
|
# base_image: The image name and credentials to use to conduct the base image pull.
|
||||||
|
# username: The username for pulling the base image (if any).
|
||||||
|
# password: The password for pulling the base image (if any).
|
||||||
|
context, dockerfile_path = self.extract_dockerfile_args(build_config)
|
||||||
|
build_arguments = {
|
||||||
|
'build_package': build_job.get_build_package_url(self.user_files),
|
||||||
|
'context': context,
|
||||||
|
'dockerfile_path': dockerfile_path,
|
||||||
|
'repository': repository_name,
|
||||||
|
'registry': self.registry_hostname,
|
||||||
|
'pull_token': build_job.repo_build.access_token.get_code(),
|
||||||
|
'push_token': build_job.repo_build.access_token.get_code(),
|
||||||
|
'tag_names': build_config.get('docker_tags', ['latest']),
|
||||||
|
'base_image': base_image_information,
|
||||||
|
}
|
||||||
|
|
||||||
|
# If the trigger has a private key, it's using git, thus we should add
|
||||||
|
# git data to the build args.
|
||||||
|
# url: url used to clone the git repository
|
||||||
|
# sha: the sha1 identifier of the commit to check out
|
||||||
|
# private_key: the key used to get read access to the git repository
|
||||||
|
|
||||||
|
# TODO(remove-unenc): Remove legacy field.
|
||||||
|
private_key = None
|
||||||
|
if build_job.repo_build.trigger is not None and \
|
||||||
|
build_job.repo_build.trigger.secure_private_key is not None:
|
||||||
|
private_key = build_job.repo_build.trigger.secure_private_key.decrypt()
|
||||||
|
|
||||||
|
if ActiveDataMigration.has_flag(ERTMigrationFlags.READ_OLD_FIELDS) and \
|
||||||
|
private_key is None and \
|
||||||
|
build_job.repo_build.trigger is not None:
|
||||||
|
private_key = build_job.repo_build.trigger.private_key
|
||||||
|
|
||||||
|
if private_key is not None:
|
||||||
|
build_arguments['git'] = {
|
||||||
|
'url': build_config['trigger_metadata'].get('git_url', ''),
|
||||||
|
'sha': BuildComponent._commit_sha(build_config),
|
||||||
|
'private_key': private_key or '',
|
||||||
|
}
|
||||||
|
|
||||||
|
# If the build args have no buildpack, mark it as a failure before sending
|
||||||
|
# it to a builder instance.
|
||||||
|
if not build_arguments['build_package'] and not build_arguments['git']:
|
||||||
|
logger.error('%s: insufficient build args: %s',
|
||||||
|
self._current_job.repo_build.uuid, build_arguments)
|
||||||
|
yield From(self._build_failure('Insufficient build arguments. No buildpack available.'))
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
# Invoke the build.
|
||||||
|
logger.debug('Invoking build: %s', self.builder_realm)
|
||||||
|
logger.debug('With Arguments: %s', build_arguments)
|
||||||
|
|
||||||
|
def build_complete_callback(result):
|
||||||
|
""" This function is used to execute a coroutine as the callback. """
|
||||||
|
trollius.ensure_future(self._build_complete(result))
|
||||||
|
|
||||||
|
self.call("io.quay.builder.build", **build_arguments).add_done_callback(build_complete_callback)
|
||||||
|
|
||||||
|
# Set the heartbeat for the future. If the builder never receives the build call,
|
||||||
|
# then this will cause a timeout after 30 seconds. We know the builder has registered
|
||||||
|
# by this point, so it makes sense to have a timeout.
|
||||||
|
self._last_heartbeat = datetime.datetime.utcnow() + BUILD_HEARTBEAT_DELAY
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def extract_dockerfile_args(build_config):
|
||||||
|
dockerfile_path = build_config.get('build_subdir', '')
|
||||||
|
context = build_config.get('context', '')
|
||||||
|
if not (dockerfile_path == '' or context == ''):
|
||||||
|
# This should not happen and can be removed when we centralize validating build_config
|
||||||
|
dockerfile_abspath = slash_join('', dockerfile_path)
|
||||||
|
if ".." in os.path.relpath(dockerfile_abspath, context):
|
||||||
|
return os.path.split(dockerfile_path)
|
||||||
|
dockerfile_path = os.path.relpath(dockerfile_abspath, context)
|
||||||
|
|
||||||
|
return context, dockerfile_path
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _commit_sha(build_config):
|
||||||
|
""" Determines whether the metadata is using an old schema or not and returns the commit. """
|
||||||
|
commit_sha = build_config['trigger_metadata'].get('commit', '')
|
||||||
|
old_commit_sha = build_config['trigger_metadata'].get('commit_sha', '')
|
||||||
|
return commit_sha or old_commit_sha
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def name_and_path(subdir):
|
||||||
|
""" Returns the dockerfile path and name """
|
||||||
|
if subdir.endswith("/"):
|
||||||
|
subdir += "Dockerfile"
|
||||||
|
elif not subdir.endswith("Dockerfile"):
|
||||||
|
subdir += "/Dockerfile"
|
||||||
|
return os.path.split(subdir)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _total_completion(statuses, total_images):
|
||||||
|
""" Returns the current amount completion relative to the total completion of a build. """
|
||||||
|
percentage_with_sizes = float(len(statuses.values())) / total_images
|
||||||
|
sent_bytes = sum([status['current'] for status in statuses.values()])
|
||||||
|
total_bytes = sum([status['total'] for status in statuses.values()])
|
||||||
|
return float(sent_bytes) / total_bytes * percentage_with_sizes
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _process_pushpull_status(status_dict, current_phase, docker_data, images):
|
||||||
|
""" Processes the status of a push or pull by updating the provided status_dict and images. """
|
||||||
|
if not docker_data:
|
||||||
|
return
|
||||||
|
|
||||||
|
num_images = 0
|
||||||
|
status_completion_key = ''
|
||||||
|
|
||||||
|
if current_phase == 'pushing':
|
||||||
|
status_completion_key = 'push_completion'
|
||||||
|
num_images = status_dict['total_commands']
|
||||||
|
elif current_phase == 'pulling':
|
||||||
|
status_completion_key = 'pull_completion'
|
||||||
|
elif current_phase == 'priming-cache':
|
||||||
|
status_completion_key = 'cache_completion'
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
|
||||||
|
if 'progressDetail' in docker_data and 'id' in docker_data:
|
||||||
|
image_id = docker_data['id']
|
||||||
|
detail = docker_data['progressDetail']
|
||||||
|
|
||||||
|
if 'current' in detail and 'total' in detail:
|
||||||
|
images[image_id] = detail
|
||||||
|
status_dict[status_completion_key] = \
|
||||||
|
BuildComponent._total_completion(images, max(len(images), num_images))
|
||||||
|
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _on_log_message(self, phase, json_data):
|
||||||
|
""" Tails log messages and updates the build status. """
|
||||||
|
# Update the heartbeat.
|
||||||
|
self._last_heartbeat = datetime.datetime.utcnow()
|
||||||
|
|
||||||
|
# Parse any of the JSON data logged.
|
||||||
|
log_data = {}
|
||||||
|
if json_data:
|
||||||
|
try:
|
||||||
|
log_data = json.loads(json_data)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Extract the current status message (if any).
|
||||||
|
fully_unwrapped = ''
|
||||||
|
keys_to_extract = ['error', 'status', 'stream']
|
||||||
|
for key in keys_to_extract:
|
||||||
|
if key in log_data:
|
||||||
|
fully_unwrapped = log_data[key]
|
||||||
|
break
|
||||||
|
|
||||||
|
# Determine if this is a step string.
|
||||||
|
current_step = None
|
||||||
|
current_status_string = str(fully_unwrapped.encode('utf-8'))
|
||||||
|
|
||||||
|
if current_status_string and phase == BUILD_PHASE.BUILDING:
|
||||||
|
current_step = extract_current_step(current_status_string)
|
||||||
|
|
||||||
|
# Parse and update the phase and the status_dict. The status dictionary contains
|
||||||
|
# the pull/push progress, as well as the current step index.
|
||||||
|
with self._build_status as status_dict:
|
||||||
|
try:
|
||||||
|
changed_phase = yield From(self._build_status.set_phase(phase, log_data.get('status_data')))
|
||||||
|
if changed_phase:
|
||||||
|
logger.debug('Build %s has entered a new phase: %s', self.builder_realm, phase)
|
||||||
|
elif self._current_job.repo_build.phase == BUILD_PHASE.CANCELLED:
|
||||||
|
build_id = self._current_job.repo_build.uuid
|
||||||
|
logger.debug('Trying to move cancelled build into phase: %s with id: %s', phase, build_id)
|
||||||
|
raise Return(False)
|
||||||
|
except InvalidRepositoryBuildException:
|
||||||
|
build_id = self._current_job.repo_build.uuid
|
||||||
|
logger.warning('Build %s was not found; repo was probably deleted', build_id)
|
||||||
|
raise Return(False)
|
||||||
|
|
||||||
|
BuildComponent._process_pushpull_status(status_dict, phase, log_data, self._image_info)
|
||||||
|
|
||||||
|
# If the current message represents the beginning of a new step, then update the
|
||||||
|
# current command index.
|
||||||
|
if current_step is not None:
|
||||||
|
status_dict['current_command'] = current_step
|
||||||
|
|
||||||
|
# If the json data contains an error, then something went wrong with a push or pull.
|
||||||
|
if 'error' in log_data:
|
||||||
|
yield From(self._build_status.set_error(log_data['error']))
|
||||||
|
|
||||||
|
if current_step is not None:
|
||||||
|
yield From(self._build_status.set_command(current_status_string))
|
||||||
|
elif phase == BUILD_PHASE.BUILDING:
|
||||||
|
yield From(self._build_status.append_log(current_status_string))
|
||||||
|
raise Return(True)
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _determine_cache_tag(self, command_comments, base_image_name, base_image_tag, base_image_id):
|
||||||
|
with self._build_status as status_dict:
|
||||||
|
status_dict['total_commands'] = len(command_comments) + 1
|
||||||
|
|
||||||
|
logger.debug('Checking cache on realm %s. Base image: %s:%s (%s)', self.builder_realm,
|
||||||
|
base_image_name, base_image_tag, base_image_id)
|
||||||
|
|
||||||
|
tag_found = self._current_job.determine_cached_tag(base_image_id, command_comments)
|
||||||
|
raise Return(tag_found or '')
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _build_failure(self, error_message, exception=None):
|
||||||
|
""" Handles and logs a failed build. """
|
||||||
|
yield From(self._build_status.set_error(error_message, {
|
||||||
|
'internal_error': str(exception) if exception else None
|
||||||
|
}))
|
||||||
|
|
||||||
|
build_id = self._current_job.repo_build.uuid
|
||||||
|
logger.warning('Build %s failed with message: %s', build_id, error_message)
|
||||||
|
|
||||||
|
# Mark that the build has finished (in an error state)
|
||||||
|
yield From(self._build_finished(BuildJobResult.ERROR))
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _build_complete(self, result):
|
||||||
|
""" Wraps up a completed build. Handles any errors and calls self._build_finished. """
|
||||||
|
build_id = self._current_job.repo_build.uuid
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Retrieve the result. This will raise an ApplicationError on any error that occurred.
|
||||||
|
result_value = result.result()
|
||||||
|
kwargs = {}
|
||||||
|
|
||||||
|
# Note: If we are hitting an older builder that didn't return ANY map data, then the result
|
||||||
|
# value will be a bool instead of a proper CallResult object.
|
||||||
|
# Therefore: we have a try-except guard here to ensure we don't hit this pitfall.
|
||||||
|
try:
|
||||||
|
kwargs = result_value.kwresults
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield From(self._build_status.set_phase(BUILD_PHASE.COMPLETE))
|
||||||
|
except InvalidRepositoryBuildException:
|
||||||
|
logger.warning('Build %s was not found; repo was probably deleted', build_id)
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
yield From(self._build_finished(BuildJobResult.COMPLETE))
|
||||||
|
|
||||||
|
# Label the pushed manifests with the build metadata.
|
||||||
|
manifest_digests = kwargs.get('digests') or []
|
||||||
|
repository = registry_model.lookup_repository(self._current_job.namespace,
|
||||||
|
self._current_job.repo_name)
|
||||||
|
if repository is not None:
|
||||||
|
for digest in manifest_digests:
|
||||||
|
with UseThenDisconnect(app.config):
|
||||||
|
manifest = registry_model.lookup_manifest_by_digest(repository, digest,
|
||||||
|
require_available=True)
|
||||||
|
if manifest is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
registry_model.create_manifest_label(manifest, INTERNAL_LABEL_BUILD_UUID,
|
||||||
|
build_id, 'internal', 'text/plain')
|
||||||
|
|
||||||
|
# Send the notification that the build has completed successfully.
|
||||||
|
self._current_job.send_notification('build_success',
|
||||||
|
image_id=kwargs.get('image_id'),
|
||||||
|
manifest_digests=manifest_digests)
|
||||||
|
except ApplicationError as aex:
|
||||||
|
worker_error = WorkerError(aex.error, aex.kwargs.get('base_error'))
|
||||||
|
|
||||||
|
# Write the error to the log.
|
||||||
|
yield From(self._build_status.set_error(worker_error.public_message(),
|
||||||
|
worker_error.extra_data(),
|
||||||
|
internal_error=worker_error.is_internal_error(),
|
||||||
|
requeued=self._current_job.has_retries_remaining()))
|
||||||
|
|
||||||
|
# Send the notification that the build has failed.
|
||||||
|
self._current_job.send_notification('build_failure',
|
||||||
|
error_message=worker_error.public_message())
|
||||||
|
|
||||||
|
# Mark the build as completed.
|
||||||
|
if worker_error.is_internal_error():
|
||||||
|
logger.exception('[BUILD INTERNAL ERROR: Remote] Build ID: %s: %s', build_id,
|
||||||
|
worker_error.public_message())
|
||||||
|
yield From(self._build_finished(BuildJobResult.INCOMPLETE))
|
||||||
|
else:
|
||||||
|
logger.debug('Got remote failure exception for build %s: %s', build_id, aex)
|
||||||
|
yield From(self._build_finished(BuildJobResult.ERROR))
|
||||||
|
|
||||||
|
# Remove the current job.
|
||||||
|
self._current_job = None
|
||||||
|
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _build_finished(self, job_status):
|
||||||
|
""" Alerts the parent that a build has completed and sets the status back to running. """
|
||||||
|
yield From(self.parent_manager.job_completed(self._current_job, job_status, self))
|
||||||
|
|
||||||
|
# Set the component back to a running state.
|
||||||
|
yield From(self._set_status(ComponentStatus.RUNNING))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ping():
|
||||||
|
""" Ping pong. """
|
||||||
|
return 'pong'
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _on_ready(self, token, version):
|
||||||
|
logger.debug('On ready called (token "%s")', token)
|
||||||
|
self._worker_version = version
|
||||||
|
|
||||||
|
if not version in SUPPORTED_WORKER_VERSIONS:
|
||||||
|
logger.warning('Build component (token "%s") is running an out-of-date version: %s', token,
|
||||||
|
version)
|
||||||
|
raise Return(False)
|
||||||
|
|
||||||
|
if self._component_status != ComponentStatus.WAITING:
|
||||||
|
logger.warning('Build component (token "%s") is already connected', self.expected_token)
|
||||||
|
raise Return(False)
|
||||||
|
|
||||||
|
if token != self.expected_token:
|
||||||
|
logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token,
|
||||||
|
token)
|
||||||
|
raise Return(False)
|
||||||
|
|
||||||
|
yield From(self._set_status(ComponentStatus.RUNNING))
|
||||||
|
|
||||||
|
# Start the heartbeat check and updating loop.
|
||||||
|
loop = trollius.get_event_loop()
|
||||||
|
loop.create_task(self._heartbeat())
|
||||||
|
logger.debug('Build worker %s is connected and ready', self.builder_realm)
|
||||||
|
raise Return(True)
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _set_status(self, phase):
|
||||||
|
if phase == ComponentStatus.RUNNING:
|
||||||
|
yield From(self.parent_manager.build_component_ready(self))
|
||||||
|
|
||||||
|
self._component_status = phase
|
||||||
|
|
||||||
|
def _on_heartbeat(self):
|
||||||
|
""" Updates the last known heartbeat. """
|
||||||
|
if self._component_status == ComponentStatus.TIMED_OUT:
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.debug('Got heartbeat on realm %s', self.builder_realm)
|
||||||
|
self._last_heartbeat = datetime.datetime.utcnow()
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _heartbeat(self):
|
||||||
|
""" Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
|
||||||
|
and updating the heartbeat in the build status dictionary (if applicable). This allows
|
||||||
|
the build system to catch crashes from either end.
|
||||||
|
"""
|
||||||
|
yield From(trollius.sleep(INITIAL_TIMEOUT))
|
||||||
|
|
||||||
|
while True:
|
||||||
|
# If the component is no longer running or actively building, nothing more to do.
|
||||||
|
if (self._component_status != ComponentStatus.RUNNING and
|
||||||
|
self._component_status != ComponentStatus.BUILDING):
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
# If there is an active build, write the heartbeat to its status.
|
||||||
|
if self._build_status is not None:
|
||||||
|
with self._build_status as status_dict:
|
||||||
|
status_dict['heartbeat'] = int(time.time())
|
||||||
|
|
||||||
|
# Mark the build item.
|
||||||
|
current_job = self._current_job
|
||||||
|
if current_job is not None:
|
||||||
|
yield From(self.parent_manager.job_heartbeat(current_job))
|
||||||
|
|
||||||
|
# Check the heartbeat from the worker.
|
||||||
|
logger.debug('Checking heartbeat on realm %s', self.builder_realm)
|
||||||
|
if (self._last_heartbeat and
|
||||||
|
self._last_heartbeat < datetime.datetime.utcnow() - HEARTBEAT_DELTA):
|
||||||
|
logger.debug('Heartbeat on realm %s has expired: %s', self.builder_realm,
|
||||||
|
self._last_heartbeat)
|
||||||
|
|
||||||
|
yield From(self._timeout())
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
logger.debug('Heartbeat on realm %s is valid: %s (%s).', self.builder_realm,
|
||||||
|
self._last_heartbeat, self._component_status)
|
||||||
|
|
||||||
|
yield From(trollius.sleep(HEARTBEAT_TIMEOUT))
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _timeout(self):
|
||||||
|
if self._component_status == ComponentStatus.TIMED_OUT:
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
yield From(self._set_status(ComponentStatus.TIMED_OUT))
|
||||||
|
logger.warning('Build component with realm %s has timed out', self.builder_realm)
|
||||||
|
|
||||||
|
# If we still have a running job, then it has not completed and we need to tell the parent
|
||||||
|
# manager.
|
||||||
|
if self._current_job is not None:
|
||||||
|
yield From(self._build_status.set_error('Build worker timed out', internal_error=True,
|
||||||
|
requeued=self._current_job.has_retries_remaining()))
|
||||||
|
|
||||||
|
build_id = self._current_job.build_uuid
|
||||||
|
logger.error('[BUILD INTERNAL ERROR: Timeout] Build ID: %s', build_id)
|
||||||
|
yield From(self.parent_manager.job_completed(self._current_job,
|
||||||
|
BuildJobResult.INCOMPLETE,
|
||||||
|
self))
|
||||||
|
|
||||||
|
# Unregister the current component so that it cannot be invoked again.
|
||||||
|
self.parent_manager.build_component_disposed(self, True)
|
||||||
|
|
||||||
|
# Remove the job reference.
|
||||||
|
self._current_job = None
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def cancel_build(self):
|
||||||
|
self.parent_manager.build_component_disposed(self, True)
|
||||||
|
self._current_job = None
|
||||||
|
yield From(self._set_status(ComponentStatus.RUNNING))
|
||||||
15
buildman/component/buildparse.py
Normal file
15
buildman/component/buildparse.py
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
import re
|
||||||
|
|
||||||
|
def extract_current_step(current_status_string):
|
||||||
|
""" Attempts to extract the current step numeric identifier from the given status string. Returns the step
|
||||||
|
number or None if none.
|
||||||
|
"""
|
||||||
|
# Older format: `Step 12 :`
|
||||||
|
# Newer format: `Step 4/13 :`
|
||||||
|
step_increment = re.search(r'Step ([0-9]+)/([0-9]+) :', current_status_string)
|
||||||
|
if step_increment:
|
||||||
|
return int(step_increment.group(1))
|
||||||
|
|
||||||
|
step_increment = re.search(r'Step ([0-9]+) :', current_status_string)
|
||||||
|
if step_increment:
|
||||||
|
return int(step_increment.group(1))
|
||||||
36
buildman/component/test/test_buildcomponent.py
Normal file
36
buildman/component/test/test_buildcomponent.py
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from buildman.component.buildcomponent import BuildComponent
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('input,expected_path,expected_file', [
|
||||||
|
("", "/", "Dockerfile"),
|
||||||
|
("/", "/", "Dockerfile"),
|
||||||
|
("/Dockerfile", "/", "Dockerfile"),
|
||||||
|
("/server.Dockerfile", "/", "server.Dockerfile"),
|
||||||
|
("/somepath", "/somepath", "Dockerfile"),
|
||||||
|
("/somepath/", "/somepath", "Dockerfile"),
|
||||||
|
("/somepath/Dockerfile", "/somepath", "Dockerfile"),
|
||||||
|
("/somepath/server.Dockerfile", "/somepath", "server.Dockerfile"),
|
||||||
|
("/somepath/some_other_path", "/somepath/some_other_path", "Dockerfile"),
|
||||||
|
("/somepath/some_other_path/", "/somepath/some_other_path", "Dockerfile"),
|
||||||
|
("/somepath/some_other_path/Dockerfile", "/somepath/some_other_path", "Dockerfile"),
|
||||||
|
("/somepath/some_other_path/server.Dockerfile", "/somepath/some_other_path", "server.Dockerfile"),
|
||||||
|
])
|
||||||
|
def test_path_is_dockerfile(input, expected_path, expected_file):
|
||||||
|
actual_path, actual_file = BuildComponent.name_and_path(input)
|
||||||
|
assert actual_path == expected_path
|
||||||
|
assert actual_file == expected_file
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('build_config,context,dockerfile_path', [
|
||||||
|
({}, '', ''),
|
||||||
|
({'build_subdir': '/builddir/Dockerfile'}, '', '/builddir/Dockerfile'),
|
||||||
|
({'context': '/builddir'}, '/builddir', ''),
|
||||||
|
({'context': '/builddir', 'build_subdir': '/builddir/Dockerfile'}, '/builddir', 'Dockerfile'),
|
||||||
|
({'context': '/some_other_dir/Dockerfile', 'build_subdir': '/builddir/Dockerfile'}, '/builddir', 'Dockerfile'),
|
||||||
|
({'context': '/', 'build_subdir':'Dockerfile'}, '/', 'Dockerfile')
|
||||||
|
])
|
||||||
|
def test_extract_dockerfile_args(build_config, context, dockerfile_path):
|
||||||
|
actual_context, actual_dockerfile_path = BuildComponent.extract_dockerfile_args(build_config)
|
||||||
|
assert context == actual_context
|
||||||
|
assert dockerfile_path == actual_dockerfile_path
|
||||||
16
buildman/component/test/test_buildparse.py
Normal file
16
buildman/component/test/test_buildparse.py
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from buildman.component.buildparse import extract_current_step
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('input,expected_step', [
|
||||||
|
("", None),
|
||||||
|
("Step a :", None),
|
||||||
|
("Step 1 :", 1),
|
||||||
|
("Step 1 : ", 1),
|
||||||
|
("Step 1/2 : ", 1),
|
||||||
|
("Step 2/17 : ", 2),
|
||||||
|
("Step 4/13 : ARG somearg=foo", 4),
|
||||||
|
])
|
||||||
|
def test_extract_current_step(input, expected_step):
|
||||||
|
assert extract_current_step(input) == expected_step
|
||||||
21
buildman/enums.py
Normal file
21
buildman/enums.py
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
from data.database import BUILD_PHASE
|
||||||
|
|
||||||
|
class BuildJobResult(object):
|
||||||
|
""" Build job result enum """
|
||||||
|
INCOMPLETE = 'incomplete'
|
||||||
|
COMPLETE = 'complete'
|
||||||
|
ERROR = 'error'
|
||||||
|
|
||||||
|
|
||||||
|
class BuildServerStatus(object):
|
||||||
|
""" Build server status enum """
|
||||||
|
STARTING = 'starting'
|
||||||
|
RUNNING = 'running'
|
||||||
|
SHUTDOWN = 'shutting_down'
|
||||||
|
EXCEPTION = 'exception'
|
||||||
|
|
||||||
|
RESULT_PHASES = {
|
||||||
|
BuildJobResult.INCOMPLETE: BUILD_PHASE.INTERNAL_ERROR,
|
||||||
|
BuildJobResult.COMPLETE: BUILD_PHASE.COMPLETE,
|
||||||
|
BuildJobResult.ERROR: BUILD_PHASE.ERROR,
|
||||||
|
}
|
||||||
0
buildman/jobutil/__init__.py
Normal file
0
buildman/jobutil/__init__.py
Normal file
183
buildman/jobutil/buildjob.py
Normal file
183
buildman/jobutil/buildjob.py
Normal file
|
|
@ -0,0 +1,183 @@
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from app import app
|
||||||
|
from cachetools.func import lru_cache
|
||||||
|
from notifications import spawn_notification
|
||||||
|
from data import model
|
||||||
|
from data.registry_model import registry_model
|
||||||
|
from data.registry_model.datatypes import RepositoryReference
|
||||||
|
from data.database import UseThenDisconnect
|
||||||
|
from util.morecollections import AttrDict
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class BuildJobLoadException(Exception):
|
||||||
|
""" Exception raised if a build job could not be instantiated for some reason. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class BuildJob(object):
|
||||||
|
""" Represents a single in-progress build job. """
|
||||||
|
def __init__(self, job_item):
|
||||||
|
self.job_item = job_item
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.job_details = json.loads(job_item.body)
|
||||||
|
self.build_notifier = BuildJobNotifier(self.build_uuid)
|
||||||
|
except ValueError:
|
||||||
|
raise BuildJobLoadException(
|
||||||
|
'Could not parse build queue item config with ID %s' % self.job_details['build_uuid']
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def retries_remaining(self):
|
||||||
|
return self.job_item.retries_remaining
|
||||||
|
|
||||||
|
def has_retries_remaining(self):
|
||||||
|
return self.job_item.retries_remaining > 0
|
||||||
|
|
||||||
|
def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None):
|
||||||
|
self.build_notifier.send_notification(kind, error_message, image_id, manifest_digests)
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _load_repo_build(self):
|
||||||
|
with UseThenDisconnect(app.config):
|
||||||
|
try:
|
||||||
|
return model.build.get_repository_build(self.build_uuid)
|
||||||
|
except model.InvalidRepositoryBuildException:
|
||||||
|
raise BuildJobLoadException(
|
||||||
|
'Could not load repository build with ID %s' % self.build_uuid)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def build_uuid(self):
|
||||||
|
""" Returns the unique UUID for this build job. """
|
||||||
|
return self.job_details['build_uuid']
|
||||||
|
|
||||||
|
@property
|
||||||
|
def namespace(self):
|
||||||
|
""" Returns the namespace under which this build is running. """
|
||||||
|
return self.repo_build.repository.namespace_user.username
|
||||||
|
|
||||||
|
@property
|
||||||
|
def repo_name(self):
|
||||||
|
""" Returns the name of the repository under which this build is running. """
|
||||||
|
return self.repo_build.repository.name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def repo_build(self):
|
||||||
|
return self._load_repo_build()
|
||||||
|
|
||||||
|
def get_build_package_url(self, user_files):
|
||||||
|
""" Returns the URL of the build package for this build, if any or empty string if none. """
|
||||||
|
archive_url = self.build_config.get('archive_url', None)
|
||||||
|
if archive_url:
|
||||||
|
return archive_url
|
||||||
|
|
||||||
|
if not self.repo_build.resource_key:
|
||||||
|
return ''
|
||||||
|
|
||||||
|
return user_files.get_file_url(self.repo_build.resource_key, '127.0.0.1', requires_cors=False)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def pull_credentials(self):
|
||||||
|
""" Returns the pull credentials for this job, or None if none. """
|
||||||
|
return self.job_details.get('pull_credentials')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def build_config(self):
|
||||||
|
try:
|
||||||
|
return json.loads(self.repo_build.job_config)
|
||||||
|
except ValueError:
|
||||||
|
raise BuildJobLoadException(
|
||||||
|
'Could not parse repository build job config with ID %s' % self.job_details['build_uuid']
|
||||||
|
)
|
||||||
|
|
||||||
|
def determine_cached_tag(self, base_image_id=None, cache_comments=None):
|
||||||
|
""" Returns the tag to pull to prime the cache or None if none. """
|
||||||
|
cached_tag = self._determine_cached_tag_by_tag()
|
||||||
|
logger.debug('Determined cached tag %s for %s: %s', cached_tag, base_image_id, cache_comments)
|
||||||
|
return cached_tag
|
||||||
|
|
||||||
|
def _determine_cached_tag_by_tag(self):
|
||||||
|
""" Determines the cached tag by looking for one of the tags being built, and seeing if it
|
||||||
|
exists in the repository. This is a fallback for when no comment information is available.
|
||||||
|
"""
|
||||||
|
with UseThenDisconnect(app.config):
|
||||||
|
tags = self.build_config.get('docker_tags', ['latest'])
|
||||||
|
repository = RepositoryReference.for_repo_obj(self.repo_build.repository)
|
||||||
|
matching_tag = registry_model.find_matching_tag(repository, tags)
|
||||||
|
if matching_tag is not None:
|
||||||
|
return matching_tag.name
|
||||||
|
|
||||||
|
most_recent_tag = registry_model.get_most_recent_tag(repository)
|
||||||
|
if most_recent_tag is not None:
|
||||||
|
return most_recent_tag.name
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class BuildJobNotifier(object):
|
||||||
|
""" A class for sending notifications to a job that only relies on the build_uuid """
|
||||||
|
|
||||||
|
def __init__(self, build_uuid):
|
||||||
|
self.build_uuid = build_uuid
|
||||||
|
|
||||||
|
@property
|
||||||
|
def repo_build(self):
|
||||||
|
return self._load_repo_build()
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _load_repo_build(self):
|
||||||
|
try:
|
||||||
|
return model.build.get_repository_build(self.build_uuid)
|
||||||
|
except model.InvalidRepositoryBuildException:
|
||||||
|
raise BuildJobLoadException(
|
||||||
|
'Could not load repository build with ID %s' % self.build_uuid)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def build_config(self):
|
||||||
|
try:
|
||||||
|
return json.loads(self.repo_build.job_config)
|
||||||
|
except ValueError:
|
||||||
|
raise BuildJobLoadException(
|
||||||
|
'Could not parse repository build job config with ID %s' % self.repo_build.uuid
|
||||||
|
)
|
||||||
|
|
||||||
|
def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None):
|
||||||
|
with UseThenDisconnect(app.config):
|
||||||
|
tags = self.build_config.get('docker_tags', ['latest'])
|
||||||
|
trigger = self.repo_build.trigger
|
||||||
|
if trigger is not None and trigger.id is not None:
|
||||||
|
trigger_kind = trigger.service.name
|
||||||
|
else:
|
||||||
|
trigger_kind = None
|
||||||
|
|
||||||
|
event_data = {
|
||||||
|
'build_id': self.repo_build.uuid,
|
||||||
|
'build_name': self.repo_build.display_name,
|
||||||
|
'docker_tags': tags,
|
||||||
|
'trigger_id': trigger.uuid if trigger is not None else None,
|
||||||
|
'trigger_kind': trigger_kind,
|
||||||
|
'trigger_metadata': self.build_config.get('trigger_metadata', {})
|
||||||
|
}
|
||||||
|
|
||||||
|
if image_id is not None:
|
||||||
|
event_data['image_id'] = image_id
|
||||||
|
|
||||||
|
if manifest_digests:
|
||||||
|
event_data['manifest_digests'] = manifest_digests
|
||||||
|
|
||||||
|
if error_message is not None:
|
||||||
|
event_data['error_message'] = error_message
|
||||||
|
|
||||||
|
# TODO: remove when more endpoints have been converted to using
|
||||||
|
# interfaces
|
||||||
|
repo = AttrDict({
|
||||||
|
'namespace_name': self.repo_build.repository.namespace_user.username,
|
||||||
|
'name': self.repo_build.repository.name,
|
||||||
|
})
|
||||||
|
spawn_notification(repo, kind, event_data,
|
||||||
|
subpage='build/%s' % self.repo_build.uuid,
|
||||||
|
pathargs=['build', self.repo_build.uuid])
|
||||||
88
buildman/jobutil/buildstatus.py
Normal file
88
buildman/jobutil/buildstatus.py
Normal file
|
|
@ -0,0 +1,88 @@
|
||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from redis import RedisError
|
||||||
|
from trollius import From, Return, coroutine
|
||||||
|
|
||||||
|
from data.database import BUILD_PHASE
|
||||||
|
from data import model
|
||||||
|
from buildman.asyncutil import AsyncWrapper
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class StatusHandler(object):
|
||||||
|
""" Context wrapper for writing status to build logs. """
|
||||||
|
|
||||||
|
def __init__(self, build_logs, repository_build_uuid):
|
||||||
|
self._current_phase = None
|
||||||
|
self._current_command = None
|
||||||
|
self._uuid = repository_build_uuid
|
||||||
|
self._build_logs = AsyncWrapper(build_logs)
|
||||||
|
self._sync_build_logs = build_logs
|
||||||
|
self._build_model = AsyncWrapper(model.build)
|
||||||
|
|
||||||
|
self._status = {
|
||||||
|
'total_commands': 0,
|
||||||
|
'current_command': None,
|
||||||
|
'push_completion': 0.0,
|
||||||
|
'pull_completion': 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Write the initial status.
|
||||||
|
self.__exit__(None, None, None)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _append_log_message(self, log_message, log_type=None, log_data=None):
|
||||||
|
log_data = log_data or {}
|
||||||
|
log_data['datetime'] = str(datetime.datetime.now())
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield From(self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data))
|
||||||
|
except RedisError:
|
||||||
|
logger.exception('Could not save build log for build %s: %s', self._uuid, log_message)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def append_log(self, log_message, extra_data=None):
|
||||||
|
if log_message is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
yield From(self._append_log_message(log_message, log_data=extra_data))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def set_command(self, command, extra_data=None):
|
||||||
|
if self._current_command == command:
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
self._current_command = command
|
||||||
|
yield From(self._append_log_message(command, self._build_logs.COMMAND, extra_data))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def set_error(self, error_message, extra_data=None, internal_error=False, requeued=False):
|
||||||
|
error_phase = BUILD_PHASE.INTERNAL_ERROR if internal_error and requeued else BUILD_PHASE.ERROR
|
||||||
|
yield From(self.set_phase(error_phase))
|
||||||
|
|
||||||
|
extra_data = extra_data or {}
|
||||||
|
extra_data['internal_error'] = internal_error
|
||||||
|
yield From(self._append_log_message(error_message, self._build_logs.ERROR, extra_data))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def set_phase(self, phase, extra_data=None):
|
||||||
|
if phase == self._current_phase:
|
||||||
|
raise Return(False)
|
||||||
|
|
||||||
|
self._current_phase = phase
|
||||||
|
yield From(self._append_log_message(phase, self._build_logs.PHASE, extra_data))
|
||||||
|
|
||||||
|
# Update the repository build with the new phase
|
||||||
|
raise Return(self._build_model.update_phase_then_close(self._uuid, phase))
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self._status
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, value, traceback):
|
||||||
|
try:
|
||||||
|
self._sync_build_logs.set_status(self._uuid, self._status)
|
||||||
|
except RedisError:
|
||||||
|
logger.exception('Could not set status of build %s to %s', self._uuid, self._status)
|
||||||
119
buildman/jobutil/workererror.py
Normal file
119
buildman/jobutil/workererror.py
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
class WorkerError(object):
|
||||||
|
""" Helper class which represents errors raised by a build worker. """
|
||||||
|
def __init__(self, error_code, base_message=None):
|
||||||
|
self._error_code = error_code
|
||||||
|
self._base_message = base_message
|
||||||
|
|
||||||
|
self._error_handlers = {
|
||||||
|
'io.quay.builder.buildpackissue': {
|
||||||
|
'message': 'Could not load build package',
|
||||||
|
'is_internal': True,
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.gitfailure': {
|
||||||
|
'message': 'Could not clone git repository',
|
||||||
|
'show_base_error': True,
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.gitcheckout': {
|
||||||
|
'message': 'Could not checkout git ref. If you force pushed recently, ' +
|
||||||
|
'the commit may be missing.',
|
||||||
|
'show_base_error': True,
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.cannotextractbuildpack': {
|
||||||
|
'message': 'Could not extract the contents of the build package'
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.cannotpullforcache': {
|
||||||
|
'message': 'Could not pull cached image',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.dockerfileissue': {
|
||||||
|
'message': 'Could not find or parse Dockerfile',
|
||||||
|
'show_base_error': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.cannotpullbaseimage': {
|
||||||
|
'message': 'Could not pull base image',
|
||||||
|
'show_base_error': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.internalerror': {
|
||||||
|
'message': 'An internal error occurred while building. Please submit a ticket.',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.buildrunerror': {
|
||||||
|
'message': 'Could not start the build process',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.builderror': {
|
||||||
|
'message': 'A build step failed',
|
||||||
|
'show_base_error': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.tagissue': {
|
||||||
|
'message': 'Could not tag built image',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.pushissue': {
|
||||||
|
'message': 'Could not push built image',
|
||||||
|
'show_base_error': True,
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.dockerconnecterror': {
|
||||||
|
'message': 'Could not connect to Docker daemon',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.missingorinvalidargument': {
|
||||||
|
'message': 'Missing required arguments for builder',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.cachelookupissue': {
|
||||||
|
'message': 'Error checking for a cached tag',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.errorduringphasetransition': {
|
||||||
|
'message': 'Error during phase transition. If this problem persists ' +
|
||||||
|
'please contact customer support.',
|
||||||
|
'is_internal': True
|
||||||
|
},
|
||||||
|
|
||||||
|
'io.quay.builder.clientrejectedtransition': {
|
||||||
|
'message': 'Build can not be finished due to user cancellation.',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def is_internal_error(self):
|
||||||
|
handler = self._error_handlers.get(self._error_code)
|
||||||
|
return handler.get('is_internal', False) if handler else True
|
||||||
|
|
||||||
|
def public_message(self):
|
||||||
|
handler = self._error_handlers.get(self._error_code)
|
||||||
|
if not handler:
|
||||||
|
return 'An unknown error occurred'
|
||||||
|
|
||||||
|
message = handler['message']
|
||||||
|
if handler.get('show_base_error', False) and self._base_message:
|
||||||
|
message = message + ': ' + self._base_message
|
||||||
|
|
||||||
|
return message
|
||||||
|
|
||||||
|
def extra_data(self):
|
||||||
|
if self._base_message:
|
||||||
|
return {
|
||||||
|
'base_error': self._base_message,
|
||||||
|
'error_code': self._error_code
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'error_code': self._error_code
|
||||||
|
}
|
||||||
0
buildman/manager/__init__.py
Normal file
0
buildman/manager/__init__.py
Normal file
71
buildman/manager/basemanager.py
Normal file
71
buildman/manager/basemanager.py
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
from trollius import coroutine
|
||||||
|
|
||||||
|
class BaseManager(object):
|
||||||
|
""" Base for all worker managers. """
|
||||||
|
def __init__(self, register_component, unregister_component, job_heartbeat_callback,
|
||||||
|
job_complete_callback, manager_hostname, heartbeat_period_sec):
|
||||||
|
self.register_component = register_component
|
||||||
|
self.unregister_component = unregister_component
|
||||||
|
self.job_heartbeat_callback = job_heartbeat_callback
|
||||||
|
self.job_complete_callback = job_complete_callback
|
||||||
|
self.manager_hostname = manager_hostname
|
||||||
|
self.heartbeat_period_sec = heartbeat_period_sec
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def job_heartbeat(self, build_job):
|
||||||
|
""" Method invoked to tell the manager that a job is still running. This method will be called
|
||||||
|
every few minutes. """
|
||||||
|
self.job_heartbeat_callback(build_job)
|
||||||
|
|
||||||
|
def overall_setup_time(self):
|
||||||
|
""" Returns the number of seconds that the build system should wait before allowing the job
|
||||||
|
to be picked up again after called 'schedule'.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def shutdown(self):
|
||||||
|
""" Indicates that the build controller server is in a shutdown state and that no new jobs
|
||||||
|
or workers should be performed. Existing workers should be cleaned up once their jobs
|
||||||
|
have completed
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def schedule(self, build_job):
|
||||||
|
""" Schedules a queue item to be built. Returns a 2-tuple with (True, None) if the item was
|
||||||
|
properly scheduled and (False, a retry timeout in seconds) if all workers are busy or an
|
||||||
|
error occurs.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def initialize(self, manager_config):
|
||||||
|
""" Runs any initialization code for the manager. Called once the server is in a ready state.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def build_component_ready(self, build_component):
|
||||||
|
""" Method invoked whenever a build component announces itself as ready.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def build_component_disposed(self, build_component, timed_out):
|
||||||
|
""" Method invoked whenever a build component has been disposed. The timed_out boolean indicates
|
||||||
|
whether the component's heartbeat timed out.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def job_completed(self, build_job, job_status, build_component):
|
||||||
|
""" Method invoked once a job_item has completed, in some manner. The job_status will be
|
||||||
|
one of: incomplete, error, complete. Implementations of this method should call coroutine
|
||||||
|
self.job_complete_callback with a status of Incomplete if they wish for the job to be
|
||||||
|
automatically requeued.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def num_workers(self):
|
||||||
|
""" Returns the number of active build workers currently registered. This includes those
|
||||||
|
that are currently busy and awaiting more work.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
27
buildman/manager/buildcanceller.py
Normal file
27
buildman/manager/buildcanceller.py
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from buildman.manager.orchestrator_canceller import OrchestratorCanceller
|
||||||
|
from buildman.manager.noop_canceller import NoopCanceller
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
CANCELLERS = {'ephemeral': OrchestratorCanceller}
|
||||||
|
|
||||||
|
|
||||||
|
class BuildCanceller(object):
|
||||||
|
""" A class to manage cancelling a build """
|
||||||
|
|
||||||
|
def __init__(self, app=None):
|
||||||
|
self.build_manager_config = app.config.get('BUILD_MANAGER')
|
||||||
|
if app is None or self.build_manager_config is None:
|
||||||
|
self.handler = NoopCanceller()
|
||||||
|
else:
|
||||||
|
self.handler = None
|
||||||
|
|
||||||
|
def try_cancel_build(self, uuid):
|
||||||
|
""" A method to kill a running build """
|
||||||
|
if self.handler is None:
|
||||||
|
canceller = CANCELLERS.get(self.build_manager_config[0], NoopCanceller)
|
||||||
|
self.handler = canceller(self.build_manager_config[1])
|
||||||
|
|
||||||
|
return self.handler.try_cancel_build(uuid)
|
||||||
92
buildman/manager/enterprise.py
Normal file
92
buildman/manager/enterprise.py
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from buildman.component.basecomponent import BaseComponent
|
||||||
|
from buildman.component.buildcomponent import BuildComponent
|
||||||
|
from buildman.manager.basemanager import BaseManager
|
||||||
|
|
||||||
|
from trollius import From, Return, coroutine
|
||||||
|
|
||||||
|
REGISTRATION_REALM = 'registration'
|
||||||
|
RETRY_TIMEOUT = 5
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class DynamicRegistrationComponent(BaseComponent):
|
||||||
|
""" Component session that handles dynamic registration of the builder components. """
|
||||||
|
|
||||||
|
def onConnect(self):
|
||||||
|
self.join(REGISTRATION_REALM)
|
||||||
|
|
||||||
|
def onJoin(self, details):
|
||||||
|
logger.debug('Registering registration method')
|
||||||
|
yield From(self.register(self._worker_register, u'io.quay.buildworker.register'))
|
||||||
|
|
||||||
|
def _worker_register(self):
|
||||||
|
realm = self.parent_manager.add_build_component()
|
||||||
|
logger.debug('Registering new build component+worker with realm %s', realm)
|
||||||
|
return realm
|
||||||
|
|
||||||
|
def kind(self):
|
||||||
|
return 'registration'
|
||||||
|
|
||||||
|
|
||||||
|
class EnterpriseManager(BaseManager):
|
||||||
|
""" Build manager implementation for the Enterprise Registry. """
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
self.ready_components = set()
|
||||||
|
self.all_components = set()
|
||||||
|
self.shutting_down = False
|
||||||
|
|
||||||
|
super(EnterpriseManager, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
def initialize(self, manager_config):
|
||||||
|
# Add a component which is used by build workers for dynamic registration. Unlike
|
||||||
|
# production, build workers in enterprise are long-lived and register dynamically.
|
||||||
|
self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent)
|
||||||
|
|
||||||
|
def overall_setup_time(self):
|
||||||
|
# Builders are already registered, so the setup time should be essentially instant. We therefore
|
||||||
|
# only return a minute here.
|
||||||
|
return 60
|
||||||
|
|
||||||
|
def add_build_component(self):
|
||||||
|
""" Adds a new build component for an Enterprise Registry. """
|
||||||
|
# Generate a new unique realm ID for the build worker.
|
||||||
|
realm = str(uuid.uuid4())
|
||||||
|
new_component = self.register_component(realm, BuildComponent, token="")
|
||||||
|
self.all_components.add(new_component)
|
||||||
|
return realm
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def schedule(self, build_job):
|
||||||
|
""" Schedules a build for an Enterprise Registry. """
|
||||||
|
if self.shutting_down or not self.ready_components:
|
||||||
|
raise Return(False, RETRY_TIMEOUT)
|
||||||
|
|
||||||
|
component = self.ready_components.pop()
|
||||||
|
|
||||||
|
yield From(component.start_build(build_job))
|
||||||
|
|
||||||
|
raise Return(True, None)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def build_component_ready(self, build_component):
|
||||||
|
self.ready_components.add(build_component)
|
||||||
|
|
||||||
|
def shutdown(self):
|
||||||
|
self.shutting_down = True
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def job_completed(self, build_job, job_status, build_component):
|
||||||
|
yield From(self.job_complete_callback(build_job, job_status))
|
||||||
|
|
||||||
|
def build_component_disposed(self, build_component, timed_out):
|
||||||
|
self.all_components.remove(build_component)
|
||||||
|
if build_component in self.ready_components:
|
||||||
|
self.ready_components.remove(build_component)
|
||||||
|
|
||||||
|
self.unregister_component(build_component)
|
||||||
|
|
||||||
|
def num_workers(self):
|
||||||
|
return len(self.all_components)
|
||||||
710
buildman/manager/ephemeral.py
Normal file
710
buildman/manager/ephemeral.py
Normal file
|
|
@ -0,0 +1,710 @@
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
import calendar
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
|
||||||
|
from collections import namedtuple
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from six import iteritems
|
||||||
|
|
||||||
|
from trollius import From, coroutine, Return, async, sleep
|
||||||
|
|
||||||
|
from app import metric_queue
|
||||||
|
from buildman.orchestrator import (orchestrator_from_config, KeyEvent,
|
||||||
|
OrchestratorError, OrchestratorConnectionError,
|
||||||
|
ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION)
|
||||||
|
from buildman.manager.basemanager import BaseManager
|
||||||
|
from buildman.manager.executor import PopenExecutor, EC2Executor, KubernetesExecutor
|
||||||
|
from buildman.component.buildcomponent import BuildComponent
|
||||||
|
from buildman.jobutil.buildjob import BuildJob
|
||||||
|
from buildman.server import BuildJobResult
|
||||||
|
from util import slash_join
|
||||||
|
from util.morecollections import AttrDict
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
JOB_PREFIX = 'building/'
|
||||||
|
LOCK_PREFIX = 'lock/'
|
||||||
|
REALM_PREFIX = 'realm/'
|
||||||
|
CANCEL_PREFIX = 'cancel/'
|
||||||
|
METRIC_PREFIX = 'metric/'
|
||||||
|
|
||||||
|
CANCELED_LOCK_PREFIX = slash_join(LOCK_PREFIX, 'job-cancelled')
|
||||||
|
EXPIRED_LOCK_PREFIX = slash_join(LOCK_PREFIX, 'job-expired')
|
||||||
|
|
||||||
|
EPHEMERAL_API_TIMEOUT = 20
|
||||||
|
EPHEMERAL_SETUP_TIMEOUT = 500
|
||||||
|
|
||||||
|
RETRY_IMMEDIATELY_SLEEP_DURATION = 0
|
||||||
|
TOO_MANY_WORKERS_SLEEP_DURATION = 10
|
||||||
|
|
||||||
|
|
||||||
|
BuildInfo = namedtuple('BuildInfo', ['component', 'build_job', 'execution_id', 'executor_name'])
|
||||||
|
|
||||||
|
|
||||||
|
class EphemeralBuilderManager(BaseManager):
|
||||||
|
""" Build manager implementation for the Enterprise Registry. """
|
||||||
|
|
||||||
|
EXECUTORS = {
|
||||||
|
'popen': PopenExecutor,
|
||||||
|
'ec2': EC2Executor,
|
||||||
|
'kubernetes': KubernetesExecutor,
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(EphemeralBuilderManager, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
self._shutting_down = False
|
||||||
|
|
||||||
|
self._manager_config = None
|
||||||
|
self._orchestrator = None
|
||||||
|
|
||||||
|
# The registered executors available for running jobs, in order.
|
||||||
|
self._ordered_executors = []
|
||||||
|
|
||||||
|
# The registered executors, mapped by their unique name.
|
||||||
|
self._executor_name_to_executor = {}
|
||||||
|
|
||||||
|
# Map from builder component to its associated job.
|
||||||
|
self._component_to_job = {}
|
||||||
|
|
||||||
|
# Map from build UUID to a BuildInfo tuple with information about the build.
|
||||||
|
self._build_uuid_to_info = {}
|
||||||
|
|
||||||
|
def overall_setup_time(self):
|
||||||
|
return EPHEMERAL_SETUP_TIMEOUT
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _mark_job_incomplete(self, build_job, build_info):
|
||||||
|
""" Marks a job as incomplete, in response to a failure to start or a timeout. """
|
||||||
|
executor_name = build_info.executor_name
|
||||||
|
execution_id = build_info.execution_id
|
||||||
|
|
||||||
|
logger.warning('Build executor failed to successfully boot with execution id %s',
|
||||||
|
execution_id)
|
||||||
|
|
||||||
|
# Take a lock to ensure that only one manager reports the build as incomplete for this
|
||||||
|
# execution.
|
||||||
|
lock_key = slash_join(self._expired_lock_prefix, build_job.build_uuid, execution_id)
|
||||||
|
acquired_lock = yield From(self._orchestrator.lock(lock_key))
|
||||||
|
if acquired_lock:
|
||||||
|
try:
|
||||||
|
# Clean up the bookkeeping for the job.
|
||||||
|
yield From(self._orchestrator.delete_key(self._job_key(build_job)))
|
||||||
|
except KeyError:
|
||||||
|
logger.debug('Could not delete job key %s; might have been removed already',
|
||||||
|
build_job.build_uuid)
|
||||||
|
|
||||||
|
logger.error('[BUILD INTERNAL ERROR] Build ID: %s. Exec name: %s. Exec ID: %s',
|
||||||
|
build_job.build_uuid, executor_name, execution_id)
|
||||||
|
yield From(self.job_complete_callback(build_job, BuildJobResult.INCOMPLETE, executor_name,
|
||||||
|
update_phase=True))
|
||||||
|
else:
|
||||||
|
logger.debug('Did not get lock for job-expiration for job %s', build_job.build_uuid)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _job_callback(self, key_change):
|
||||||
|
"""
|
||||||
|
This is the callback invoked when keys related to jobs are changed.
|
||||||
|
It ignores all events related to the creation of new jobs.
|
||||||
|
Deletes or expirations cause checks to ensure they've been properly marked as completed.
|
||||||
|
|
||||||
|
:param key_change: the event and value produced by a key changing in the orchestrator
|
||||||
|
:type key_change: :class:`KeyChange`
|
||||||
|
"""
|
||||||
|
if key_change.event in (KeyEvent.CREATE, KeyEvent.SET):
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
elif key_change.event in (KeyEvent.DELETE, KeyEvent.EXPIRE):
|
||||||
|
# Handle the expiration/deletion.
|
||||||
|
job_metadata = json.loads(key_change.value)
|
||||||
|
build_job = BuildJob(AttrDict(job_metadata['job_queue_item']))
|
||||||
|
logger.debug('Got "%s" of job %s', key_change.event, build_job.build_uuid)
|
||||||
|
|
||||||
|
# Get the build info.
|
||||||
|
build_info = self._build_uuid_to_info.get(build_job.build_uuid, None)
|
||||||
|
if build_info is None:
|
||||||
|
logger.debug('No build info for "%s" job %s (%s); probably already deleted by this manager',
|
||||||
|
key_change.event, build_job.build_uuid, job_metadata)
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
if key_change.event != KeyEvent.EXPIRE:
|
||||||
|
# If the etcd action was not an expiration, then it was already deleted by some manager and
|
||||||
|
# the execution was therefore already shutdown. All that's left is to remove the build info.
|
||||||
|
self._build_uuid_to_info.pop(build_job.build_uuid, None)
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
logger.debug('got expiration for job %s with metadata: %s', build_job.build_uuid,
|
||||||
|
job_metadata)
|
||||||
|
|
||||||
|
if not job_metadata.get('had_heartbeat', False):
|
||||||
|
# If we have not yet received a heartbeat, then the node failed to boot in some way.
|
||||||
|
# We mark the job as incomplete here.
|
||||||
|
yield From(self._mark_job_incomplete(build_job, build_info))
|
||||||
|
|
||||||
|
# Finally, we terminate the build execution for the job. We don't do this under a lock as
|
||||||
|
# terminating a node is an atomic operation; better to make sure it is terminated than not.
|
||||||
|
logger.info('Terminating expired build executor for job %s with execution id %s',
|
||||||
|
build_job.build_uuid, build_info.execution_id)
|
||||||
|
yield From(self.kill_builder_executor(build_job.build_uuid))
|
||||||
|
else:
|
||||||
|
logger.warning('Unexpected KeyEvent (%s) on job key: %s', key_change.event, key_change.key)
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _realm_callback(self, key_change):
|
||||||
|
logger.debug('realm callback for key: %s', key_change.key)
|
||||||
|
if key_change.event == KeyEvent.CREATE:
|
||||||
|
# Listen on the realm created by ourselves or another worker.
|
||||||
|
realm_spec = json.loads(key_change.value)
|
||||||
|
self._register_realm(realm_spec)
|
||||||
|
|
||||||
|
elif key_change.event in (KeyEvent.DELETE, KeyEvent.EXPIRE):
|
||||||
|
# Stop listening for new connections on the realm, if we did not get the connection.
|
||||||
|
realm_spec = json.loads(key_change.value)
|
||||||
|
realm_id = realm_spec['realm']
|
||||||
|
|
||||||
|
build_job = BuildJob(AttrDict(realm_spec['job_queue_item']))
|
||||||
|
build_uuid = build_job.build_uuid
|
||||||
|
|
||||||
|
logger.debug('Realm key %s for build %s was %s', realm_id, build_uuid, key_change.event)
|
||||||
|
build_info = self._build_uuid_to_info.get(build_uuid, None)
|
||||||
|
if build_info is not None:
|
||||||
|
# Pop off the component and if we find one, then the build has not connected to this
|
||||||
|
# manager, so we can safely unregister its component.
|
||||||
|
component = self._component_to_job.pop(build_info.component, None)
|
||||||
|
if component is not None:
|
||||||
|
# We were not the manager which the worker connected to, remove the bookkeeping for it
|
||||||
|
logger.debug('Unregistering unused component for build %s', build_uuid)
|
||||||
|
self.unregister_component(build_info.component)
|
||||||
|
|
||||||
|
# If the realm has expired, then perform cleanup of the executor.
|
||||||
|
if key_change.event == KeyEvent.EXPIRE:
|
||||||
|
execution_id = realm_spec.get('execution_id', None)
|
||||||
|
executor_name = realm_spec.get('executor_name', 'EC2Executor')
|
||||||
|
|
||||||
|
# Cleanup the job, since it never started.
|
||||||
|
logger.debug('Job %s for incomplete marking: %s', build_uuid, build_info)
|
||||||
|
if build_info is not None:
|
||||||
|
yield From(self._mark_job_incomplete(build_job, build_info))
|
||||||
|
|
||||||
|
# Cleanup the executor.
|
||||||
|
logger.info('Realm %s expired for job %s, terminating executor %s with execution id %s',
|
||||||
|
realm_id, build_uuid, executor_name, execution_id)
|
||||||
|
yield From(self.terminate_executor(executor_name, execution_id))
|
||||||
|
|
||||||
|
else:
|
||||||
|
logger.warning('Unexpected action (%s) on realm key: %s', key_change.event, key_change.key)
|
||||||
|
|
||||||
|
|
||||||
|
def _register_realm(self, realm_spec):
|
||||||
|
logger.debug('Got call to register realm %s with manager', realm_spec['realm'])
|
||||||
|
|
||||||
|
# Create the build information block for the registered realm.
|
||||||
|
build_job = BuildJob(AttrDict(realm_spec['job_queue_item']))
|
||||||
|
execution_id = realm_spec.get('execution_id', None)
|
||||||
|
executor_name = realm_spec.get('executor_name', 'EC2Executor')
|
||||||
|
|
||||||
|
logger.debug('Registering realm %s with manager: %s', realm_spec['realm'], realm_spec)
|
||||||
|
component = self.register_component(realm_spec['realm'], BuildComponent,
|
||||||
|
token=realm_spec['token'])
|
||||||
|
|
||||||
|
build_info = BuildInfo(component=component, build_job=build_job, execution_id=execution_id,
|
||||||
|
executor_name=executor_name)
|
||||||
|
|
||||||
|
self._component_to_job[component] = build_job
|
||||||
|
self._build_uuid_to_info[build_job.build_uuid] = build_info
|
||||||
|
|
||||||
|
logger.debug('Registered realm %s with manager', realm_spec['realm'])
|
||||||
|
return component
|
||||||
|
|
||||||
|
@property
|
||||||
|
def registered_executors(self):
|
||||||
|
return self._ordered_executors
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _register_existing_realms(self):
|
||||||
|
try:
|
||||||
|
all_realms = yield From(self._orchestrator.get_prefixed_keys(self._realm_prefix))
|
||||||
|
|
||||||
|
# Register all existing realms found.
|
||||||
|
encountered = {self._register_realm(json.loads(realm_data))
|
||||||
|
for _realm, realm_data in all_realms}
|
||||||
|
|
||||||
|
# Remove any components not encountered so we can clean up.
|
||||||
|
for component, job in iteritems(self._component_to_job):
|
||||||
|
if not component in encountered:
|
||||||
|
self._component_to_job.pop(component, None)
|
||||||
|
self._build_uuid_to_info.pop(job.build_uuid, None)
|
||||||
|
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _load_executor(self, executor_kind_name, executor_config):
|
||||||
|
executor_klass = EphemeralBuilderManager.EXECUTORS.get(executor_kind_name)
|
||||||
|
if executor_klass is None:
|
||||||
|
logger.error('Unknown executor %s; skipping install', executor_kind_name)
|
||||||
|
return
|
||||||
|
|
||||||
|
executor = executor_klass(executor_config, self.manager_hostname)
|
||||||
|
if executor.name in self._executor_name_to_executor:
|
||||||
|
raise Exception('Executor with name %s already registered' % executor.name)
|
||||||
|
|
||||||
|
self._ordered_executors.append(executor)
|
||||||
|
self._executor_name_to_executor[executor.name] = executor
|
||||||
|
|
||||||
|
def _config_prefix(self, key):
|
||||||
|
if self._manager_config.get('ORCHESTRATOR') is None:
|
||||||
|
return key
|
||||||
|
|
||||||
|
prefix = self._manager_config.get('ORCHESTRATOR_PREFIX', '')
|
||||||
|
return slash_join(prefix, key).lstrip('/') + '/'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _job_prefix(self):
|
||||||
|
return self._config_prefix(JOB_PREFIX)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _realm_prefix(self):
|
||||||
|
return self._config_prefix(REALM_PREFIX)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _cancel_prefix(self):
|
||||||
|
return self._config_prefix(CANCEL_PREFIX)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _metric_prefix(self):
|
||||||
|
return self._config_prefix(METRIC_PREFIX)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _expired_lock_prefix(self):
|
||||||
|
return self._config_prefix(EXPIRED_LOCK_PREFIX)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _canceled_lock_prefix(self):
|
||||||
|
return self._config_prefix(CANCELED_LOCK_PREFIX)
|
||||||
|
|
||||||
|
def _metric_key(self, realm):
|
||||||
|
"""
|
||||||
|
Create a key which is used to track a job in the Orchestrator.
|
||||||
|
|
||||||
|
:param realm: realm for the build
|
||||||
|
:type realm: str
|
||||||
|
:returns: key used to track jobs
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
return slash_join(self._metric_prefix, realm)
|
||||||
|
|
||||||
|
def _job_key(self, build_job):
|
||||||
|
"""
|
||||||
|
Creates a key which is used to track a job in the Orchestrator.
|
||||||
|
|
||||||
|
:param build_job: unique job identifier for a build
|
||||||
|
:type build_job: str
|
||||||
|
:returns: key used to track the job
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
return slash_join(self._job_prefix, build_job.job_details['build_uuid'])
|
||||||
|
|
||||||
|
def _realm_key(self, realm):
|
||||||
|
"""
|
||||||
|
Create a key which is used to track an incoming connection on a realm.
|
||||||
|
|
||||||
|
:param realm: realm for the build
|
||||||
|
:type realm: str
|
||||||
|
:returns: key used to track the connection to the realm
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
return slash_join(self._realm_prefix, realm)
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(self, manager_config):
|
||||||
|
logger.debug('Calling initialize')
|
||||||
|
self._manager_config = manager_config
|
||||||
|
|
||||||
|
# Note: Executor config can be defined either as a single block of EXECUTOR_CONFIG (old style)
|
||||||
|
# or as a new set of executor configurations, with the order determining how we fallback. We
|
||||||
|
# check for both here to ensure backwards compatibility.
|
||||||
|
if manager_config.get('EXECUTORS'):
|
||||||
|
for executor_config in manager_config['EXECUTORS']:
|
||||||
|
self._load_executor(executor_config.get('EXECUTOR'), executor_config)
|
||||||
|
else:
|
||||||
|
self._load_executor(manager_config.get('EXECUTOR'), manager_config.get('EXECUTOR_CONFIG'))
|
||||||
|
|
||||||
|
logger.debug('calling orchestrator_from_config')
|
||||||
|
self._orchestrator = orchestrator_from_config(manager_config)
|
||||||
|
|
||||||
|
logger.debug('setting on_key_change callbacks for job, cancel, realm')
|
||||||
|
self._orchestrator.on_key_change(self._job_prefix, self._job_callback)
|
||||||
|
self._orchestrator.on_key_change(self._cancel_prefix, self._cancel_callback)
|
||||||
|
self._orchestrator.on_key_change(self._realm_prefix, self._realm_callback,
|
||||||
|
restarter=self._register_existing_realms)
|
||||||
|
|
||||||
|
# Load components for all realms currently known to the cluster
|
||||||
|
async(self._register_existing_realms())
|
||||||
|
|
||||||
|
def shutdown(self):
|
||||||
|
logger.debug('Shutting down worker.')
|
||||||
|
if self._orchestrator is not None:
|
||||||
|
self._orchestrator.shutdown()
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def schedule(self, build_job):
|
||||||
|
build_uuid = build_job.job_details['build_uuid']
|
||||||
|
logger.debug('Calling schedule with job: %s', build_uuid)
|
||||||
|
|
||||||
|
# Check if there are worker slots available by checking the number of jobs in the orchestrator
|
||||||
|
allowed_worker_count = self._manager_config.get('ALLOWED_WORKER_COUNT', 1)
|
||||||
|
try:
|
||||||
|
active_jobs = yield From(self._orchestrator.get_prefixed_keys(self._job_prefix))
|
||||||
|
workers_alive = len(active_jobs)
|
||||||
|
except KeyError:
|
||||||
|
workers_alive = 0
|
||||||
|
except OrchestratorConnectionError:
|
||||||
|
logger.exception('Could not read job count from orchestrator for job due to orchestrator being down')
|
||||||
|
raise Return(False, ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION)
|
||||||
|
except OrchestratorError:
|
||||||
|
logger.exception('Exception when reading job count from orchestrator for job: %s', build_uuid)
|
||||||
|
raise Return(False, RETRY_IMMEDIATELY_SLEEP_DURATION)
|
||||||
|
|
||||||
|
logger.debug('Total jobs (scheduling job %s): %s', build_uuid, workers_alive)
|
||||||
|
|
||||||
|
if workers_alive >= allowed_worker_count:
|
||||||
|
logger.info('Too many workers alive, unable to start new worker for build job: %s. %s >= %s',
|
||||||
|
build_uuid, workers_alive, allowed_worker_count)
|
||||||
|
raise Return(False, TOO_MANY_WORKERS_SLEEP_DURATION)
|
||||||
|
|
||||||
|
job_key = self._job_key(build_job)
|
||||||
|
|
||||||
|
# First try to take a lock for this job, meaning we will be responsible for its lifeline
|
||||||
|
realm = str(uuid.uuid4())
|
||||||
|
token = str(uuid.uuid4())
|
||||||
|
nonce = str(uuid.uuid4())
|
||||||
|
|
||||||
|
machine_max_expiration = self._manager_config.get('MACHINE_MAX_TIME', 7200)
|
||||||
|
max_expiration = datetime.utcnow() + timedelta(seconds=machine_max_expiration)
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
'max_expiration': calendar.timegm(max_expiration.timetuple()),
|
||||||
|
'nonce': nonce,
|
||||||
|
'had_heartbeat': False,
|
||||||
|
'job_queue_item': build_job.job_item,
|
||||||
|
}
|
||||||
|
|
||||||
|
lock_payload = json.dumps(payload)
|
||||||
|
logger.debug('Writing key for job %s with expiration in %s seconds', build_uuid,
|
||||||
|
EPHEMERAL_SETUP_TIMEOUT)
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield From(self._orchestrator.set_key(job_key, lock_payload, overwrite=False,
|
||||||
|
expiration=EPHEMERAL_SETUP_TIMEOUT))
|
||||||
|
except KeyError:
|
||||||
|
logger.warning('Job: %s already exists in orchestrator, timeout may be misconfigured',
|
||||||
|
build_uuid)
|
||||||
|
raise Return(False, EPHEMERAL_API_TIMEOUT)
|
||||||
|
except OrchestratorConnectionError:
|
||||||
|
logger.exception('Exception when writing job %s to orchestrator; could not connect',
|
||||||
|
build_uuid)
|
||||||
|
raise Return(False, ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION)
|
||||||
|
except OrchestratorError:
|
||||||
|
logger.exception('Exception when writing job %s to orchestrator', build_uuid)
|
||||||
|
raise Return(False, RETRY_IMMEDIATELY_SLEEP_DURATION)
|
||||||
|
|
||||||
|
# Got a lock, now lets boot the job via one of the registered executors.
|
||||||
|
started_with_executor = None
|
||||||
|
execution_id = None
|
||||||
|
|
||||||
|
logger.debug("Registered executors are: %s", [ex.name for ex in self._ordered_executors])
|
||||||
|
for executor in self._ordered_executors:
|
||||||
|
# Check if we can use this executor based on its whitelist, by namespace.
|
||||||
|
namespace = build_job.namespace
|
||||||
|
if not executor.allowed_for_namespace(namespace):
|
||||||
|
logger.debug('Job %s (namespace: %s) cannot use executor %s', build_uuid, namespace,
|
||||||
|
executor.name)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check if we can use this executor based on the retries remaining.
|
||||||
|
if executor.minimum_retry_threshold > build_job.retries_remaining:
|
||||||
|
metric_queue.builder_fallback.Inc()
|
||||||
|
logger.debug('Job %s cannot use executor %s as it is below retry threshold %s (retry #%s)',
|
||||||
|
build_uuid, executor.name, executor.minimum_retry_threshold,
|
||||||
|
build_job.retries_remaining)
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.debug('Starting builder for job %s with selected executor: %s', build_uuid,
|
||||||
|
executor.name)
|
||||||
|
|
||||||
|
try:
|
||||||
|
execution_id = yield From(executor.start_builder(realm, token, build_uuid))
|
||||||
|
except:
|
||||||
|
try:
|
||||||
|
metric_queue.build_start_failure.Inc(labelvalues=[executor.name])
|
||||||
|
metric_queue.put_deprecated(('ExecutorFailure-%s' % executor.name), 1, unit='Count')
|
||||||
|
except:
|
||||||
|
logger.exception('Exception when writing failure metric for execution %s for job %s',
|
||||||
|
execution_id, build_uuid)
|
||||||
|
|
||||||
|
logger.exception('Exception when starting builder for job: %s', build_uuid)
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
metric_queue.build_start_success.Inc(labelvalues=[executor.name])
|
||||||
|
except:
|
||||||
|
logger.exception('Exception when writing success metric for execution %s for job %s',
|
||||||
|
execution_id, build_uuid)
|
||||||
|
|
||||||
|
try:
|
||||||
|
metric_queue.ephemeral_build_workers.Inc()
|
||||||
|
except:
|
||||||
|
logger.exception('Exception when writing start metrics for execution %s for job %s',
|
||||||
|
execution_id, build_uuid)
|
||||||
|
|
||||||
|
started_with_executor = executor
|
||||||
|
|
||||||
|
# Break out of the loop now that we've started a builder successfully.
|
||||||
|
break
|
||||||
|
|
||||||
|
# If we didn't start the job, cleanup and return it to the queue.
|
||||||
|
if started_with_executor is None:
|
||||||
|
logger.error('Could not start ephemeral worker for build %s', build_uuid)
|
||||||
|
|
||||||
|
# Delete the associated build job record.
|
||||||
|
yield From(self._orchestrator.delete_key(job_key))
|
||||||
|
raise Return(False, EPHEMERAL_API_TIMEOUT)
|
||||||
|
|
||||||
|
# Job was started!
|
||||||
|
logger.debug('Started execution with ID %s for job: %s with executor: %s',
|
||||||
|
execution_id, build_uuid, started_with_executor.name)
|
||||||
|
|
||||||
|
# Store metric data
|
||||||
|
metric_spec = json.dumps({
|
||||||
|
'executor_name': started_with_executor.name,
|
||||||
|
'start_time': time.time(),
|
||||||
|
})
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield From(self._orchestrator.set_key(self._metric_key(realm), metric_spec, overwrite=False,
|
||||||
|
expiration=machine_max_expiration + 10))
|
||||||
|
except KeyError:
|
||||||
|
logger.error('Realm %s already exists in orchestrator for job %s ' +
|
||||||
|
'UUID collision or something is very very wrong.', realm, build_uuid)
|
||||||
|
except OrchestratorError:
|
||||||
|
logger.exception('Exception when writing realm %s to orchestrator for job %s',
|
||||||
|
realm, build_uuid)
|
||||||
|
|
||||||
|
# Store the realm spec which will allow any manager to accept this builder when it connects
|
||||||
|
realm_spec = json.dumps({
|
||||||
|
'realm': realm,
|
||||||
|
'token': token,
|
||||||
|
'execution_id': execution_id,
|
||||||
|
'executor_name': started_with_executor.name,
|
||||||
|
'job_queue_item': build_job.job_item,
|
||||||
|
})
|
||||||
|
|
||||||
|
try:
|
||||||
|
setup_time = started_with_executor.setup_time or self.overall_setup_time()
|
||||||
|
logger.debug('Writing job key for job %s using executor %s with ID %s and ttl %s', build_uuid,
|
||||||
|
started_with_executor.name, execution_id, setup_time)
|
||||||
|
yield From(self._orchestrator.set_key(self._realm_key(realm), realm_spec,
|
||||||
|
expiration=setup_time))
|
||||||
|
except OrchestratorConnectionError:
|
||||||
|
logger.exception('Exception when writing realm %s to orchestrator for job %s',
|
||||||
|
realm, build_uuid)
|
||||||
|
raise Return(False, ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION)
|
||||||
|
except OrchestratorError:
|
||||||
|
logger.exception('Exception when writing realm %s to orchestrator for job %s',
|
||||||
|
realm, build_uuid)
|
||||||
|
raise Return(False, setup_time)
|
||||||
|
|
||||||
|
logger.debug('Builder spawn complete for job %s using executor %s with ID %s ',
|
||||||
|
build_uuid, started_with_executor.name, execution_id)
|
||||||
|
raise Return(True, None)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def build_component_ready(self, build_component):
|
||||||
|
logger.debug('Got component ready for component with realm %s', build_component.builder_realm)
|
||||||
|
|
||||||
|
# Pop off the job for the component.
|
||||||
|
# We do so before we send out the watch below, as it will also remove this mapping.
|
||||||
|
job = self._component_to_job.pop(build_component, None)
|
||||||
|
if job is None:
|
||||||
|
# This will occur once the build finishes, so no need to worry about it.
|
||||||
|
# We log in case it happens outside of the expected flow.
|
||||||
|
logger.debug('Could not find job for the build component on realm %s; component is ready',
|
||||||
|
build_component.builder_realm)
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
# Start the build job.
|
||||||
|
logger.debug('Sending build %s to newly ready component on realm %s',
|
||||||
|
job.build_uuid, build_component.builder_realm)
|
||||||
|
yield From(build_component.start_build(job))
|
||||||
|
|
||||||
|
yield From(self._write_duration_metric(metric_queue.builder_time_to_build,
|
||||||
|
build_component.builder_realm))
|
||||||
|
|
||||||
|
# Clean up the bookkeeping for allowing any manager to take the job.
|
||||||
|
try:
|
||||||
|
yield From(self._orchestrator.delete_key(self._realm_key(build_component.builder_realm)))
|
||||||
|
except KeyError:
|
||||||
|
logger.warning('Could not delete realm key %s', build_component.builder_realm)
|
||||||
|
|
||||||
|
def build_component_disposed(self, build_component, timed_out):
|
||||||
|
logger.debug('Calling build_component_disposed.')
|
||||||
|
self.unregister_component(build_component)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def job_completed(self, build_job, job_status, build_component):
|
||||||
|
logger.debug('Calling job_completed for job %s with status: %s',
|
||||||
|
build_job.build_uuid, job_status)
|
||||||
|
|
||||||
|
yield From(self._write_duration_metric(metric_queue.build_time, build_component.builder_realm))
|
||||||
|
|
||||||
|
# Mark the job as completed. Since this is being invoked from the component, we don't need
|
||||||
|
# to ask for the phase to be updated as well.
|
||||||
|
build_info = self._build_uuid_to_info.get(build_job.build_uuid, None)
|
||||||
|
executor_name = build_info.executor_name if build_info else None
|
||||||
|
yield From(self.job_complete_callback(build_job, job_status, executor_name, update_phase=False))
|
||||||
|
|
||||||
|
# Kill the ephemeral builder.
|
||||||
|
yield From(self.kill_builder_executor(build_job.build_uuid))
|
||||||
|
|
||||||
|
# Delete the build job from the orchestrator.
|
||||||
|
try:
|
||||||
|
job_key = self._job_key(build_job)
|
||||||
|
yield From(self._orchestrator.delete_key(job_key))
|
||||||
|
except KeyError:
|
||||||
|
logger.debug('Builder is asking for job to be removed, but work already completed')
|
||||||
|
except OrchestratorConnectionError:
|
||||||
|
logger.exception('Could not remove job key as orchestrator is not available')
|
||||||
|
yield From(sleep(ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION))
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
# Delete the metric from the orchestrator.
|
||||||
|
try:
|
||||||
|
metric_key = self._metric_key(build_component.builder_realm)
|
||||||
|
yield From(self._orchestrator.delete_key(metric_key))
|
||||||
|
except KeyError:
|
||||||
|
logger.debug('Builder is asking for metric to be removed, but key not found')
|
||||||
|
except OrchestratorConnectionError:
|
||||||
|
logger.exception('Could not remove metric key as orchestrator is not available')
|
||||||
|
yield From(sleep(ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION))
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
logger.debug('job_completed for job %s with status: %s', build_job.build_uuid, job_status)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def kill_builder_executor(self, build_uuid):
|
||||||
|
logger.info('Starting termination of executor for job %s', build_uuid)
|
||||||
|
build_info = self._build_uuid_to_info.pop(build_uuid, None)
|
||||||
|
if build_info is None:
|
||||||
|
logger.debug('Build information not found for build %s; skipping termination', build_uuid)
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
# Remove the build's component.
|
||||||
|
self._component_to_job.pop(build_info.component, None)
|
||||||
|
|
||||||
|
# Stop the build node/executor itself.
|
||||||
|
yield From(self.terminate_executor(build_info.executor_name, build_info.execution_id))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def terminate_executor(self, executor_name, execution_id):
|
||||||
|
executor = self._executor_name_to_executor.get(executor_name)
|
||||||
|
if executor is None:
|
||||||
|
logger.error('Could not find registered executor %s', executor_name)
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
# Terminate the executor's execution.
|
||||||
|
logger.info('Terminating executor %s with execution id %s', executor_name, execution_id)
|
||||||
|
yield From(executor.stop_builder(execution_id))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def job_heartbeat(self, build_job):
|
||||||
|
"""
|
||||||
|
:param build_job: the identifier for the build
|
||||||
|
:type build_job: str
|
||||||
|
"""
|
||||||
|
self.job_heartbeat_callback(build_job)
|
||||||
|
self._extend_job_in_orchestrator(build_job)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _extend_job_in_orchestrator(self, build_job):
|
||||||
|
try:
|
||||||
|
job_data = yield From(self._orchestrator.get_key(self._job_key(build_job)))
|
||||||
|
except KeyError:
|
||||||
|
logger.info('Job %s no longer exists in the orchestrator', build_job.build_uuid)
|
||||||
|
raise Return()
|
||||||
|
except OrchestratorConnectionError:
|
||||||
|
logger.exception('failed to connect when attempted to extend job')
|
||||||
|
|
||||||
|
build_job_metadata = json.loads(job_data)
|
||||||
|
|
||||||
|
max_expiration = datetime.utcfromtimestamp(build_job_metadata['max_expiration'])
|
||||||
|
max_expiration_remaining = max_expiration - datetime.utcnow()
|
||||||
|
max_expiration_sec = max(0, int(max_expiration_remaining.total_seconds()))
|
||||||
|
|
||||||
|
ttl = min(self.heartbeat_period_sec * 2, max_expiration_sec)
|
||||||
|
payload = {
|
||||||
|
'job_queue_item': build_job.job_item,
|
||||||
|
'max_expiration': build_job_metadata['max_expiration'],
|
||||||
|
'had_heartbeat': True,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield From(self._orchestrator.set_key(self._job_key(build_job), json.dumps(payload),
|
||||||
|
expiration=ttl))
|
||||||
|
except OrchestratorConnectionError:
|
||||||
|
logger.exception('Could not update heartbeat for job as the orchestrator is not available')
|
||||||
|
yield From(sleep(ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _write_duration_metric(self, metric, realm):
|
||||||
|
"""
|
||||||
|
:returns: True if the metric was written, otherwise False
|
||||||
|
:rtype: bool
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
metric_data = yield From(self._orchestrator.get_key(self._metric_key(realm)))
|
||||||
|
parsed_metric_data = json.loads(metric_data)
|
||||||
|
start_time = parsed_metric_data['start_time']
|
||||||
|
metric.Observe(time.time() - start_time,
|
||||||
|
labelvalues=[parsed_metric_data.get('executor_name',
|
||||||
|
'unknown')])
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Could not write metric for realm %s", realm)
|
||||||
|
|
||||||
|
def num_workers(self):
|
||||||
|
"""
|
||||||
|
The number of workers we're managing locally.
|
||||||
|
|
||||||
|
:returns: the number of the workers locally managed
|
||||||
|
:rtype: int
|
||||||
|
"""
|
||||||
|
return len(self._component_to_job)
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _cancel_callback(self, key_change):
|
||||||
|
if key_change.event not in (KeyEvent.CREATE, KeyEvent.SET):
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
build_uuid = key_change.value
|
||||||
|
build_info = self._build_uuid_to_info.get(build_uuid, None)
|
||||||
|
if build_info is None:
|
||||||
|
logger.debug('No build info for "%s" job %s', key_change.event, build_uuid)
|
||||||
|
raise Return(False)
|
||||||
|
|
||||||
|
lock_key = slash_join(self._canceled_lock_prefix,
|
||||||
|
build_uuid, build_info.execution_id)
|
||||||
|
lock_acquired = yield From(self._orchestrator.lock(lock_key))
|
||||||
|
if lock_acquired:
|
||||||
|
builder_realm = build_info.component.builder_realm
|
||||||
|
yield From(self.kill_builder_executor(build_uuid))
|
||||||
|
yield From(self._orchestrator.delete_key(self._realm_key(builder_realm)))
|
||||||
|
yield From(self._orchestrator.delete_key(self._metric_key(builder_realm)))
|
||||||
|
yield From(self._orchestrator.delete_key(slash_join(self._job_prefix, build_uuid)))
|
||||||
|
|
||||||
|
# This is outside the lock so we can un-register the component wherever it is registered to.
|
||||||
|
yield From(build_info.component.cancel_build())
|
||||||
37
buildman/manager/etcd_canceller.py
Normal file
37
buildman/manager/etcd_canceller.py
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
import logging
|
||||||
|
import etcd
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class EtcdCanceller(object):
|
||||||
|
""" A class that sends a message to etcd to cancel a build """
|
||||||
|
|
||||||
|
def __init__(self, config):
|
||||||
|
etcd_host = config.get('ETCD_HOST', '127.0.0.1')
|
||||||
|
etcd_port = config.get('ETCD_PORT', 2379)
|
||||||
|
etcd_ca_cert = config.get('ETCD_CA_CERT', None)
|
||||||
|
etcd_auth = config.get('ETCD_CERT_AND_KEY', None)
|
||||||
|
if etcd_auth is not None:
|
||||||
|
etcd_auth = tuple(etcd_auth)
|
||||||
|
|
||||||
|
etcd_protocol = 'http' if etcd_auth is None else 'https'
|
||||||
|
logger.debug('Connecting to etcd on %s:%s', etcd_host, etcd_port)
|
||||||
|
self._cancel_prefix = config.get('ETCD_CANCEL_PREFIX', 'cancel/')
|
||||||
|
self._etcd_client = etcd.Client(
|
||||||
|
host=etcd_host,
|
||||||
|
port=etcd_port,
|
||||||
|
cert=etcd_auth,
|
||||||
|
ca_cert=etcd_ca_cert,
|
||||||
|
protocol=etcd_protocol,
|
||||||
|
read_timeout=5)
|
||||||
|
|
||||||
|
def try_cancel_build(self, build_uuid):
|
||||||
|
""" Writes etcd message to cancel build_uuid. """
|
||||||
|
logger.info("Cancelling build %s".format(build_uuid))
|
||||||
|
try:
|
||||||
|
self._etcd_client.write("{}{}".format(self._cancel_prefix, build_uuid), build_uuid, ttl=60)
|
||||||
|
return True
|
||||||
|
except etcd.EtcdException:
|
||||||
|
logger.exception("Failed to write to etcd client %s", build_uuid)
|
||||||
|
return False
|
||||||
560
buildman/manager/executor.py
Normal file
560
buildman/manager/executor.py
Normal file
|
|
@ -0,0 +1,560 @@
|
||||||
|
import datetime
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
import boto.ec2
|
||||||
|
import cachetools.func
|
||||||
|
import requests
|
||||||
|
import trollius
|
||||||
|
|
||||||
|
from container_cloud_config import CloudConfigContext
|
||||||
|
from jinja2 import FileSystemLoader, Environment
|
||||||
|
from trollius import coroutine, From, Return, get_event_loop
|
||||||
|
|
||||||
|
import release
|
||||||
|
|
||||||
|
from buildman.asyncutil import AsyncWrapper
|
||||||
|
from app import metric_queue, app
|
||||||
|
from util.metrics.metricqueue import duration_collector_async
|
||||||
|
from _init import ROOT_DIR
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
ONE_HOUR = 60*60
|
||||||
|
|
||||||
|
_TAG_RETRY_COUNT = 3 # Number of times to retry adding tags.
|
||||||
|
_TAG_RETRY_SLEEP = 2 # Number of seconds to wait between tag retries.
|
||||||
|
|
||||||
|
ENV = Environment(loader=FileSystemLoader(os.path.join(ROOT_DIR, "buildman/templates")))
|
||||||
|
TEMPLATE = ENV.get_template('cloudconfig.yaml')
|
||||||
|
CloudConfigContext().populate_jinja_environment(ENV)
|
||||||
|
|
||||||
|
class ExecutorException(Exception):
|
||||||
|
""" Exception raised when there is a problem starting or stopping a builder.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class BuilderExecutor(object):
|
||||||
|
def __init__(self, executor_config, manager_hostname):
|
||||||
|
""" Interface which can be plugged into the EphemeralNodeManager to provide a strategy for
|
||||||
|
starting and stopping builders.
|
||||||
|
"""
|
||||||
|
self.executor_config = executor_config
|
||||||
|
self.manager_hostname = manager_hostname
|
||||||
|
|
||||||
|
default_websocket_scheme = 'wss' if app.config['PREFERRED_URL_SCHEME'] == 'https' else 'ws'
|
||||||
|
self.websocket_scheme = executor_config.get("WEBSOCKET_SCHEME", default_websocket_scheme)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name(self):
|
||||||
|
""" Name returns the unique name for this executor. """
|
||||||
|
return self.executor_config.get('NAME') or self.__class__.__name__
|
||||||
|
|
||||||
|
@property
|
||||||
|
def setup_time(self):
|
||||||
|
""" Returns the amount of time (in seconds) to wait for the execution to start for the build.
|
||||||
|
If None, the manager's default will be used.
|
||||||
|
"""
|
||||||
|
return self.executor_config.get('SETUP_TIME')
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def start_builder(self, realm, token, build_uuid):
|
||||||
|
""" Create a builder with the specified config. Returns a unique id which can be used to manage
|
||||||
|
the builder.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def stop_builder(self, builder_id):
|
||||||
|
""" Stop a builder which is currently running.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def allowed_for_namespace(self, namespace):
|
||||||
|
""" Returns true if this executor can be used for builds in the given namespace. """
|
||||||
|
|
||||||
|
# Check for an explicit namespace whitelist.
|
||||||
|
namespace_whitelist = self.executor_config.get('NAMESPACE_WHITELIST')
|
||||||
|
if namespace_whitelist is not None and namespace in namespace_whitelist:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Check for a staged rollout percentage. If found, we hash the namespace and, if it is found
|
||||||
|
# in the first X% of the character space, we allow this executor to be used.
|
||||||
|
staged_rollout = self.executor_config.get('STAGED_ROLLOUT')
|
||||||
|
if staged_rollout is not None:
|
||||||
|
bucket = int(hashlib.sha256(namespace).hexdigest()[-2:], 16)
|
||||||
|
return bucket < (256 * staged_rollout)
|
||||||
|
|
||||||
|
# If there are no restrictions in place, we are free to use this executor.
|
||||||
|
return staged_rollout is None and namespace_whitelist is None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def minimum_retry_threshold(self):
|
||||||
|
""" Returns the minimum number of retries required for this executor to be used or 0 if
|
||||||
|
none. """
|
||||||
|
return self.executor_config.get('MINIMUM_RETRY_THRESHOLD', 0)
|
||||||
|
|
||||||
|
def generate_cloud_config(self, realm, token, build_uuid, coreos_channel,
|
||||||
|
manager_hostname, quay_username=None,
|
||||||
|
quay_password=None):
|
||||||
|
if quay_username is None:
|
||||||
|
quay_username = self.executor_config['QUAY_USERNAME']
|
||||||
|
|
||||||
|
if quay_password is None:
|
||||||
|
quay_password = self.executor_config['QUAY_PASSWORD']
|
||||||
|
|
||||||
|
return TEMPLATE.render(
|
||||||
|
realm=realm,
|
||||||
|
token=token,
|
||||||
|
build_uuid=build_uuid,
|
||||||
|
quay_username=quay_username,
|
||||||
|
quay_password=quay_password,
|
||||||
|
manager_hostname=manager_hostname,
|
||||||
|
websocket_scheme=self.websocket_scheme,
|
||||||
|
coreos_channel=coreos_channel,
|
||||||
|
worker_image=self.executor_config.get('WORKER_IMAGE', 'quay.io/coreos/registry-build-worker'),
|
||||||
|
worker_tag=self.executor_config['WORKER_TAG'],
|
||||||
|
logentries_token=self.executor_config.get('LOGENTRIES_TOKEN', None),
|
||||||
|
volume_size=self.executor_config.get('VOLUME_SIZE', '42G'),
|
||||||
|
max_lifetime_s=self.executor_config.get('MAX_LIFETIME_S', 10800),
|
||||||
|
ssh_authorized_keys=self.executor_config.get('SSH_AUTHORIZED_KEYS', []),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class EC2Executor(BuilderExecutor):
|
||||||
|
""" Implementation of BuilderExecutor which uses libcloud to start machines on a variety of cloud
|
||||||
|
providers.
|
||||||
|
"""
|
||||||
|
COREOS_STACK_URL = 'http://%s.release.core-os.net/amd64-usr/current/coreos_production_ami_hvm.txt'
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
self._loop = get_event_loop()
|
||||||
|
super(EC2Executor, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
def _get_conn(self):
|
||||||
|
""" Creates an ec2 connection which can be used to manage instances.
|
||||||
|
"""
|
||||||
|
return AsyncWrapper(boto.ec2.connect_to_region(
|
||||||
|
self.executor_config['EC2_REGION'],
|
||||||
|
aws_access_key_id=self.executor_config['AWS_ACCESS_KEY'],
|
||||||
|
aws_secret_access_key=self.executor_config['AWS_SECRET_KEY'],
|
||||||
|
))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@cachetools.func.ttl_cache(ttl=ONE_HOUR)
|
||||||
|
def _get_coreos_ami(cls, ec2_region, coreos_channel):
|
||||||
|
""" Retrieve the CoreOS AMI id from the canonical listing.
|
||||||
|
"""
|
||||||
|
stack_list_string = requests.get(EC2Executor.COREOS_STACK_URL % coreos_channel).text
|
||||||
|
stack_amis = dict([stack.split('=') for stack in stack_list_string.split('|')])
|
||||||
|
return stack_amis[ec2_region]
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
@duration_collector_async(metric_queue.builder_time_to_start, ['ec2'])
|
||||||
|
def start_builder(self, realm, token, build_uuid):
|
||||||
|
region = self.executor_config['EC2_REGION']
|
||||||
|
channel = self.executor_config.get('COREOS_CHANNEL', 'stable')
|
||||||
|
|
||||||
|
coreos_ami = self.executor_config.get('COREOS_AMI', None)
|
||||||
|
if coreos_ami is None:
|
||||||
|
get_ami_callable = partial(self._get_coreos_ami, region, channel)
|
||||||
|
coreos_ami = yield From(self._loop.run_in_executor(None, get_ami_callable))
|
||||||
|
|
||||||
|
user_data = self.generate_cloud_config(realm, token, build_uuid, channel, self.manager_hostname)
|
||||||
|
logger.debug('Generated cloud config for build %s: %s', build_uuid, user_data)
|
||||||
|
|
||||||
|
ec2_conn = self._get_conn()
|
||||||
|
|
||||||
|
ssd_root_ebs = boto.ec2.blockdevicemapping.BlockDeviceType(
|
||||||
|
size=int(self.executor_config.get('BLOCK_DEVICE_SIZE', 48)),
|
||||||
|
volume_type='gp2',
|
||||||
|
delete_on_termination=True,
|
||||||
|
)
|
||||||
|
block_devices = boto.ec2.blockdevicemapping.BlockDeviceMapping()
|
||||||
|
block_devices['/dev/xvda'] = ssd_root_ebs
|
||||||
|
|
||||||
|
interfaces = None
|
||||||
|
if self.executor_config.get('EC2_VPC_SUBNET_ID', None) is not None:
|
||||||
|
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
|
||||||
|
subnet_id=self.executor_config['EC2_VPC_SUBNET_ID'],
|
||||||
|
groups=self.executor_config['EC2_SECURITY_GROUP_IDS'],
|
||||||
|
associate_public_ip_address=True,
|
||||||
|
)
|
||||||
|
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
|
||||||
|
|
||||||
|
try:
|
||||||
|
reservation = yield From(ec2_conn.run_instances(
|
||||||
|
coreos_ami,
|
||||||
|
instance_type=self.executor_config['EC2_INSTANCE_TYPE'],
|
||||||
|
key_name=self.executor_config.get('EC2_KEY_NAME', None),
|
||||||
|
user_data=user_data,
|
||||||
|
instance_initiated_shutdown_behavior='terminate',
|
||||||
|
block_device_map=block_devices,
|
||||||
|
network_interfaces=interfaces,
|
||||||
|
))
|
||||||
|
except boto.exception.EC2ResponseError as ec2e:
|
||||||
|
logger.exception('Unable to spawn builder instance')
|
||||||
|
metric_queue.ephemeral_build_worker_failure.Inc()
|
||||||
|
raise ec2e
|
||||||
|
|
||||||
|
if not reservation.instances:
|
||||||
|
raise ExecutorException('Unable to spawn builder instance.')
|
||||||
|
elif len(reservation.instances) != 1:
|
||||||
|
raise ExecutorException('EC2 started wrong number of instances!')
|
||||||
|
|
||||||
|
launched = AsyncWrapper(reservation.instances[0])
|
||||||
|
|
||||||
|
# Sleep a few seconds to wait for AWS to spawn the instance.
|
||||||
|
yield From(trollius.sleep(_TAG_RETRY_SLEEP))
|
||||||
|
|
||||||
|
# Tag the instance with its metadata.
|
||||||
|
for i in range(0, _TAG_RETRY_COUNT):
|
||||||
|
try:
|
||||||
|
yield From(launched.add_tags({
|
||||||
|
'Name': 'Quay Ephemeral Builder',
|
||||||
|
'Realm': realm,
|
||||||
|
'Token': token,
|
||||||
|
'BuildUUID': build_uuid,
|
||||||
|
}))
|
||||||
|
except boto.exception.EC2ResponseError as ec2e:
|
||||||
|
if ec2e.error_code == 'InvalidInstanceID.NotFound':
|
||||||
|
if i < _TAG_RETRY_COUNT - 1:
|
||||||
|
logger.warning('Failed to write EC2 tags for instance %s for build %s (attempt #%s)',
|
||||||
|
launched.id, build_uuid, i)
|
||||||
|
yield From(trollius.sleep(_TAG_RETRY_SLEEP))
|
||||||
|
continue
|
||||||
|
|
||||||
|
raise ExecutorException('Unable to find builder instance.')
|
||||||
|
|
||||||
|
logger.exception('Failed to write EC2 tags (attempt #%s)', i)
|
||||||
|
|
||||||
|
logger.debug('Machine with ID %s started for build %s', launched.id, build_uuid)
|
||||||
|
raise Return(launched.id)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def stop_builder(self, builder_id):
|
||||||
|
try:
|
||||||
|
ec2_conn = self._get_conn()
|
||||||
|
terminated_instances = yield From(ec2_conn.terminate_instances([builder_id]))
|
||||||
|
except boto.exception.EC2ResponseError as ec2e:
|
||||||
|
if ec2e.error_code == 'InvalidInstanceID.NotFound':
|
||||||
|
logger.debug('Instance %s already terminated', builder_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.exception('Exception when trying to terminate instance %s', builder_id)
|
||||||
|
raise
|
||||||
|
|
||||||
|
if builder_id not in [si.id for si in terminated_instances]:
|
||||||
|
raise ExecutorException('Unable to terminate instance: %s' % builder_id)
|
||||||
|
|
||||||
|
|
||||||
|
class PopenExecutor(BuilderExecutor):
|
||||||
|
""" Implementation of BuilderExecutor which uses Popen to fork a quay-builder process.
|
||||||
|
"""
|
||||||
|
def __init__(self, executor_config, manager_hostname):
|
||||||
|
self._jobs = {}
|
||||||
|
|
||||||
|
super(PopenExecutor, self).__init__(executor_config, manager_hostname)
|
||||||
|
|
||||||
|
""" Executor which uses Popen to fork a quay-builder process.
|
||||||
|
"""
|
||||||
|
@coroutine
|
||||||
|
@duration_collector_async(metric_queue.builder_time_to_start, ['fork'])
|
||||||
|
def start_builder(self, realm, token, build_uuid):
|
||||||
|
# Now start a machine for this job, adding the machine id to the etcd information
|
||||||
|
logger.debug('Forking process for build')
|
||||||
|
|
||||||
|
ws_host = os.environ.get("BUILDMAN_WS_HOST", "localhost")
|
||||||
|
ws_port = os.environ.get("BUILDMAN_WS_PORT", "8787")
|
||||||
|
builder_env = {
|
||||||
|
'TOKEN': token,
|
||||||
|
'REALM': realm,
|
||||||
|
'ENDPOINT': 'ws://%s:%s' % (ws_host, ws_port),
|
||||||
|
'DOCKER_TLS_VERIFY': os.environ.get('DOCKER_TLS_VERIFY', ''),
|
||||||
|
'DOCKER_CERT_PATH': os.environ.get('DOCKER_CERT_PATH', ''),
|
||||||
|
'DOCKER_HOST': os.environ.get('DOCKER_HOST', ''),
|
||||||
|
'PATH': "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||||
|
}
|
||||||
|
|
||||||
|
logpipe = LogPipe(logging.INFO)
|
||||||
|
spawned = subprocess.Popen(os.environ.get('BUILDER_BINARY_LOCATION',
|
||||||
|
'/usr/local/bin/quay-builder'),
|
||||||
|
stdout=logpipe,
|
||||||
|
stderr=logpipe,
|
||||||
|
env=builder_env)
|
||||||
|
|
||||||
|
builder_id = str(uuid.uuid4())
|
||||||
|
self._jobs[builder_id] = (spawned, logpipe)
|
||||||
|
logger.debug('Builder spawned with id: %s', builder_id)
|
||||||
|
raise Return(builder_id)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def stop_builder(self, builder_id):
|
||||||
|
if builder_id not in self._jobs:
|
||||||
|
raise ExecutorException('Builder id not being tracked by executor.')
|
||||||
|
|
||||||
|
logger.debug('Killing builder with id: %s', builder_id)
|
||||||
|
spawned, logpipe = self._jobs[builder_id]
|
||||||
|
|
||||||
|
if spawned.poll() is None:
|
||||||
|
spawned.kill()
|
||||||
|
logpipe.close()
|
||||||
|
|
||||||
|
|
||||||
|
class KubernetesExecutor(BuilderExecutor):
|
||||||
|
""" Executes build jobs by creating Kubernetes jobs which run a qemu-kvm virtual
|
||||||
|
machine in a pod """
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(KubernetesExecutor, self).__init__(*args, **kwargs)
|
||||||
|
self._loop = get_event_loop()
|
||||||
|
self.namespace = self.executor_config.get('BUILDER_NAMESPACE', 'builder')
|
||||||
|
self.image = self.executor_config.get('BUILDER_VM_CONTAINER_IMAGE',
|
||||||
|
'quay.io/quay/quay-builder-qemu-coreos:stable')
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _request(self, method, path, **kwargs):
|
||||||
|
request_options = dict(kwargs)
|
||||||
|
|
||||||
|
tls_cert = self.executor_config.get('K8S_API_TLS_CERT')
|
||||||
|
tls_key = self.executor_config.get('K8S_API_TLS_KEY')
|
||||||
|
tls_ca = self.executor_config.get('K8S_API_TLS_CA')
|
||||||
|
service_account_token = self.executor_config.get('SERVICE_ACCOUNT_TOKEN')
|
||||||
|
|
||||||
|
if 'timeout' not in request_options:
|
||||||
|
request_options['timeout'] = self.executor_config.get("K8S_API_TIMEOUT", 20)
|
||||||
|
|
||||||
|
if service_account_token:
|
||||||
|
scheme = 'https'
|
||||||
|
request_options['headers'] = {'Authorization': 'Bearer ' + service_account_token}
|
||||||
|
logger.debug('Using service account token for Kubernetes authentication')
|
||||||
|
elif tls_cert and tls_key:
|
||||||
|
scheme = 'https'
|
||||||
|
request_options['cert'] = (tls_cert, tls_key)
|
||||||
|
logger.debug('Using tls certificate and key for Kubernetes authentication')
|
||||||
|
if tls_ca:
|
||||||
|
request_options['verify'] = tls_ca
|
||||||
|
else:
|
||||||
|
scheme = 'http'
|
||||||
|
|
||||||
|
server = self.executor_config.get('K8S_API_SERVER', 'localhost:8080')
|
||||||
|
url = '%s://%s%s' % (scheme, server, path)
|
||||||
|
|
||||||
|
logger.debug('Executor config: %s', self.executor_config)
|
||||||
|
logger.debug('Kubernetes request: %s %s: %s', method, url, request_options)
|
||||||
|
res = requests.request(method, url, **request_options)
|
||||||
|
logger.debug('Kubernetes response: %s: %s', res.status_code, res.text)
|
||||||
|
raise Return(res)
|
||||||
|
|
||||||
|
def _jobs_path(self):
|
||||||
|
return '/apis/batch/v1/namespaces/%s/jobs' % self.namespace
|
||||||
|
|
||||||
|
def _job_path(self, build_uuid):
|
||||||
|
return '%s/%s' % (self._jobs_path(), build_uuid)
|
||||||
|
|
||||||
|
def _kubernetes_distribution(self):
|
||||||
|
return self.executor_config.get('KUBERNETES_DISTRIBUTION', 'basic').lower()
|
||||||
|
|
||||||
|
def _is_basic_kubernetes_distribution(self):
|
||||||
|
return self._kubernetes_distribution() == 'basic'
|
||||||
|
|
||||||
|
def _is_openshift_kubernetes_distribution(self):
|
||||||
|
return self._kubernetes_distribution() == 'openshift'
|
||||||
|
|
||||||
|
def _build_job_container_resources(self):
|
||||||
|
# Minimum acceptable free resources for this container to "fit" in a quota
|
||||||
|
# These may be lower than the absolute limits if the cluster is knowingly
|
||||||
|
# oversubscribed by some amount.
|
||||||
|
container_requests = {
|
||||||
|
'memory' : self.executor_config.get('CONTAINER_MEMORY_REQUEST', '3968Mi'),
|
||||||
|
}
|
||||||
|
|
||||||
|
container_limits = {
|
||||||
|
'memory' : self.executor_config.get('CONTAINER_MEMORY_LIMITS', '5120Mi'),
|
||||||
|
'cpu' : self.executor_config.get('CONTAINER_CPU_LIMITS', '1000m'),
|
||||||
|
}
|
||||||
|
|
||||||
|
resources = {
|
||||||
|
'requests': container_requests,
|
||||||
|
}
|
||||||
|
|
||||||
|
if self._is_openshift_kubernetes_distribution():
|
||||||
|
resources['requests']['cpu'] = self.executor_config.get('CONTAINER_CPU_REQUEST', '500m')
|
||||||
|
resources['limits'] = container_limits
|
||||||
|
|
||||||
|
return resources
|
||||||
|
|
||||||
|
def _build_job_containers(self, user_data):
|
||||||
|
vm_memory_limit = self.executor_config.get('VM_MEMORY_LIMIT', '4G')
|
||||||
|
vm_volume_size = self.executor_config.get('VOLUME_SIZE', '32G')
|
||||||
|
|
||||||
|
container = {
|
||||||
|
'name': 'builder',
|
||||||
|
'imagePullPolicy': 'IfNotPresent',
|
||||||
|
'image': self.image,
|
||||||
|
'securityContext': {'privileged': True},
|
||||||
|
'env': [
|
||||||
|
{'name': 'USERDATA', 'value': user_data},
|
||||||
|
{'name': 'VM_MEMORY', 'value': vm_memory_limit},
|
||||||
|
{'name': 'VM_VOLUME_SIZE', 'value': vm_volume_size},
|
||||||
|
],
|
||||||
|
'resources': self._build_job_container_resources(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if self._is_basic_kubernetes_distribution():
|
||||||
|
container['volumeMounts'] = [{'name': 'secrets-mask','mountPath': '/var/run/secrets/kubernetes.io/serviceaccount'}]
|
||||||
|
|
||||||
|
return container
|
||||||
|
|
||||||
|
def _job_resource(self, build_uuid, user_data, coreos_channel='stable'):
|
||||||
|
image_pull_secret_name = self.executor_config.get('IMAGE_PULL_SECRET_NAME', 'builder')
|
||||||
|
service_account = self.executor_config.get('SERVICE_ACCOUNT_NAME', 'quay-builder-sa')
|
||||||
|
node_selector_label_key = self.executor_config.get('NODE_SELECTOR_LABEL_KEY', 'beta.kubernetes.io/instance-type')
|
||||||
|
node_selector_label_value = self.executor_config.get('NODE_SELECTOR_LABEL_VALUE', '')
|
||||||
|
|
||||||
|
node_selector = {
|
||||||
|
node_selector_label_key : node_selector_label_value
|
||||||
|
}
|
||||||
|
|
||||||
|
release_sha = release.GIT_HEAD or 'none'
|
||||||
|
if ' ' in release_sha:
|
||||||
|
release_sha = 'HEAD'
|
||||||
|
|
||||||
|
job_resource = {
|
||||||
|
'apiVersion': 'batch/v1',
|
||||||
|
'kind': 'Job',
|
||||||
|
'metadata': {
|
||||||
|
'namespace': self.namespace,
|
||||||
|
'generateName': build_uuid + '-',
|
||||||
|
'labels': {
|
||||||
|
'build': build_uuid,
|
||||||
|
'time': datetime.datetime.now().strftime('%Y-%m-%d-%H'),
|
||||||
|
'manager': socket.gethostname(),
|
||||||
|
'quay-sha': release_sha,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'spec' : {
|
||||||
|
'activeDeadlineSeconds': self.executor_config.get('MAXIMUM_JOB_TIME', 7200),
|
||||||
|
'template': {
|
||||||
|
'metadata': {
|
||||||
|
'labels': {
|
||||||
|
'build': build_uuid,
|
||||||
|
'time': datetime.datetime.now().strftime('%Y-%m-%d-%H'),
|
||||||
|
'manager': socket.gethostname(),
|
||||||
|
'quay-sha': release_sha,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'spec': {
|
||||||
|
'imagePullSecrets': [{ 'name': image_pull_secret_name }],
|
||||||
|
'restartPolicy': 'Never',
|
||||||
|
'dnsPolicy': 'Default',
|
||||||
|
'containers': [self._build_job_containers(user_data)],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if self._is_openshift_kubernetes_distribution():
|
||||||
|
# Setting `automountServiceAccountToken` to false will prevent automounting API credentials for a service account.
|
||||||
|
job_resource['spec']['template']['spec']['automountServiceAccountToken'] = False
|
||||||
|
|
||||||
|
# Use dedicated service account that has no authorization to any resources.
|
||||||
|
job_resource['spec']['template']['spec']['serviceAccount'] = service_account
|
||||||
|
|
||||||
|
# Setting `enableServiceLinks` to false prevents information about other services from being injected into pod's
|
||||||
|
# environment variables. Pod has no visibility into other services on the cluster.
|
||||||
|
job_resource['spec']['template']['spec']['enableServiceLinks'] = False
|
||||||
|
|
||||||
|
if node_selector_label_value.strip() != '':
|
||||||
|
job_resource['spec']['template']['spec']['nodeSelector'] = node_selector
|
||||||
|
|
||||||
|
if self._is_basic_kubernetes_distribution():
|
||||||
|
# This volume is a hack to mask the token for the namespace's
|
||||||
|
# default service account, which is placed in a file mounted under
|
||||||
|
# `/var/run/secrets/kubernetes.io/serviceaccount` in all pods.
|
||||||
|
# There's currently no other way to just disable the service
|
||||||
|
# account at either the pod or namespace level.
|
||||||
|
#
|
||||||
|
# https://github.com/kubernetes/kubernetes/issues/16779
|
||||||
|
#
|
||||||
|
job_resource['spec']['template']['spec']['volumes'] = [{'name': 'secrets-mask','emptyDir': {'medium': 'Memory'}}]
|
||||||
|
|
||||||
|
return job_resource
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
@duration_collector_async(metric_queue.builder_time_to_start, ['k8s'])
|
||||||
|
def start_builder(self, realm, token, build_uuid):
|
||||||
|
# generate resource
|
||||||
|
channel = self.executor_config.get('COREOS_CHANNEL', 'stable')
|
||||||
|
user_data = self.generate_cloud_config(realm, token, build_uuid, channel, self.manager_hostname)
|
||||||
|
resource = self._job_resource(build_uuid, user_data, channel)
|
||||||
|
logger.debug('Using Kubernetes Distribution: %s', self._kubernetes_distribution())
|
||||||
|
logger.debug('Generated kubernetes resource:\n%s', resource)
|
||||||
|
|
||||||
|
# schedule
|
||||||
|
create_job = yield From(self._request('POST', self._jobs_path(), json=resource))
|
||||||
|
if int(create_job.status_code / 100) != 2:
|
||||||
|
raise ExecutorException('Failed to create job: %s: %s: %s' %
|
||||||
|
(build_uuid, create_job.status_code, create_job.text))
|
||||||
|
|
||||||
|
job = create_job.json()
|
||||||
|
raise Return(job['metadata']['name'])
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def stop_builder(self, builder_id):
|
||||||
|
pods_path = '/api/v1/namespaces/%s/pods' % self.namespace
|
||||||
|
|
||||||
|
# Delete the job itself.
|
||||||
|
try:
|
||||||
|
yield From(self._request('DELETE', self._job_path(builder_id)))
|
||||||
|
except:
|
||||||
|
logger.exception('Failed to send delete job call for job %s', builder_id)
|
||||||
|
|
||||||
|
# Delete the pod(s) for the job.
|
||||||
|
selectorString = "job-name=%s" % builder_id
|
||||||
|
try:
|
||||||
|
yield From(self._request('DELETE', pods_path, params=dict(labelSelector=selectorString)))
|
||||||
|
except:
|
||||||
|
logger.exception("Failed to send delete pod call for job %s", builder_id)
|
||||||
|
|
||||||
|
|
||||||
|
class LogPipe(threading.Thread):
|
||||||
|
""" Adapted from http://codereview.stackexchange.com/a/17959
|
||||||
|
"""
|
||||||
|
def __init__(self, level):
|
||||||
|
"""Setup the object with a logger and a loglevel
|
||||||
|
and start the thread
|
||||||
|
"""
|
||||||
|
threading.Thread.__init__(self)
|
||||||
|
self.daemon = False
|
||||||
|
self.level = level
|
||||||
|
self.fd_read, self.fd_write = os.pipe()
|
||||||
|
self.pipe_reader = os.fdopen(self.fd_read)
|
||||||
|
self.start()
|
||||||
|
|
||||||
|
def fileno(self):
|
||||||
|
"""Return the write file descriptor of the pipe
|
||||||
|
"""
|
||||||
|
return self.fd_write
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
"""Run the thread, logging everything.
|
||||||
|
"""
|
||||||
|
for line in iter(self.pipe_reader.readline, ''):
|
||||||
|
logging.log(self.level, line.strip('\n'))
|
||||||
|
|
||||||
|
self.pipe_reader.close()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
"""Close the write end of the pipe.
|
||||||
|
"""
|
||||||
|
os.close(self.fd_write)
|
||||||
8
buildman/manager/noop_canceller.py
Normal file
8
buildman/manager/noop_canceller.py
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
class NoopCanceller(object):
|
||||||
|
""" A class that can not cancel a build """
|
||||||
|
def __init__(self, config=None):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def try_cancel_build(self, uuid):
|
||||||
|
""" Does nothing and fails to cancel build. """
|
||||||
|
return False
|
||||||
26
buildman/manager/orchestrator_canceller.py
Normal file
26
buildman/manager/orchestrator_canceller.py
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from buildman.orchestrator import orchestrator_from_config, OrchestratorError
|
||||||
|
from util import slash_join
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
CANCEL_PREFIX = 'cancel/'
|
||||||
|
|
||||||
|
|
||||||
|
class OrchestratorCanceller(object):
|
||||||
|
""" An asynchronous way to cancel a build with any Orchestrator. """
|
||||||
|
def __init__(self, config):
|
||||||
|
self._orchestrator = orchestrator_from_config(config, canceller_only=True)
|
||||||
|
|
||||||
|
def try_cancel_build(self, build_uuid):
|
||||||
|
logger.info('Cancelling build %s', build_uuid)
|
||||||
|
cancel_key = slash_join(CANCEL_PREFIX, build_uuid)
|
||||||
|
try:
|
||||||
|
self._orchestrator.set_key_sync(cancel_key, build_uuid, expiration=60)
|
||||||
|
return True
|
||||||
|
except OrchestratorError:
|
||||||
|
logger.exception('Failed to write cancel action to redis with uuid %s', build_uuid)
|
||||||
|
return False
|
||||||
753
buildman/orchestrator.py
Normal file
753
buildman/orchestrator.py
Normal file
|
|
@ -0,0 +1,753 @@
|
||||||
|
from abc import ABCMeta, abstractmethod
|
||||||
|
from collections import namedtuple
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
from enum import IntEnum, unique
|
||||||
|
from six import add_metaclass, iteritems
|
||||||
|
from trollius import async, coroutine, From, Return
|
||||||
|
from urllib3.exceptions import ReadTimeoutError, ProtocolError
|
||||||
|
|
||||||
|
import etcd
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from buildman.asyncutil import wrap_with_threadpool
|
||||||
|
from util import slash_join
|
||||||
|
from util.expiresdict import ExpiresDict
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
ONE_DAY = 60 * 60 * 24
|
||||||
|
ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION = 5
|
||||||
|
DEFAULT_LOCK_EXPIRATION = 10000
|
||||||
|
|
||||||
|
ETCD_READ_TIMEOUT = 5
|
||||||
|
ETCD_MAX_WATCH_TIMEOUT = 30
|
||||||
|
|
||||||
|
REDIS_EXPIRING_SUFFIX = '/expiring'
|
||||||
|
REDIS_DEFAULT_PUBSUB_KEY = 'orchestrator_events'
|
||||||
|
REDIS_EVENT_KIND_MESSAGE = 'message'
|
||||||
|
REDIS_EVENT_KIND_PMESSAGE = 'pmessage'
|
||||||
|
REDIS_NONEXPIRING_KEY = -1
|
||||||
|
|
||||||
|
# This constant defines the Redis configuration flags used to watch [K]eyspace and e[x]pired
|
||||||
|
# events on keys. For more info, see https://redis.io/topics/notifications#configuration
|
||||||
|
REDIS_KEYSPACE_EVENT_CONFIG_VALUE = 'Kx'
|
||||||
|
REDIS_KEYSPACE_EVENT_CONFIG_KEY = 'notify-keyspace-events'
|
||||||
|
REDIS_KEYSPACE_KEY_PATTERN = '__keyspace@%s__:%s'
|
||||||
|
REDIS_EXPIRED_KEYSPACE_PATTERN = slash_join(REDIS_KEYSPACE_KEY_PATTERN, REDIS_EXPIRING_SUFFIX)
|
||||||
|
REDIS_EXPIRED_KEYSPACE_REGEX = re.compile(REDIS_EXPIRED_KEYSPACE_PATTERN % (r'(\S+)', r'(\S+)'))
|
||||||
|
|
||||||
|
|
||||||
|
def orchestrator_from_config(manager_config, canceller_only=False):
|
||||||
|
"""
|
||||||
|
Allocates a new Orchestrator from the 'ORCHESTRATOR' block from provided manager config.
|
||||||
|
Checks for legacy configuration prefixed with 'ETCD_' when the 'ORCHESTRATOR' is not present.
|
||||||
|
|
||||||
|
:param manager_config: the configuration for the orchestrator
|
||||||
|
:type manager_config: dict
|
||||||
|
:rtype: :class: Orchestrator
|
||||||
|
"""
|
||||||
|
# Legacy codepath only knows how to configure etcd.
|
||||||
|
if manager_config.get('ORCHESTRATOR') is None:
|
||||||
|
manager_config['ORCHESTRATOR'] = {key: value
|
||||||
|
for (key, value) in iteritems(manager_config)
|
||||||
|
if key.startswith('ETCD_') and not key.endswith('_PREFIX')}
|
||||||
|
|
||||||
|
# Sanity check that legacy prefixes are no longer being used.
|
||||||
|
for key in manager_config['ORCHESTRATOR'].keys():
|
||||||
|
words = key.split('_')
|
||||||
|
if len(words) > 1 and words[-1].lower() == 'prefix':
|
||||||
|
raise AssertionError('legacy prefix used, use ORCHESTRATOR_PREFIX instead')
|
||||||
|
|
||||||
|
def _dict_key_prefix(d):
|
||||||
|
"""
|
||||||
|
:param d: the dict that has keys prefixed with underscore
|
||||||
|
:type d: {str: any}
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
return d.keys()[0].split('_', 1)[0].lower()
|
||||||
|
|
||||||
|
orchestrator_name = _dict_key_prefix(manager_config['ORCHESTRATOR'])
|
||||||
|
|
||||||
|
def format_key(key):
|
||||||
|
return key.lower().split('_', 1)[1]
|
||||||
|
|
||||||
|
orchestrator_kwargs = {format_key(key): value
|
||||||
|
for (key, value) in iteritems(manager_config['ORCHESTRATOR'])}
|
||||||
|
|
||||||
|
if manager_config.get('ORCHESTRATOR_PREFIX') is not None:
|
||||||
|
orchestrator_kwargs['orchestrator_prefix'] = manager_config['ORCHESTRATOR_PREFIX']
|
||||||
|
|
||||||
|
orchestrator_kwargs['canceller_only'] = canceller_only
|
||||||
|
|
||||||
|
logger.debug('attempting to create orchestrator %s with kwargs %s',
|
||||||
|
orchestrator_name, orchestrator_kwargs)
|
||||||
|
return orchestrator_by_name(orchestrator_name, **orchestrator_kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def orchestrator_by_name(name, **kwargs):
|
||||||
|
_ORCHESTRATORS = {
|
||||||
|
'etcd': Etcd2Orchestrator,
|
||||||
|
'mem': MemoryOrchestrator,
|
||||||
|
'redis': RedisOrchestrator,
|
||||||
|
}
|
||||||
|
return _ORCHESTRATORS.get(name, MemoryOrchestrator)(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class OrchestratorError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: replace with ConnectionError when this codebase is Python 3.
|
||||||
|
class OrchestratorConnectionError(OrchestratorError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@unique
|
||||||
|
class KeyEvent(IntEnum):
|
||||||
|
CREATE = 1
|
||||||
|
SET = 2
|
||||||
|
DELETE = 3
|
||||||
|
EXPIRE = 4
|
||||||
|
|
||||||
|
|
||||||
|
class KeyChange(namedtuple('KeyChange', ['event', 'key', 'value'])):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@add_metaclass(ABCMeta)
|
||||||
|
class Orchestrator(object):
|
||||||
|
"""
|
||||||
|
Orchestrator is the interface that is used to synchronize the build states
|
||||||
|
across build managers.
|
||||||
|
|
||||||
|
This interface assumes that storage is being done by a key-value store
|
||||||
|
that supports watching for events on keys.
|
||||||
|
|
||||||
|
Missing keys should return KeyError; otherwise, errors should raise an
|
||||||
|
OrchestratorError.
|
||||||
|
|
||||||
|
:param key_prefix: the prefix of keys being watched
|
||||||
|
:type key_prefix: str
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def on_key_change(self, key, callback, restarter=None):
|
||||||
|
"""
|
||||||
|
|
||||||
|
The callback parameter takes in a KeyChange object as a parameter.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_prefixed_keys(self, prefix):
|
||||||
|
"""
|
||||||
|
|
||||||
|
:returns: a dict of key value pairs beginning with prefix
|
||||||
|
:rtype: {str: str}
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_key(self, key):
|
||||||
|
"""
|
||||||
|
|
||||||
|
:returns: the value stored at the provided key
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def set_key(self, key, value, overwrite=False, expiration=None):
|
||||||
|
"""
|
||||||
|
|
||||||
|
:param key: the identifier for the value
|
||||||
|
:type key: str
|
||||||
|
:param value: the value being stored
|
||||||
|
:type value: str
|
||||||
|
:param overwrite: whether or not a KeyError is thrown if the key already exists
|
||||||
|
:type overwrite: bool
|
||||||
|
:param expiration: the duration in seconds that a key should be available
|
||||||
|
:type expiration: int
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def set_key_sync(self, key, value, overwrite=False, expiration=None):
|
||||||
|
"""
|
||||||
|
set_key, but without trollius coroutines.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete_key(self, key):
|
||||||
|
"""
|
||||||
|
Deletes a key that has been set in the orchestrator.
|
||||||
|
|
||||||
|
:param key: the identifier for the key
|
||||||
|
:type key: str
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
|
||||||
|
"""
|
||||||
|
Takes a lock for synchronizing exclusive operations cluster-wide.
|
||||||
|
|
||||||
|
:param key: the identifier for the lock
|
||||||
|
:type key: str
|
||||||
|
:param expiration: the duration until the lock expires
|
||||||
|
:type expiration: :class:`datetime.timedelta` or int (seconds)
|
||||||
|
:returns: whether or not the lock was acquired
|
||||||
|
:rtype: bool
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def shutdown():
|
||||||
|
"""
|
||||||
|
This function should shutdown any final resources allocated by the Orchestrator.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _sleep_orchestrator():
|
||||||
|
"""
|
||||||
|
This function blocks the trollius event loop by sleeping in order to backoff if a failure
|
||||||
|
such as a ConnectionError has occurred.
|
||||||
|
"""
|
||||||
|
logger.exception('Connecting to etcd failed; sleeping for %s and then trying again',
|
||||||
|
ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION)
|
||||||
|
time.sleep(ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION)
|
||||||
|
logger.exception('Connecting to etcd failed; slept for %s and now trying again',
|
||||||
|
ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION)
|
||||||
|
|
||||||
|
|
||||||
|
class EtcdAction(object):
|
||||||
|
""" Enumeration of the various kinds of etcd actions we can observe via a watch. """
|
||||||
|
GET = 'get'
|
||||||
|
SET = 'set'
|
||||||
|
EXPIRE = 'expire'
|
||||||
|
UPDATE = 'update'
|
||||||
|
DELETE = 'delete'
|
||||||
|
CREATE = 'create'
|
||||||
|
COMPARE_AND_SWAP = 'compareAndSwap'
|
||||||
|
COMPARE_AND_DELETE = 'compareAndDelete'
|
||||||
|
|
||||||
|
|
||||||
|
class Etcd2Orchestrator(Orchestrator):
|
||||||
|
def __init__(self, host='127.0.0.1', port=2379, cert_and_key=None, ca_cert=None,
|
||||||
|
client_threads=5, canceller_only=False, **kwargs):
|
||||||
|
self.is_canceller_only = canceller_only
|
||||||
|
|
||||||
|
logger.debug('initializing async etcd client')
|
||||||
|
self._sync_etcd_client = etcd.Client(
|
||||||
|
host=host,
|
||||||
|
port=port,
|
||||||
|
cert=tuple(cert_and_key) if cert_and_key is not None else None,
|
||||||
|
ca_cert=ca_cert,
|
||||||
|
protocol='http' if cert_and_key is None else 'https',
|
||||||
|
read_timeout=ETCD_READ_TIMEOUT,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not self.is_canceller_only:
|
||||||
|
(self._etcd_client, self._async_executor) = wrap_with_threadpool(self._sync_etcd_client,
|
||||||
|
client_threads)
|
||||||
|
|
||||||
|
logger.debug('creating initial orchestrator state')
|
||||||
|
self._shutting_down = False
|
||||||
|
self._watch_tasks = {}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _sanity_check_ttl(ttl):
|
||||||
|
"""
|
||||||
|
A TTL of < 0 in etcd results in the key *never being expired*.
|
||||||
|
We use a max here to ensure that if the TTL is < 0, the key will expire immediately.
|
||||||
|
"""
|
||||||
|
return max(ttl, 0)
|
||||||
|
|
||||||
|
def _watch_etcd(self, key, callback, restarter=None, start_index=None):
|
||||||
|
def callback_wrapper(changed_key_future):
|
||||||
|
new_index = start_index
|
||||||
|
etcd_result = None
|
||||||
|
|
||||||
|
if not changed_key_future.cancelled():
|
||||||
|
try:
|
||||||
|
etcd_result = changed_key_future.result()
|
||||||
|
existing_index = getattr(etcd_result, 'etcd_index', None)
|
||||||
|
new_index = etcd_result.modifiedIndex + 1
|
||||||
|
|
||||||
|
logger.debug('Got watch of key: %s at #%s with result: %s',
|
||||||
|
key, existing_index, etcd_result)
|
||||||
|
|
||||||
|
except ReadTimeoutError:
|
||||||
|
logger.debug('Read-timeout on etcd watch %s, rescheduling', key)
|
||||||
|
|
||||||
|
except etcd.EtcdEventIndexCleared:
|
||||||
|
# This happens if etcd2 has moved forward too fast for us to start watching at the index
|
||||||
|
# we retrieved. We therefore start a new watch at HEAD and (if specified) call the
|
||||||
|
# restarter method which should conduct a read and reset the state of the manager.
|
||||||
|
logger.debug('Etcd moved forward too quickly. Restarting watch cycle.')
|
||||||
|
new_index = None
|
||||||
|
if restarter is not None:
|
||||||
|
async(restarter())
|
||||||
|
|
||||||
|
except (KeyError, etcd.EtcdKeyError):
|
||||||
|
logger.debug('Etcd key already cleared: %s', key)
|
||||||
|
return
|
||||||
|
|
||||||
|
except etcd.EtcdConnectionFailed:
|
||||||
|
_sleep_orchestrator()
|
||||||
|
|
||||||
|
except etcd.EtcdException as eex:
|
||||||
|
# TODO: This is a quick and dirty hack and should be replaced with a proper
|
||||||
|
# exception check.
|
||||||
|
if str(eex.message).find('Read timed out') >= 0:
|
||||||
|
logger.debug('Read-timeout on etcd watch %s, rescheduling', key)
|
||||||
|
else:
|
||||||
|
logger.exception('Exception on etcd watch: %s', key)
|
||||||
|
|
||||||
|
except ProtocolError:
|
||||||
|
logger.exception('Exception on etcd watch: %s', key)
|
||||||
|
|
||||||
|
if key not in self._watch_tasks or self._watch_tasks[key].done():
|
||||||
|
self._watch_etcd(key, callback, start_index=new_index, restarter=restarter)
|
||||||
|
|
||||||
|
if etcd_result and etcd_result.value is not None:
|
||||||
|
async(callback(self._etcd_result_to_keychange(etcd_result)))
|
||||||
|
|
||||||
|
if not self._shutting_down:
|
||||||
|
logger.debug('Scheduling watch of key: %s at start index %s', key, start_index)
|
||||||
|
watch_future = self._etcd_client.watch(key, recursive=True, index=start_index,
|
||||||
|
timeout=ETCD_MAX_WATCH_TIMEOUT)
|
||||||
|
watch_future.add_done_callback(callback_wrapper)
|
||||||
|
|
||||||
|
self._watch_tasks[key] = async(watch_future)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _etcd_result_to_keychange(etcd_result):
|
||||||
|
event = Etcd2Orchestrator._etcd_result_to_keyevent(etcd_result)
|
||||||
|
return KeyChange(event, etcd_result.key, etcd_result.value)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _etcd_result_to_keyevent(etcd_result):
|
||||||
|
if etcd_result.action == EtcdAction.CREATE:
|
||||||
|
return KeyEvent.CREATE
|
||||||
|
if etcd_result.action == EtcdAction.SET:
|
||||||
|
return KeyEvent.CREATE if etcd_result.createdIndex == etcd_result.modifiedIndex else KeyEvent.SET
|
||||||
|
if etcd_result.action == EtcdAction.DELETE:
|
||||||
|
return KeyEvent.DELETE
|
||||||
|
if etcd_result.action == EtcdAction.EXPIRE:
|
||||||
|
return KeyEvent.EXPIRE
|
||||||
|
raise AssertionError('etcd action must have equivalant keyevent')
|
||||||
|
|
||||||
|
def on_key_change(self, key, callback, restarter=None):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
logger.debug('creating watch on %s', key)
|
||||||
|
self._watch_etcd(key, callback, restarter=restarter)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def get_prefixed_keys(self, prefix):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
try:
|
||||||
|
etcd_result = yield From(self._etcd_client.read(prefix, recursive=True))
|
||||||
|
raise Return({leaf.key: leaf.value for leaf in etcd_result.leaves})
|
||||||
|
except etcd.EtcdKeyError:
|
||||||
|
raise KeyError
|
||||||
|
except etcd.EtcdConnectionFailed as ex:
|
||||||
|
raise OrchestratorConnectionError(ex)
|
||||||
|
except etcd.EtcdException as ex:
|
||||||
|
raise OrchestratorError(ex)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def get_key(self, key):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Ignore pylint: the value property on EtcdResult is added dynamically using setattr.
|
||||||
|
etcd_result = yield From(self._etcd_client.read(key))
|
||||||
|
raise Return(etcd_result.value)
|
||||||
|
except etcd.EtcdKeyError:
|
||||||
|
raise KeyError
|
||||||
|
except etcd.EtcdConnectionFailed as ex:
|
||||||
|
raise OrchestratorConnectionError(ex)
|
||||||
|
except etcd.EtcdException as ex:
|
||||||
|
raise OrchestratorError(ex)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def set_key(self, key, value, overwrite=False, expiration=None):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
yield From(self._etcd_client.write(key, value, prevExists=overwrite,
|
||||||
|
ttl=self._sanity_check_ttl(expiration)))
|
||||||
|
|
||||||
|
def set_key_sync(self, key, value, overwrite=False, expiration=None):
|
||||||
|
self._sync_etcd_client.write(key, value, prevExists=overwrite,
|
||||||
|
ttl=self._sanity_check_ttl(expiration))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def delete_key(self, key):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield From(self._etcd_client.delete(key))
|
||||||
|
except etcd.EtcdKeyError:
|
||||||
|
raise KeyError
|
||||||
|
except etcd.EtcdConnectionFailed as ex:
|
||||||
|
raise OrchestratorConnectionError(ex)
|
||||||
|
except etcd.EtcdException as ex:
|
||||||
|
raise OrchestratorError(ex)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield From(self._etcd_client.write(key, {}, prevExist=False,
|
||||||
|
ttl=self._sanity_check_ttl(expiration)))
|
||||||
|
raise Return(True)
|
||||||
|
except (KeyError, etcd.EtcdKeyError):
|
||||||
|
raise Return(False)
|
||||||
|
except etcd.EtcdConnectionFailed:
|
||||||
|
logger.exception('Could not get etcd atomic lock as etcd is down')
|
||||||
|
raise Return(False)
|
||||||
|
except etcd.EtcdException as ex:
|
||||||
|
raise OrchestratorError(ex)
|
||||||
|
|
||||||
|
def shutdown(self):
|
||||||
|
logger.debug('Shutting down etcd client.')
|
||||||
|
self._shutting_down = True
|
||||||
|
|
||||||
|
if self.is_canceller_only:
|
||||||
|
return
|
||||||
|
|
||||||
|
for (key, _), task in self._watch_tasks.items():
|
||||||
|
if not task.done():
|
||||||
|
logger.debug('Canceling watch task for %s', key)
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
if self._async_executor is not None:
|
||||||
|
self._async_executor.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
class MemoryOrchestrator(Orchestrator):
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
self.state = ExpiresDict()
|
||||||
|
self.callbacks = {}
|
||||||
|
|
||||||
|
def _callbacks_prefixed(self, prefix):
|
||||||
|
return (callback for (key, callback) in iteritems(self.callbacks)
|
||||||
|
if key.startswith(prefix))
|
||||||
|
|
||||||
|
def on_key_change(self, key, callback, restarter=None):
|
||||||
|
self.callbacks[key] = callback
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def get_prefixed_keys(self, prefix):
|
||||||
|
raise Return({k: value for (k, value) in self.state.items()
|
||||||
|
if k.startswith(prefix)})
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def get_key(self, key):
|
||||||
|
raise Return(self.state[key])
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def set_key(self, key, value, overwrite=False, expiration=None):
|
||||||
|
preexisting_key = 'key' in self.state
|
||||||
|
if preexisting_key and not overwrite:
|
||||||
|
raise KeyError
|
||||||
|
|
||||||
|
absolute_expiration = None
|
||||||
|
if expiration is not None:
|
||||||
|
absolute_expiration = datetime.datetime.now() + datetime.timedelta(seconds=expiration)
|
||||||
|
|
||||||
|
self.state.set(key, value, expires=absolute_expiration)
|
||||||
|
|
||||||
|
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
|
||||||
|
for callback in self._callbacks_prefixed(key):
|
||||||
|
yield From(callback(KeyChange(event, key, value)))
|
||||||
|
|
||||||
|
def set_key_sync(self, key, value, overwrite=False, expiration=None):
|
||||||
|
"""
|
||||||
|
set_key, but without trollius coroutines.
|
||||||
|
"""
|
||||||
|
preexisting_key = 'key' in self.state
|
||||||
|
if preexisting_key and not overwrite:
|
||||||
|
raise KeyError
|
||||||
|
|
||||||
|
absolute_expiration = None
|
||||||
|
if expiration is not None:
|
||||||
|
absolute_expiration = datetime.datetime.now() + datetime.timedelta(seconds=expiration)
|
||||||
|
|
||||||
|
self.state.set(key, value, expires=absolute_expiration)
|
||||||
|
|
||||||
|
event = KeyEvent.CREATE if not preexisting_key else KeyEvent.SET
|
||||||
|
for callback in self._callbacks_prefixed(key):
|
||||||
|
callback(KeyChange(event, key, value))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def delete_key(self, key):
|
||||||
|
value = self.state[key]
|
||||||
|
del self.state[key]
|
||||||
|
|
||||||
|
for callback in self._callbacks_prefixed(key):
|
||||||
|
yield From(callback(KeyChange(KeyEvent.DELETE, key, value)))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
|
||||||
|
if key in self.state:
|
||||||
|
raise Return(False)
|
||||||
|
self.state.set(key, None, expires=expiration)
|
||||||
|
raise Return(True)
|
||||||
|
|
||||||
|
def shutdown(self):
|
||||||
|
self.state = None
|
||||||
|
self.callbacks = None
|
||||||
|
|
||||||
|
|
||||||
|
class RedisOrchestrator(Orchestrator):
|
||||||
|
def __init__(self, host='127.0.0.1', port=6379, password=None, db=0, cert_and_key=None,
|
||||||
|
ca_cert=None, client_threads=5, ssl=False, skip_keyspace_event_setup=False,
|
||||||
|
canceller_only=False, **kwargs):
|
||||||
|
self.is_canceller_only = canceller_only
|
||||||
|
(cert, key) = tuple(cert_and_key) if cert_and_key is not None else (None, None)
|
||||||
|
self._sync_client = redis.StrictRedis(
|
||||||
|
host=host,
|
||||||
|
port=port,
|
||||||
|
password=password,
|
||||||
|
db=db,
|
||||||
|
ssl_certfile=cert,
|
||||||
|
ssl_keyfile=key,
|
||||||
|
ssl_ca_certs=ca_cert,
|
||||||
|
ssl=ssl,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._shutting_down = False
|
||||||
|
self._tasks = {}
|
||||||
|
self._watched_keys = {}
|
||||||
|
self._pubsub_key = slash_join(kwargs.get('orchestrator_prefix', ''),
|
||||||
|
REDIS_DEFAULT_PUBSUB_KEY).lstrip('/')
|
||||||
|
|
||||||
|
if not self.is_canceller_only:
|
||||||
|
(self._client, self._async_executor) = wrap_with_threadpool(self._sync_client, client_threads)
|
||||||
|
|
||||||
|
# Configure a subscription to watch events that the orchestrator manually publishes.
|
||||||
|
logger.debug('creating pubsub with key %s', self._pubsub_key)
|
||||||
|
published_pubsub = self._sync_client.pubsub()
|
||||||
|
published_pubsub.subscribe(self._pubsub_key)
|
||||||
|
(self._pubsub, self._async_executor_pub) = wrap_with_threadpool(published_pubsub)
|
||||||
|
self._watch_published_key()
|
||||||
|
|
||||||
|
# Configure a subscription to watch expired keyspace events.
|
||||||
|
if not skip_keyspace_event_setup:
|
||||||
|
self._sync_client.config_set(REDIS_KEYSPACE_EVENT_CONFIG_KEY,
|
||||||
|
REDIS_KEYSPACE_EVENT_CONFIG_VALUE)
|
||||||
|
|
||||||
|
expiring_pubsub = self._sync_client.pubsub()
|
||||||
|
expiring_pubsub.psubscribe(REDIS_EXPIRED_KEYSPACE_PATTERN % (db, '*'))
|
||||||
|
(self._pubsub_expiring, self._async_executor_ex) = wrap_with_threadpool(expiring_pubsub)
|
||||||
|
self._watch_expiring_key()
|
||||||
|
|
||||||
|
def _watch_published_key(self):
|
||||||
|
def published_callback_wrapper(event_future):
|
||||||
|
logger.debug('published callback called')
|
||||||
|
event_result = None
|
||||||
|
|
||||||
|
if not event_future.cancelled():
|
||||||
|
try:
|
||||||
|
event_result = event_future.result()
|
||||||
|
(redis_event, event_key, event_value) = event_result
|
||||||
|
logger.debug('Got watch of key: (%s, %s, %s)', redis_event, event_key, event_value)
|
||||||
|
except redis.ConnectionError:
|
||||||
|
_sleep_orchestrator()
|
||||||
|
except redis.RedisError:
|
||||||
|
logger.exception('Exception watching redis publish: %s', event_key)
|
||||||
|
|
||||||
|
# Schedule creating a new future if this one has been consumed.
|
||||||
|
if 'pub' not in self._tasks or self._tasks['pub'].done():
|
||||||
|
self._watch_published_key()
|
||||||
|
|
||||||
|
if event_result is not None and redis_event == REDIS_EVENT_KIND_MESSAGE:
|
||||||
|
keychange = self._publish_to_keychange(event_value)
|
||||||
|
for watched_key, callback in iteritems(self._watched_keys):
|
||||||
|
if keychange.key.startswith(watched_key):
|
||||||
|
async(callback(keychange))
|
||||||
|
|
||||||
|
if not self._shutting_down:
|
||||||
|
logger.debug('Scheduling watch of publish stream')
|
||||||
|
watch_future = self._pubsub.parse_response()
|
||||||
|
watch_future.add_done_callback(published_callback_wrapper)
|
||||||
|
self._tasks['pub'] = async(watch_future)
|
||||||
|
|
||||||
|
def _watch_expiring_key(self):
|
||||||
|
def expiring_callback_wrapper(event_future):
|
||||||
|
logger.debug('expiring callback called')
|
||||||
|
event_result = None
|
||||||
|
|
||||||
|
if not event_future.cancelled():
|
||||||
|
try:
|
||||||
|
event_result = event_future.result()
|
||||||
|
if self._is_expired_keyspace_event(event_result):
|
||||||
|
# Get the value of the original key before the expiration happened.
|
||||||
|
key = self._key_from_expiration(event_future)
|
||||||
|
expired_value = yield From(self._client.get(key))
|
||||||
|
|
||||||
|
# $KEY/expiring is gone, but the original key still remains, set an expiration for it
|
||||||
|
# so that other managers have time to get the event and still read the expired value.
|
||||||
|
yield From(self._client.expire(key, ONE_DAY))
|
||||||
|
except redis.ConnectionError:
|
||||||
|
_sleep_orchestrator()
|
||||||
|
except redis.RedisError:
|
||||||
|
logger.exception('Exception watching redis expirations: %s', key)
|
||||||
|
|
||||||
|
# Schedule creating a new future if this one has been consumed.
|
||||||
|
if 'expire' not in self._tasks or self._tasks['expire'].done():
|
||||||
|
self._watch_expiring_key()
|
||||||
|
|
||||||
|
if self._is_expired_keyspace_event(event_result) and expired_value is not None:
|
||||||
|
for watched_key, callback in iteritems(self._watched_keys):
|
||||||
|
if key.startswith(watched_key):
|
||||||
|
async(callback(KeyChange(KeyEvent.EXPIRE, key, expired_value)))
|
||||||
|
|
||||||
|
if not self._shutting_down:
|
||||||
|
logger.debug('Scheduling watch of expiration')
|
||||||
|
watch_future = self._pubsub_expiring.parse_response()
|
||||||
|
watch_future.add_done_callback(expiring_callback_wrapper)
|
||||||
|
self._tasks['expire'] = async(watch_future)
|
||||||
|
|
||||||
|
def on_key_change(self, key, callback, restarter=None):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
logger.debug('watching key: %s', key)
|
||||||
|
self._watched_keys[key] = callback
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_expired_keyspace_event(event_result):
|
||||||
|
"""
|
||||||
|
Sanity check that this isn't an unrelated keyspace event.
|
||||||
|
There could be a more efficient keyspace event config to avoid this client-side filter.
|
||||||
|
"""
|
||||||
|
if event_result is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
(redis_event, _pattern, matched_key, expired) = event_result
|
||||||
|
return (redis_event == REDIS_EVENT_KIND_PMESSAGE and
|
||||||
|
expired == 'expired' and
|
||||||
|
REDIS_EXPIRED_KEYSPACE_REGEX.match(matched_key) is not None)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _key_from_expiration(event_result):
|
||||||
|
(_redis_event, _pattern, matched_key, _expired) = event_result
|
||||||
|
return REDIS_EXPIRED_KEYSPACE_REGEX.match(matched_key).groups()[1]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _publish_to_keychange(event_value):
|
||||||
|
e = json.loads(event_value)
|
||||||
|
return KeyChange(KeyEvent(e['event']), e['key'], e['value'])
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def get_prefixed_keys(self, prefix):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
# TODO: This can probably be done with redis pipelines to make it transactional.
|
||||||
|
keys = yield From(self._client.keys(prefix + '*'))
|
||||||
|
|
||||||
|
# Yielding to the event loop is required, thus this cannot be written as a dict comprehension.
|
||||||
|
results = {}
|
||||||
|
for key in keys:
|
||||||
|
if key.endswith(REDIS_EXPIRING_SUFFIX):
|
||||||
|
continue
|
||||||
|
ttl = yield From(self._client.ttl(key))
|
||||||
|
if ttl != REDIS_NONEXPIRING_KEY:
|
||||||
|
# Only redis keys without expirations are live build manager keys.
|
||||||
|
value = yield From(self._client.get(key))
|
||||||
|
results.update({key: value})
|
||||||
|
|
||||||
|
raise Return(results)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def get_key(self, key):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
value = yield From(self._client.get(key))
|
||||||
|
raise Return(value)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def set_key(self, key, value, overwrite=False, expiration=None):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
already_exists = yield From(self._client.exists(key))
|
||||||
|
|
||||||
|
yield From(self._client.set(key, value, xx=overwrite))
|
||||||
|
if expiration is not None:
|
||||||
|
yield From(self._client.set(slash_join(key, REDIS_EXPIRING_SUFFIX), value,
|
||||||
|
xx=overwrite, ex=expiration))
|
||||||
|
|
||||||
|
key_event = KeyEvent.SET if already_exists else KeyEvent.CREATE
|
||||||
|
yield From(self._publish(event=key_event, key=key, value=value))
|
||||||
|
|
||||||
|
def set_key_sync(self, key, value, overwrite=False, expiration=None):
|
||||||
|
already_exists = self._sync_client.exists(key)
|
||||||
|
|
||||||
|
self._sync_client.set(key, value, xx=overwrite)
|
||||||
|
if expiration is not None:
|
||||||
|
self._sync_client.set(slash_join(key, REDIS_EXPIRING_SUFFIX), value,
|
||||||
|
xx=overwrite, ex=expiration)
|
||||||
|
|
||||||
|
self._sync_client.publish(self._pubsub_key, json.dumps({
|
||||||
|
'event': int(KeyEvent.SET if already_exists else KeyEvent.CREATE),
|
||||||
|
'key': key,
|
||||||
|
'value': value,
|
||||||
|
}))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _publish(self, **kwargs):
|
||||||
|
kwargs['event'] = int(kwargs['event'])
|
||||||
|
event_json = json.dumps(kwargs)
|
||||||
|
logger.debug('publishing event: %s', event_json)
|
||||||
|
yield From(self._client.publish(self._pubsub_key, event_json))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def delete_key(self, key):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
value = yield From(self._client.get(key))
|
||||||
|
yield From(self._client.delete(key))
|
||||||
|
yield From(self._client.delete(slash_join(key, REDIS_EXPIRING_SUFFIX)))
|
||||||
|
yield From(self._publish(event=KeyEvent.DELETE, key=key, value=value))
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def lock(self, key, expiration=DEFAULT_LOCK_EXPIRATION):
|
||||||
|
assert not self.is_canceller_only
|
||||||
|
|
||||||
|
yield From(self.set_key(key, '', ex=expiration))
|
||||||
|
raise Return(True)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def shutdown(self):
|
||||||
|
logger.debug('Shutting down redis client.')
|
||||||
|
|
||||||
|
self._shutting_down = True
|
||||||
|
|
||||||
|
if self.is_canceller_only:
|
||||||
|
return
|
||||||
|
|
||||||
|
for key, task in iteritems(self._tasks):
|
||||||
|
if not task.done():
|
||||||
|
logger.debug('Canceling watch task for %s', key)
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
if self._async_executor is not None:
|
||||||
|
self._async_executor.shutdown()
|
||||||
|
if self._async_executor_ex is not None:
|
||||||
|
self._async_executor_ex.shutdown()
|
||||||
|
if self._async_executor_pub is not None:
|
||||||
|
self._async_executor_pub.shutdown()
|
||||||
26
buildman/qemu-coreos/Dockerfile
Normal file
26
buildman/qemu-coreos/Dockerfile
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
FROM debian
|
||||||
|
|
||||||
|
RUN apt-get clean && apt-get update && apt-get upgrade -y # 03APR2017
|
||||||
|
RUN apt-get install -y \
|
||||||
|
bzip2 \
|
||||||
|
curl \
|
||||||
|
openssh-client \
|
||||||
|
qemu-kvm
|
||||||
|
|
||||||
|
ARG channel=stable
|
||||||
|
ARG version=current
|
||||||
|
|
||||||
|
RUN echo "Downloading http://${channel}.release.core-os.net/amd64-usr/${version}/coreos_production_qemu_image.img.bz2"
|
||||||
|
RUN curl -s -O http://${channel}.release.core-os.net/amd64-usr/${version}/coreos_production_qemu_image.img.bz2 && \
|
||||||
|
bzip2 -d coreos_production_qemu_image.img.bz2
|
||||||
|
|
||||||
|
RUN apt-get remove -y curl bzip2 && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||||
|
|
||||||
|
COPY start.sh /start.sh
|
||||||
|
|
||||||
|
LABEL com.coreos.channel ${channel}
|
||||||
|
LABEL com.coreos.version ${version}
|
||||||
|
|
||||||
|
ENTRYPOINT ["/bin/bash", "/start.sh"]
|
||||||
5
buildman/qemu-coreos/README.md
Normal file
5
buildman/qemu-coreos/README.md
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
# Builder Image
|
||||||
|
|
||||||
|
```
|
||||||
|
docker build --build-arg channel=stable --build-arg version=current -t quay.io/quay/quay-builder-qemu-coreos:staging .
|
||||||
|
```
|
||||||
26
buildman/qemu-coreos/start.sh
Normal file
26
buildman/qemu-coreos/start.sh
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
VM_VOLUME_SIZE="${VM_VOLUME_SIZE:-32G}"
|
||||||
|
VM_MEMORY="${VM_MEMORY:-4G}"
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -x
|
||||||
|
set -o nounset
|
||||||
|
|
||||||
|
mkdir -p /userdata/openstack/latest
|
||||||
|
echo "${USERDATA}" > /userdata/openstack/latest/user_data
|
||||||
|
|
||||||
|
time qemu-img resize ./coreos_production_qemu_image.img "${VM_VOLUME_SIZE}"
|
||||||
|
|
||||||
|
qemu-system-x86_64 \
|
||||||
|
-enable-kvm \
|
||||||
|
-cpu host \
|
||||||
|
-device virtio-9p-pci,fsdev=conf,mount_tag=config-2 \
|
||||||
|
-nographic \
|
||||||
|
-drive if=virtio,file=./coreos_production_qemu_image.img \
|
||||||
|
-fsdev local,id=conf,security_model=none,readonly,path=/userdata \
|
||||||
|
-m "${VM_MEMORY}" \
|
||||||
|
-machine accel=kvm \
|
||||||
|
-net nic,model=virtio \
|
||||||
|
-net user,hostfwd=tcp::2222-:22 \
|
||||||
|
-smp 2
|
||||||
266
buildman/server.py
Normal file
266
buildman/server.py
Normal file
|
|
@ -0,0 +1,266 @@
|
||||||
|
import logging
|
||||||
|
import json
|
||||||
|
import trollius
|
||||||
|
|
||||||
|
from threading import Event
|
||||||
|
from datetime import timedelta
|
||||||
|
from trollius.coroutines import From
|
||||||
|
from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory
|
||||||
|
from autobahn.asyncio.websocket import WampWebSocketServerFactory
|
||||||
|
from autobahn.wamp import types
|
||||||
|
from aiowsgi import create_server as create_wsgi_server
|
||||||
|
from flask import Flask
|
||||||
|
|
||||||
|
from buildman.enums import BuildJobResult, BuildServerStatus, RESULT_PHASES
|
||||||
|
from buildman.jobutil.buildstatus import StatusHandler
|
||||||
|
from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
|
||||||
|
from data import database, model
|
||||||
|
from app import app, metric_queue
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
WORK_CHECK_TIMEOUT = 10
|
||||||
|
TIMEOUT_PERIOD_MINUTES = 20
|
||||||
|
JOB_TIMEOUT_SECONDS = 300
|
||||||
|
SETUP_LEEWAY_SECONDS = 30
|
||||||
|
MINIMUM_JOB_EXTENSION = timedelta(minutes=1)
|
||||||
|
|
||||||
|
HEARTBEAT_PERIOD_SEC = 30
|
||||||
|
|
||||||
|
|
||||||
|
class BuilderServer(object):
|
||||||
|
""" Server which handles both HTTP and WAMP requests, managing the full state of the build
|
||||||
|
controller.
|
||||||
|
"""
|
||||||
|
def __init__(self, registry_hostname, queue, build_logs, user_files, lifecycle_manager_klass,
|
||||||
|
lifecycle_manager_config, manager_hostname):
|
||||||
|
self._loop = None
|
||||||
|
self._current_status = BuildServerStatus.STARTING
|
||||||
|
self._current_components = []
|
||||||
|
self._realm_map = {}
|
||||||
|
self._job_count = 0
|
||||||
|
|
||||||
|
self._session_factory = RouterSessionFactory(RouterFactory())
|
||||||
|
self._registry_hostname = registry_hostname
|
||||||
|
self._queue = queue
|
||||||
|
self._build_logs = build_logs
|
||||||
|
self._user_files = user_files
|
||||||
|
self._lifecycle_manager = lifecycle_manager_klass(
|
||||||
|
self._register_component,
|
||||||
|
self._unregister_component,
|
||||||
|
self._job_heartbeat,
|
||||||
|
self._job_complete,
|
||||||
|
manager_hostname,
|
||||||
|
HEARTBEAT_PERIOD_SEC,
|
||||||
|
)
|
||||||
|
self._lifecycle_manager_config = lifecycle_manager_config
|
||||||
|
|
||||||
|
self._shutdown_event = Event()
|
||||||
|
self._current_status = BuildServerStatus.RUNNING
|
||||||
|
|
||||||
|
self._register_controller()
|
||||||
|
|
||||||
|
def _register_controller(self):
|
||||||
|
controller_app = Flask('controller')
|
||||||
|
server = self
|
||||||
|
|
||||||
|
@controller_app.route('/status')
|
||||||
|
def status():
|
||||||
|
metrics = server._queue.get_metrics()
|
||||||
|
(running_count, available_not_running_count, available_count) = metrics
|
||||||
|
|
||||||
|
workers = [component for component in server._current_components
|
||||||
|
if component.kind() == 'builder']
|
||||||
|
|
||||||
|
data = {
|
||||||
|
'status': server._current_status,
|
||||||
|
'running_local': server._job_count,
|
||||||
|
'running_total': running_count,
|
||||||
|
'workers': len(workers),
|
||||||
|
'job_total': available_count + running_count
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.dumps(data)
|
||||||
|
|
||||||
|
self._controller_app = controller_app
|
||||||
|
|
||||||
|
def run(self, host, websocket_port, controller_port, ssl=None):
|
||||||
|
logger.debug('Initializing the lifecycle manager')
|
||||||
|
self._lifecycle_manager.initialize(self._lifecycle_manager_config)
|
||||||
|
|
||||||
|
logger.debug('Initializing all members of the event loop')
|
||||||
|
loop = trollius.get_event_loop()
|
||||||
|
|
||||||
|
logger.debug('Starting server on port %s, with controller on port %s', websocket_port,
|
||||||
|
controller_port)
|
||||||
|
|
||||||
|
try:
|
||||||
|
loop.run_until_complete(self._initialize(loop, host, websocket_port, controller_port, ssl))
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
loop.close()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
logger.debug('Requested server shutdown')
|
||||||
|
self._current_status = BuildServerStatus.SHUTDOWN
|
||||||
|
self._lifecycle_manager.shutdown()
|
||||||
|
self._shutdown_event.wait()
|
||||||
|
logger.debug('Shutting down server')
|
||||||
|
|
||||||
|
def _register_component(self, realm, component_klass, **kwargs):
|
||||||
|
""" Registers a component with the server. The component_klass must derive from
|
||||||
|
BaseComponent.
|
||||||
|
"""
|
||||||
|
logger.debug('Registering component with realm %s', realm)
|
||||||
|
if realm in self._realm_map:
|
||||||
|
logger.debug('Component with realm %s already registered', realm)
|
||||||
|
return self._realm_map[realm]
|
||||||
|
|
||||||
|
component = component_klass(types.ComponentConfig(realm=realm), realm=realm, **kwargs)
|
||||||
|
component.server = self
|
||||||
|
component.parent_manager = self._lifecycle_manager
|
||||||
|
component.build_logs = self._build_logs
|
||||||
|
component.user_files = self._user_files
|
||||||
|
component.registry_hostname = self._registry_hostname
|
||||||
|
|
||||||
|
self._realm_map[realm] = component
|
||||||
|
self._current_components.append(component)
|
||||||
|
self._session_factory.add(component)
|
||||||
|
return component
|
||||||
|
|
||||||
|
def _unregister_component(self, component):
|
||||||
|
logger.debug('Unregistering component with realm %s and token %s',
|
||||||
|
component.builder_realm, component.expected_token)
|
||||||
|
|
||||||
|
self._realm_map.pop(component.builder_realm, None)
|
||||||
|
|
||||||
|
if component in self._current_components:
|
||||||
|
self._current_components.remove(component)
|
||||||
|
self._session_factory.remove(component)
|
||||||
|
|
||||||
|
def _job_heartbeat(self, build_job):
|
||||||
|
self._queue.extend_processing(build_job.job_item, seconds_from_now=JOB_TIMEOUT_SECONDS,
|
||||||
|
minimum_extension=MINIMUM_JOB_EXTENSION)
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _job_complete(self, build_job, job_status, executor_name=None, update_phase=False):
|
||||||
|
if job_status == BuildJobResult.INCOMPLETE:
|
||||||
|
logger.warning('[BUILD INCOMPLETE: job complete] Build ID: %s. No retry restore.',
|
||||||
|
build_job.repo_build.uuid)
|
||||||
|
self._queue.incomplete(build_job.job_item, restore_retry=False, retry_after=30)
|
||||||
|
else:
|
||||||
|
self._queue.complete(build_job.job_item)
|
||||||
|
|
||||||
|
# Update the trigger failure tracking (if applicable).
|
||||||
|
if build_job.repo_build.trigger is not None:
|
||||||
|
model.build.update_trigger_disable_status(build_job.repo_build.trigger,
|
||||||
|
RESULT_PHASES[job_status])
|
||||||
|
|
||||||
|
if update_phase:
|
||||||
|
status_handler = StatusHandler(self._build_logs, build_job.repo_build.uuid)
|
||||||
|
yield From(status_handler.set_phase(RESULT_PHASES[job_status]))
|
||||||
|
|
||||||
|
self._job_count = self._job_count - 1
|
||||||
|
|
||||||
|
if self._current_status == BuildServerStatus.SHUTDOWN and not self._job_count:
|
||||||
|
self._shutdown_event.set()
|
||||||
|
|
||||||
|
_report_completion_status(build_job, job_status, executor_name)
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _work_checker(self):
|
||||||
|
logger.debug('Initializing work checker')
|
||||||
|
while self._current_status == BuildServerStatus.RUNNING:
|
||||||
|
with database.CloseForLongOperation(app.config):
|
||||||
|
yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
|
||||||
|
|
||||||
|
logger.debug('Checking for more work for %d active workers',
|
||||||
|
self._lifecycle_manager.num_workers())
|
||||||
|
|
||||||
|
processing_time = self._lifecycle_manager.overall_setup_time() + SETUP_LEEWAY_SECONDS
|
||||||
|
job_item = self._queue.get(processing_time=processing_time, ordering_required=True)
|
||||||
|
if job_item is None:
|
||||||
|
logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
build_job = BuildJob(job_item)
|
||||||
|
except BuildJobLoadException as irbe:
|
||||||
|
logger.warning('[BUILD INCOMPLETE: job load exception] Job data: %s. No retry restore.',
|
||||||
|
job_item.body)
|
||||||
|
logger.exception(irbe)
|
||||||
|
self._queue.incomplete(job_item, restore_retry=False)
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.debug('Checking for an avaliable worker for build job %s',
|
||||||
|
build_job.repo_build.uuid)
|
||||||
|
|
||||||
|
try:
|
||||||
|
schedule_success, retry_timeout = yield From(self._lifecycle_manager.schedule(build_job))
|
||||||
|
except:
|
||||||
|
logger.warning('[BUILD INCOMPLETE: scheduling] Build ID: %s. Retry restored.',
|
||||||
|
build_job.repo_build.uuid)
|
||||||
|
logger.exception('Exception when scheduling job: %s', build_job.repo_build.uuid)
|
||||||
|
self._current_status = BuildServerStatus.EXCEPTION
|
||||||
|
self._queue.incomplete(job_item, restore_retry=True, retry_after=WORK_CHECK_TIMEOUT)
|
||||||
|
return
|
||||||
|
|
||||||
|
if schedule_success:
|
||||||
|
logger.debug('Marking build %s as scheduled', build_job.repo_build.uuid)
|
||||||
|
status_handler = StatusHandler(self._build_logs, build_job.repo_build.uuid)
|
||||||
|
yield From(status_handler.set_phase(database.BUILD_PHASE.BUILD_SCHEDULED))
|
||||||
|
|
||||||
|
self._job_count = self._job_count + 1
|
||||||
|
logger.debug('Build job %s scheduled. Running: %s', build_job.repo_build.uuid,
|
||||||
|
self._job_count)
|
||||||
|
else:
|
||||||
|
logger.warning('[BUILD INCOMPLETE: no schedule] Build ID: %s. Retry restored.',
|
||||||
|
build_job.repo_build.uuid)
|
||||||
|
logger.debug('All workers are busy for job %s Requeuing after %s seconds.',
|
||||||
|
build_job.repo_build.uuid, retry_timeout)
|
||||||
|
self._queue.incomplete(job_item, restore_retry=True, retry_after=retry_timeout)
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _queue_metrics_updater(self):
|
||||||
|
logger.debug('Initializing queue metrics updater')
|
||||||
|
while self._current_status == BuildServerStatus.RUNNING:
|
||||||
|
logger.debug('Writing metrics')
|
||||||
|
self._queue.update_metrics()
|
||||||
|
|
||||||
|
logger.debug('Metrics going to sleep for 30 seconds')
|
||||||
|
yield From(trollius.sleep(30))
|
||||||
|
|
||||||
|
@trollius.coroutine
|
||||||
|
def _initialize(self, loop, host, websocket_port, controller_port, ssl=None):
|
||||||
|
self._loop = loop
|
||||||
|
|
||||||
|
# Create the WAMP server.
|
||||||
|
transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=False)
|
||||||
|
transport_factory.setProtocolOptions(failByDrop=True)
|
||||||
|
|
||||||
|
# Initialize the controller server and the WAMP server
|
||||||
|
create_wsgi_server(self._controller_app, loop=loop, host=host, port=controller_port, ssl=ssl)
|
||||||
|
yield From(loop.create_server(transport_factory, host, websocket_port, ssl=ssl))
|
||||||
|
|
||||||
|
# Initialize the metrics updater
|
||||||
|
trollius.async(self._queue_metrics_updater())
|
||||||
|
|
||||||
|
# Initialize the work queue checker.
|
||||||
|
yield From(self._work_checker())
|
||||||
|
|
||||||
|
def _report_completion_status(build_job, status, executor_name):
|
||||||
|
metric_queue.build_counter.Inc(labelvalues=[status])
|
||||||
|
metric_queue.repository_build_completed.Inc(labelvalues=[build_job.namespace, build_job.repo_name,
|
||||||
|
status, executor_name or 'executor'])
|
||||||
|
if status == BuildJobResult.COMPLETE:
|
||||||
|
status_name = 'CompleteBuilds'
|
||||||
|
elif status == BuildJobResult.ERROR:
|
||||||
|
status_name = 'FailedBuilds'
|
||||||
|
elif status == BuildJobResult.INCOMPLETE:
|
||||||
|
status_name = 'IncompletedBuilds'
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
|
||||||
|
metric_queue.put_deprecated(status_name, 1, unit='Count')
|
||||||
102
buildman/templates/cloudconfig.yaml
Normal file
102
buildman/templates/cloudconfig.yaml
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
#cloud-config
|
||||||
|
|
||||||
|
hostname: {{ build_uuid | default('quay-builder', True) }}
|
||||||
|
|
||||||
|
users:
|
||||||
|
groups:
|
||||||
|
- sudo
|
||||||
|
- docker
|
||||||
|
|
||||||
|
{% if ssh_authorized_keys -%}
|
||||||
|
ssh_authorized_keys:
|
||||||
|
{% for ssh_key in ssh_authorized_keys -%}
|
||||||
|
- {{ ssh_key }}
|
||||||
|
{%- endfor %}
|
||||||
|
{%- endif %}
|
||||||
|
|
||||||
|
write_files:
|
||||||
|
- path: /root/disable-aws-metadata.sh
|
||||||
|
permission: '0755'
|
||||||
|
content: |
|
||||||
|
iptables -t nat -I PREROUTING -p tcp -d 169.254.169.254 --dport 80 -j DNAT --to-destination 1.1.1.1
|
||||||
|
|
||||||
|
- path: /etc/docker/daemon.json
|
||||||
|
permission: '0644'
|
||||||
|
content: |
|
||||||
|
{
|
||||||
|
"storage-driver": "overlay2"
|
||||||
|
}
|
||||||
|
|
||||||
|
- path: /root/overrides.list
|
||||||
|
permission: '0644'
|
||||||
|
content: |
|
||||||
|
REALM={{ realm }}
|
||||||
|
TOKEN={{ token }}
|
||||||
|
SERVER={{ websocket_scheme }}://{{ manager_hostname }}
|
||||||
|
{% if logentries_token -%}
|
||||||
|
LOGENTRIES_TOKEN={{ logentries_token }}
|
||||||
|
{%- endif %}
|
||||||
|
|
||||||
|
coreos:
|
||||||
|
update:
|
||||||
|
reboot-strategy: off
|
||||||
|
group: {{ coreos_channel }}
|
||||||
|
|
||||||
|
units:
|
||||||
|
- name: update-engine.service
|
||||||
|
command: stop
|
||||||
|
- name: locksmithd.service
|
||||||
|
command: stop
|
||||||
|
- name: systemd-journal-gatewayd.socket
|
||||||
|
command: start
|
||||||
|
enable: yes
|
||||||
|
content: |
|
||||||
|
[Unit]
|
||||||
|
Description=Journal Gateway Service Socket
|
||||||
|
[Socket]
|
||||||
|
ListenStream=/var/run/journald.sock
|
||||||
|
Service=systemd-journal-gatewayd.service
|
||||||
|
[Install]
|
||||||
|
WantedBy=sockets.target
|
||||||
|
{{ dockersystemd('quay-builder',
|
||||||
|
worker_image,
|
||||||
|
quay_username,
|
||||||
|
quay_password,
|
||||||
|
worker_tag,
|
||||||
|
extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/ca-certificates:/etc/ssl/certs',
|
||||||
|
exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'],
|
||||||
|
flattened=True,
|
||||||
|
restart_policy='no'
|
||||||
|
) | indent(4) }}
|
||||||
|
{% if logentries_token -%}
|
||||||
|
# https://github.com/kelseyhightower/journal-2-logentries/pull/11 so moved journal-2-logentries to coreos
|
||||||
|
{{ dockersystemd('builder-logs',
|
||||||
|
'quay.io/coreos/journal-2-logentries',
|
||||||
|
extra_args='--env-file /root/overrides.list -v /run/journald.sock:/run/journald.sock',
|
||||||
|
flattened=True,
|
||||||
|
after_units=['quay-builder.service']
|
||||||
|
) | indent(4) }}
|
||||||
|
{%- endif %}
|
||||||
|
- name: disable-aws-metadata.service
|
||||||
|
command: start
|
||||||
|
enable: yes
|
||||||
|
content: |
|
||||||
|
[Unit]
|
||||||
|
Description=Disable AWS metadata service
|
||||||
|
Before=network-pre.target
|
||||||
|
Wants=network-pre.target
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/root/disable-aws-metadata.sh
|
||||||
|
RemainAfterExit=yes
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
- name: machine-lifetime.service
|
||||||
|
command: start
|
||||||
|
enable: yes
|
||||||
|
content: |
|
||||||
|
[Unit]
|
||||||
|
Description=Machine Lifetime Service
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/bin/sh -xc "/bin/sleep {{ max_lifetime_s }}; /usr/bin/systemctl --no-block poweroff"
|
||||||
679
buildman/test/test_buildman.py
Normal file
679
buildman/test/test_buildman.py
Normal file
|
|
@ -0,0 +1,679 @@
|
||||||
|
import unittest
|
||||||
|
import json
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from mock import Mock, ANY
|
||||||
|
from six import iteritems
|
||||||
|
from trollius import coroutine, get_event_loop, From, Future, Return
|
||||||
|
|
||||||
|
from app import metric_queue
|
||||||
|
from buildman.asyncutil import AsyncWrapper
|
||||||
|
from buildman.component.buildcomponent import BuildComponent
|
||||||
|
from buildman.manager.ephemeral import (EphemeralBuilderManager, REALM_PREFIX,
|
||||||
|
JOB_PREFIX)
|
||||||
|
from buildman.manager.executor import BuilderExecutor, ExecutorException
|
||||||
|
from buildman.orchestrator import KeyEvent, KeyChange
|
||||||
|
from buildman.server import BuildJobResult
|
||||||
|
from util import slash_join
|
||||||
|
from util.metrics.metricqueue import duration_collector_async
|
||||||
|
|
||||||
|
|
||||||
|
BUILD_UUID = 'deadbeef-dead-beef-dead-deadbeefdead'
|
||||||
|
REALM_ID = '1234-realm'
|
||||||
|
|
||||||
|
|
||||||
|
def async_test(f):
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
coro = coroutine(f)
|
||||||
|
future = coro(*args, **kwargs)
|
||||||
|
loop = get_event_loop()
|
||||||
|
loop.run_until_complete(future)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
class TestExecutor(BuilderExecutor):
|
||||||
|
job_started = None
|
||||||
|
job_stopped = None
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
@duration_collector_async(metric_queue.builder_time_to_start, labelvalues=["testlabel"])
|
||||||
|
def start_builder(self, realm, token, build_uuid):
|
||||||
|
self.job_started = str(uuid.uuid4())
|
||||||
|
raise Return(self.job_started)
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def stop_builder(self, execution_id):
|
||||||
|
self.job_stopped = execution_id
|
||||||
|
|
||||||
|
|
||||||
|
class BadExecutor(BuilderExecutor):
|
||||||
|
@coroutine
|
||||||
|
@duration_collector_async(metric_queue.builder_time_to_start, labelvalues=["testlabel"])
|
||||||
|
def start_builder(self, realm, token, build_uuid):
|
||||||
|
raise ExecutorException('raised on purpose!')
|
||||||
|
|
||||||
|
|
||||||
|
class EphemeralBuilderTestCase(unittest.TestCase):
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
self.etcd_client_mock = None
|
||||||
|
super(EphemeralBuilderTestCase, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _create_completed_future(result=None):
|
||||||
|
def inner(*args, **kwargs):
|
||||||
|
new_future = Future()
|
||||||
|
new_future.set_result(result)
|
||||||
|
return new_future
|
||||||
|
return inner
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._existing_executors = dict(EphemeralBuilderManager.EXECUTORS)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS = self._existing_executors
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _register_component(self, realm_spec, build_component, token):
|
||||||
|
raise Return('hello')
|
||||||
|
|
||||||
|
def _create_build_job(self, namespace='namespace', retries=3):
|
||||||
|
mock_job = Mock()
|
||||||
|
mock_job.job_details = {'build_uuid': BUILD_UUID}
|
||||||
|
mock_job.job_item = {
|
||||||
|
'body': json.dumps(mock_job.job_details),
|
||||||
|
'id': 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
mock_job.namespace = namespace
|
||||||
|
mock_job.retries_remaining = retries
|
||||||
|
mock_job.build_uuid = BUILD_UUID
|
||||||
|
return mock_job
|
||||||
|
|
||||||
|
|
||||||
|
class TestEphemeralLifecycle(EphemeralBuilderTestCase):
|
||||||
|
""" Tests the various lifecycles of the ephemeral builder and its interaction with etcd. """
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(TestEphemeralLifecycle, self).__init__(*args, **kwargs)
|
||||||
|
self.etcd_client_mock = None
|
||||||
|
self.test_executor = None
|
||||||
|
|
||||||
|
def _create_completed_future(self, result=None):
|
||||||
|
def inner(*args, **kwargs):
|
||||||
|
new_future = Future()
|
||||||
|
new_future.set_result(result)
|
||||||
|
return new_future
|
||||||
|
return inner
|
||||||
|
|
||||||
|
def _create_mock_executor(self, *args, **kwargs):
|
||||||
|
self.test_executor = Mock(spec=BuilderExecutor)
|
||||||
|
self.test_executor.start_builder = Mock(side_effect=self._create_completed_future('123'))
|
||||||
|
self.test_executor.stop_builder = Mock(side_effect=self._create_completed_future())
|
||||||
|
self.test_executor.setup_time = 60
|
||||||
|
self.test_executor.name = 'MockExecutor'
|
||||||
|
self.test_executor.minimum_retry_threshold = 0
|
||||||
|
return self.test_executor
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestEphemeralLifecycle, self).setUp()
|
||||||
|
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = self._create_mock_executor
|
||||||
|
|
||||||
|
self.register_component_callback = Mock()
|
||||||
|
self.unregister_component_callback = Mock()
|
||||||
|
self.job_heartbeat_callback = Mock()
|
||||||
|
self.job_complete_callback = AsyncWrapper(Mock())
|
||||||
|
|
||||||
|
self.manager = EphemeralBuilderManager(
|
||||||
|
self.register_component_callback,
|
||||||
|
self.unregister_component_callback,
|
||||||
|
self.job_heartbeat_callback,
|
||||||
|
self.job_complete_callback,
|
||||||
|
'127.0.0.1',
|
||||||
|
30,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Ensure that that the realm and building callbacks have been registered
|
||||||
|
callback_keys = [key for key in self.manager._orchestrator.callbacks]
|
||||||
|
self.assertIn(REALM_PREFIX, callback_keys)
|
||||||
|
self.assertIn(JOB_PREFIX, callback_keys)
|
||||||
|
|
||||||
|
self.mock_job = self._create_build_job()
|
||||||
|
self.mock_job_key = slash_join('building', BUILD_UUID)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
super(TestEphemeralLifecycle, self).tearDown()
|
||||||
|
self.manager.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def _setup_job_for_managers(self):
|
||||||
|
test_component = Mock(spec=BuildComponent)
|
||||||
|
test_component.builder_realm = REALM_ID
|
||||||
|
test_component.start_build = Mock(side_effect=self._create_completed_future())
|
||||||
|
self.register_component_callback.return_value = test_component
|
||||||
|
|
||||||
|
is_scheduled = yield From(self.manager.schedule(self.mock_job))
|
||||||
|
self.assertTrue(is_scheduled)
|
||||||
|
self.assertEqual(self.test_executor.start_builder.call_count, 1)
|
||||||
|
|
||||||
|
# Ensure that that the job, realm, and metric callbacks have been registered
|
||||||
|
callback_keys = [key for key in self.manager._orchestrator.callbacks]
|
||||||
|
self.assertIn(self.mock_job_key, self.manager._orchestrator.state)
|
||||||
|
self.assertIn(REALM_PREFIX, callback_keys)
|
||||||
|
# TODO: assert metric key has been set
|
||||||
|
|
||||||
|
realm_for_build = self._find_realm_key(self.manager._orchestrator, BUILD_UUID)
|
||||||
|
|
||||||
|
raw_realm_data = yield From(self.manager._orchestrator.get_key(slash_join('realm',
|
||||||
|
realm_for_build)))
|
||||||
|
realm_data = json.loads(raw_realm_data)
|
||||||
|
realm_data['realm'] = REALM_ID
|
||||||
|
|
||||||
|
# Right now the job is not registered with any managers because etcd has not accepted the job
|
||||||
|
self.assertEqual(self.register_component_callback.call_count, 0)
|
||||||
|
|
||||||
|
# Fire off a realm changed with the same data.
|
||||||
|
yield From(self.manager._realm_callback(
|
||||||
|
KeyChange(KeyEvent.CREATE,
|
||||||
|
slash_join(REALM_PREFIX, REALM_ID),
|
||||||
|
json.dumps(realm_data))))
|
||||||
|
|
||||||
|
# Ensure that we have at least one component node.
|
||||||
|
self.assertEqual(self.register_component_callback.call_count, 1)
|
||||||
|
self.assertEqual(1, self.manager.num_workers())
|
||||||
|
|
||||||
|
# Ensure that the build info exists.
|
||||||
|
self.assertIsNotNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
|
||||||
|
|
||||||
|
raise Return(test_component)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _find_realm_key(orchestrator, build_uuid):
|
||||||
|
for key, value in iteritems(orchestrator.state):
|
||||||
|
if key.startswith(REALM_PREFIX):
|
||||||
|
parsed_value = json.loads(value)
|
||||||
|
body = json.loads(parsed_value['job_queue_item']['body'])
|
||||||
|
if body['build_uuid'] == build_uuid:
|
||||||
|
return parsed_value['realm']
|
||||||
|
continue
|
||||||
|
raise KeyError
|
||||||
|
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_schedule_and_complete(self):
|
||||||
|
# Test that a job is properly registered with all of the managers
|
||||||
|
test_component = yield From(self._setup_job_for_managers())
|
||||||
|
|
||||||
|
# Take the job ourselves
|
||||||
|
yield From(self.manager.build_component_ready(test_component))
|
||||||
|
|
||||||
|
self.assertIsNotNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
|
||||||
|
|
||||||
|
# Finish the job
|
||||||
|
yield From(self.manager.job_completed(self.mock_job, BuildJobResult.COMPLETE, test_component))
|
||||||
|
|
||||||
|
# Ensure that the executor kills the job.
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
|
||||||
|
|
||||||
|
# Ensure the build information is cleaned up.
|
||||||
|
self.assertIsNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
|
||||||
|
self.assertEqual(0, self.manager.num_workers())
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_another_manager_takes_job(self):
|
||||||
|
# Prepare a job to be taken by another manager
|
||||||
|
test_component = yield From(self._setup_job_for_managers())
|
||||||
|
|
||||||
|
yield From(self.manager._realm_callback(
|
||||||
|
KeyChange(KeyEvent.DELETE,
|
||||||
|
slash_join(REALM_PREFIX, REALM_ID),
|
||||||
|
json.dumps({'realm': REALM_ID,
|
||||||
|
'token': 'beef',
|
||||||
|
'execution_id': '123',
|
||||||
|
'job_queue_item': self.mock_job.job_item}))))
|
||||||
|
|
||||||
|
self.unregister_component_callback.assert_called_once_with(test_component)
|
||||||
|
|
||||||
|
# Ensure that the executor does not kill the job.
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
|
||||||
|
|
||||||
|
# Ensure that we still have the build info, but not the component.
|
||||||
|
self.assertEqual(0, self.manager.num_workers())
|
||||||
|
self.assertIsNotNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
|
||||||
|
|
||||||
|
# Delete the job once it has "completed".
|
||||||
|
yield From(self.manager._job_callback(
|
||||||
|
KeyChange(KeyEvent.DELETE,
|
||||||
|
self.mock_job_key,
|
||||||
|
json.dumps({'had_heartbeat': False,
|
||||||
|
'job_queue_item': self.mock_job.job_item}))))
|
||||||
|
|
||||||
|
# Ensure the job was removed from the info, but stop was not called.
|
||||||
|
self.assertIsNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_job_started_by_other_manager(self):
|
||||||
|
# Ensure that that the building callbacks have been registered
|
||||||
|
callback_keys = [key for key in self.manager._orchestrator.callbacks]
|
||||||
|
self.assertIn(JOB_PREFIX, callback_keys)
|
||||||
|
|
||||||
|
# Send a signal to the callback that the job has been created.
|
||||||
|
yield From(self.manager._job_callback(
|
||||||
|
KeyChange(KeyEvent.CREATE,
|
||||||
|
self.mock_job_key,
|
||||||
|
json.dumps({'had_heartbeat': False,
|
||||||
|
'job_queue_item': self.mock_job.job_item}))))
|
||||||
|
|
||||||
|
# Ensure the create does nothing.
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_expiring_worker_not_started(self):
|
||||||
|
# Ensure that that the building callbacks have been registered
|
||||||
|
callback_keys = [key for key in self.manager._orchestrator.callbacks]
|
||||||
|
self.assertIn(JOB_PREFIX, callback_keys)
|
||||||
|
|
||||||
|
# Send a signal to the callback that a worker has expired
|
||||||
|
yield From(self.manager._job_callback(
|
||||||
|
KeyChange(KeyEvent.EXPIRE,
|
||||||
|
self.mock_job_key,
|
||||||
|
json.dumps({'had_heartbeat': True,
|
||||||
|
'job_queue_item': self.mock_job.job_item}))))
|
||||||
|
|
||||||
|
# Since the realm was never registered, expiration should do nothing.
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_expiring_worker_started(self):
|
||||||
|
test_component = yield From(self._setup_job_for_managers())
|
||||||
|
|
||||||
|
# Ensure that that the building callbacks have been registered
|
||||||
|
callback_keys = [key for key in self.manager._orchestrator.callbacks]
|
||||||
|
self.assertIn(JOB_PREFIX, callback_keys)
|
||||||
|
|
||||||
|
yield From(self.manager._job_callback(
|
||||||
|
KeyChange(KeyEvent.EXPIRE,
|
||||||
|
self.mock_job_key,
|
||||||
|
json.dumps({'had_heartbeat': True,
|
||||||
|
'job_queue_item': self.mock_job.job_item}))))
|
||||||
|
|
||||||
|
self.test_executor.stop_builder.assert_called_once_with('123')
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_buildjob_deleted(self):
|
||||||
|
test_component = yield From(self._setup_job_for_managers())
|
||||||
|
|
||||||
|
# Ensure that that the building callbacks have been registered
|
||||||
|
callback_keys = [key for key in self.manager._orchestrator.callbacks]
|
||||||
|
self.assertIn(JOB_PREFIX, callback_keys)
|
||||||
|
|
||||||
|
# Send a signal to the callback that a worker has expired
|
||||||
|
yield From(self.manager._job_callback(
|
||||||
|
KeyChange(KeyEvent.DELETE,
|
||||||
|
self.mock_job_key,
|
||||||
|
json.dumps({'had_heartbeat': False,
|
||||||
|
'job_queue_item': self.mock_job.job_item}))))
|
||||||
|
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
|
||||||
|
self.assertEqual(self.job_complete_callback.call_count, 0)
|
||||||
|
self.assertIsNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_builder_never_starts(self):
|
||||||
|
test_component = yield From(self._setup_job_for_managers())
|
||||||
|
|
||||||
|
# Ensure that that the building callbacks have been registered
|
||||||
|
callback_keys = [key for key in self.manager._orchestrator.callbacks]
|
||||||
|
self.assertIn(JOB_PREFIX, callback_keys)
|
||||||
|
|
||||||
|
# Send a signal to the callback that a worker has expired
|
||||||
|
yield From(self.manager._job_callback(
|
||||||
|
KeyChange(KeyEvent.EXPIRE,
|
||||||
|
self.mock_job_key,
|
||||||
|
json.dumps({'had_heartbeat': False,
|
||||||
|
'job_queue_item': self.mock_job.job_item}))))
|
||||||
|
|
||||||
|
self.test_executor.stop_builder.assert_called_once_with('123')
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
|
||||||
|
|
||||||
|
# Ensure the job was marked as incomplete, with an update_phase to True (so the DB record and
|
||||||
|
# logs are updated as well)
|
||||||
|
yield From(self.job_complete_callback.assert_called_once_with(ANY, BuildJobResult.INCOMPLETE,
|
||||||
|
'MockExecutor',
|
||||||
|
update_phase=True))
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_change_worker(self):
|
||||||
|
# Send a signal to the callback that a worker key has been changed
|
||||||
|
self.manager._job_callback(KeyChange(KeyEvent.SET, self.mock_job_key, 'value'))
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_realm_expired(self):
|
||||||
|
test_component = yield From(self._setup_job_for_managers())
|
||||||
|
|
||||||
|
# Send a signal to the callback that a realm has expired
|
||||||
|
yield From(self.manager._realm_callback(KeyChange(
|
||||||
|
KeyEvent.EXPIRE,
|
||||||
|
self.mock_job_key,
|
||||||
|
json.dumps({
|
||||||
|
'realm': REALM_ID,
|
||||||
|
'execution_id': 'foobar',
|
||||||
|
'executor_name': 'MockExecutor',
|
||||||
|
'job_queue_item': {'body': '{"build_uuid": "fakeid"}'},
|
||||||
|
}))))
|
||||||
|
|
||||||
|
# Ensure that the cleanup code for the executor was called.
|
||||||
|
self.test_executor.stop_builder.assert_called_once_with('foobar')
|
||||||
|
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
|
||||||
|
|
||||||
|
|
||||||
|
class TestEphemeral(EphemeralBuilderTestCase):
|
||||||
|
""" Simple unit tests for the ephemeral builder around config management, starting and stopping
|
||||||
|
jobs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestEphemeral, self).setUp()
|
||||||
|
|
||||||
|
unregister_component_callback = Mock()
|
||||||
|
job_heartbeat_callback = Mock()
|
||||||
|
|
||||||
|
@coroutine
|
||||||
|
def job_complete_callback(*args, **kwargs):
|
||||||
|
raise Return()
|
||||||
|
|
||||||
|
self.manager = EphemeralBuilderManager(
|
||||||
|
self._register_component,
|
||||||
|
unregister_component_callback,
|
||||||
|
job_heartbeat_callback,
|
||||||
|
job_complete_callback,
|
||||||
|
'127.0.0.1',
|
||||||
|
30,
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
super(TestEphemeral, self).tearDown()
|
||||||
|
self.manager.shutdown()
|
||||||
|
|
||||||
|
def test_verify_executor_oldconfig(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'EXECUTOR_CONFIG': dict(MINIMUM_RETRY_THRESHOLD=42),
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Ensure that we have a single test executor.
|
||||||
|
self.assertEqual(1, len(self.manager.registered_executors))
|
||||||
|
self.assertEqual(42, self.manager.registered_executors[0].minimum_retry_threshold)
|
||||||
|
self.assertEqual('TestExecutor', self.manager.registered_executors[0].name)
|
||||||
|
|
||||||
|
def test_verify_executor_newconfig(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTORS': [{
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 42
|
||||||
|
}],
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Ensure that we have a single test executor.
|
||||||
|
self.assertEqual(1, len(self.manager.registered_executors))
|
||||||
|
self.assertEqual(42, self.manager.registered_executors[0].minimum_retry_threshold)
|
||||||
|
|
||||||
|
|
||||||
|
def test_multiple_executors_samename(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
EphemeralBuilderManager.EXECUTORS['anotherexecutor'] = TestExecutor
|
||||||
|
|
||||||
|
with self.assertRaises(Exception):
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTORS': [
|
||||||
|
{
|
||||||
|
'NAME': 'primary',
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 42
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'primary',
|
||||||
|
'EXECUTOR': 'anotherexecutor',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 24
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_multiple_executors(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
EphemeralBuilderManager.EXECUTORS['anotherexecutor'] = TestExecutor
|
||||||
|
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTORS': [
|
||||||
|
{
|
||||||
|
'NAME': 'primary',
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 42
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'secondary',
|
||||||
|
'EXECUTOR': 'anotherexecutor',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 24
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Ensure that we have a two test executors.
|
||||||
|
self.assertEqual(2, len(self.manager.registered_executors))
|
||||||
|
self.assertEqual(42, self.manager.registered_executors[0].minimum_retry_threshold)
|
||||||
|
self.assertEqual(24, self.manager.registered_executors[1].minimum_retry_threshold)
|
||||||
|
|
||||||
|
def test_skip_invalid_executor(self):
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTORS': [
|
||||||
|
{
|
||||||
|
'EXECUTOR': 'unknown',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 42
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
self.assertEqual(0, len(self.manager.registered_executors))
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_schedule_job_namespace_filter(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTORS': [{
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'NAMESPACE_WHITELIST': ['something'],
|
||||||
|
}],
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Try with a build job in an invalid namespace.
|
||||||
|
build_job = self._create_build_job(namespace='somethingelse')
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertFalse(result[0])
|
||||||
|
|
||||||
|
# Try with a valid namespace.
|
||||||
|
build_job = self._create_build_job(namespace='something')
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_schedule_job_retries_filter(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTORS': [{
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 2,
|
||||||
|
}],
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Try with a build job that has too few retries.
|
||||||
|
build_job = self._create_build_job(retries=1)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertFalse(result[0])
|
||||||
|
|
||||||
|
# Try with a valid job.
|
||||||
|
build_job = self._create_build_job(retries=2)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_schedule_job_executor_fallback(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['primary'] = TestExecutor
|
||||||
|
EphemeralBuilderManager.EXECUTORS['secondary'] = TestExecutor
|
||||||
|
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTORS': [
|
||||||
|
{
|
||||||
|
'NAME': 'primary',
|
||||||
|
'EXECUTOR': 'primary',
|
||||||
|
'NAMESPACE_WHITELIST': ['something'],
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'secondary',
|
||||||
|
'EXECUTOR': 'secondary',
|
||||||
|
'MINIMUM_RETRY_THRESHOLD': 2,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'ALLOWED_WORKER_COUNT': 5,
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Try a job not matching the primary's namespace filter. Should schedule on secondary.
|
||||||
|
build_job = self._create_build_job(namespace='somethingelse')
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
self.assertIsNone(self.manager.registered_executors[0].job_started)
|
||||||
|
self.assertIsNotNone(self.manager.registered_executors[1].job_started)
|
||||||
|
|
||||||
|
self.manager.registered_executors[0].job_started = None
|
||||||
|
self.manager.registered_executors[1].job_started = None
|
||||||
|
|
||||||
|
# Try a job not matching the primary's retry minimum. Should schedule on secondary.
|
||||||
|
build_job = self._create_build_job(namespace='something', retries=2)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
self.assertIsNone(self.manager.registered_executors[0].job_started)
|
||||||
|
self.assertIsNotNone(self.manager.registered_executors[1].job_started)
|
||||||
|
|
||||||
|
self.manager.registered_executors[0].job_started = None
|
||||||
|
self.manager.registered_executors[1].job_started = None
|
||||||
|
|
||||||
|
# Try a job matching the primary. Should schedule on the primary.
|
||||||
|
build_job = self._create_build_job(namespace='something', retries=3)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
self.assertIsNotNone(self.manager.registered_executors[0].job_started)
|
||||||
|
self.assertIsNone(self.manager.registered_executors[1].job_started)
|
||||||
|
|
||||||
|
self.manager.registered_executors[0].job_started = None
|
||||||
|
self.manager.registered_executors[1].job_started = None
|
||||||
|
|
||||||
|
# Try a job not matching either's restrictions.
|
||||||
|
build_job = self._create_build_job(namespace='somethingelse', retries=1)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertFalse(result[0])
|
||||||
|
|
||||||
|
self.assertIsNone(self.manager.registered_executors[0].job_started)
|
||||||
|
self.assertIsNone(self.manager.registered_executors[1].job_started)
|
||||||
|
|
||||||
|
self.manager.registered_executors[0].job_started = None
|
||||||
|
self.manager.registered_executors[1].job_started = None
|
||||||
|
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_schedule_job_single_executor(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'EXECUTOR_CONFIG': {},
|
||||||
|
'ALLOWED_WORKER_COUNT': 5,
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
build_job = self._create_build_job(namespace='something', retries=3)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
self.assertIsNotNone(self.manager.registered_executors[0].job_started)
|
||||||
|
self.manager.registered_executors[0].job_started = None
|
||||||
|
|
||||||
|
|
||||||
|
build_job = self._create_build_job(namespace='something', retries=0)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
self.assertIsNotNone(self.manager.registered_executors[0].job_started)
|
||||||
|
self.manager.registered_executors[0].job_started = None
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_executor_exception(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['bad'] = BadExecutor
|
||||||
|
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTOR': 'bad',
|
||||||
|
'EXECUTOR_CONFIG': {},
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
build_job = self._create_build_job(namespace='something', retries=3)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertFalse(result[0])
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
def test_schedule_and_stop(self):
|
||||||
|
EphemeralBuilderManager.EXECUTORS['test'] = TestExecutor
|
||||||
|
|
||||||
|
self.manager.initialize({
|
||||||
|
'EXECUTOR': 'test',
|
||||||
|
'EXECUTOR_CONFIG': {},
|
||||||
|
'ORCHESTRATOR': {'MEM_CONFIG': None},
|
||||||
|
})
|
||||||
|
|
||||||
|
# Start the build job.
|
||||||
|
build_job = self._create_build_job(namespace='something', retries=3)
|
||||||
|
result = yield From(self.manager.schedule(build_job))
|
||||||
|
self.assertTrue(result[0])
|
||||||
|
|
||||||
|
executor = self.manager.registered_executors[0]
|
||||||
|
self.assertIsNotNone(executor.job_started)
|
||||||
|
|
||||||
|
# Register the realm so the build information is added.
|
||||||
|
yield From(self.manager._register_realm({
|
||||||
|
'realm': str(uuid.uuid4()),
|
||||||
|
'token': str(uuid.uuid4()),
|
||||||
|
'execution_id': executor.job_started,
|
||||||
|
'executor_name': 'TestExecutor',
|
||||||
|
'build_uuid': build_job.build_uuid,
|
||||||
|
'job_queue_item': build_job.job_item,
|
||||||
|
}))
|
||||||
|
|
||||||
|
# Stop the build job.
|
||||||
|
yield From(self.manager.kill_builder_executor(build_job.build_uuid))
|
||||||
|
self.assertEqual(executor.job_stopped, executor.job_started)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
1
buildstatus/building.svg
Normal file
1
buildstatus/building.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="117" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><mask id="a"><rect width="117" height="20" rx="3" fill="#fff"/></mask><g mask="url(#a)"><path fill="#555" d="M0 0h63v20H0z"/><path fill="#dfb317" d="M63 0h54v20H63z"/><path fill="url(#b)" d="M0 0h117v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11"><text x="32.5" y="15" fill="#010101" fill-opacity=".3">container</text><text x="32.5" y="14">container</text><text x="89" y="15" fill="#010101" fill-opacity=".3">building</text><text x="89" y="14">building</text></g></svg>
|
||||||
|
After Width: | Height: | Size: 748 B |
1
buildstatus/cancelled.svg
Normal file
1
buildstatus/cancelled.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="123" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><mask id="a"><rect width="123" height="20" rx="3" fill="#fff"/></mask><g mask="url(#a)"><path fill="#555" d="M0 0h63v20H0z"/><path fill="#9f9f9f" d="M63 0h60v20H63z"/><path fill="url(#b)" d="M0 0h123v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11"><text x="32.5" y="15" fill="#010101" fill-opacity=".3">container</text><text x="32.5" y="14">container</text><text x="92" y="15" fill="#010101" fill-opacity=".3">cancelled</text><text x="92" y="14">cancelled</text></g></svg>
|
||||||
|
After Width: | Height: | Size: 751 B |
1
buildstatus/failed.svg
Normal file
1
buildstatus/failed.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="104" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><mask id="a"><rect width="104" height="20" rx="3" fill="#fff"/></mask><g mask="url(#a)"><path fill="#555" d="M0 0h63v20H0z"/><path fill="#e05d44" d="M63 0h41v20H63z"/><path fill="url(#b)" d="M0 0h104v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11"><text x="32.5" y="15" fill="#010101" fill-opacity=".3">container</text><text x="32.5" y="14">container</text><text x="82.5" y="15" fill="#010101" fill-opacity=".3">failed</text><text x="82.5" y="14">failed</text></g></svg>
|
||||||
|
After Width: | Height: | Size: 748 B |
1
buildstatus/none.svg
Normal file
1
buildstatus/none.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="101" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><mask id="a"><rect width="101" height="20" rx="3" fill="#fff"/></mask><g mask="url(#a)"><path fill="#555" d="M0 0h63v20H0z"/><path fill="#9f9f9f" d="M63 0h38v20H63z"/><path fill="url(#b)" d="M0 0h101v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11"><text x="32.5" y="15" fill="#010101" fill-opacity=".3">container</text><text x="32.5" y="14">container</text><text x="81" y="15" fill="#010101" fill-opacity=".3">none</text><text x="81" y="14">none</text></g></svg>
|
||||||
|
After Width: | Height: | Size: 740 B |
1
buildstatus/ready.svg
Normal file
1
buildstatus/ready.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><mask id="a"><rect width="106" height="20" rx="3" fill="#fff"/></mask><g mask="url(#a)"><path fill="#555" d="M0 0h63v20H0z"/><path fill="#97CA00" d="M63 0h43v20H63z"/><path fill="url(#b)" d="M0 0h106v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11"><text x="32.5" y="15" fill="#010101" fill-opacity=".3">container</text><text x="32.5" y="14">container</text><text x="83.5" y="15" fill="#010101" fill-opacity=".3">ready</text><text x="83.5" y="14">ready</text></g></svg>
|
||||||
|
After Width: | Height: | Size: 746 B |
5
buildtrigger/__init__.py
Normal file
5
buildtrigger/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
import buildtrigger.bitbuckethandler
|
||||||
|
import buildtrigger.customhandler
|
||||||
|
import buildtrigger.githubhandler
|
||||||
|
import buildtrigger.gitlabhandler
|
||||||
|
|
||||||
367
buildtrigger/basehandler.py
Normal file
367
buildtrigger/basehandler.py
Normal file
|
|
@ -0,0 +1,367 @@
|
||||||
|
import os
|
||||||
|
from abc import ABCMeta, abstractmethod
|
||||||
|
from jsonschema import validate
|
||||||
|
from six import add_metaclass
|
||||||
|
|
||||||
|
from active_migration import ActiveDataMigration, ERTMigrationFlags
|
||||||
|
from endpoints.building import PreparedBuild
|
||||||
|
from data import model
|
||||||
|
from buildtrigger.triggerutil import get_trigger_config, InvalidServiceException
|
||||||
|
|
||||||
|
NAMESPACES_SCHEMA = {
|
||||||
|
'type': 'array',
|
||||||
|
'items': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'personal': {
|
||||||
|
'type': 'boolean',
|
||||||
|
'description': 'True if the namespace is the user\'s personal namespace',
|
||||||
|
},
|
||||||
|
'score': {
|
||||||
|
'type': 'number',
|
||||||
|
'description': 'Score of the relevance of the namespace',
|
||||||
|
},
|
||||||
|
'avatar_url': {
|
||||||
|
'type': ['string', 'null'],
|
||||||
|
'description': 'URL of the avatar for this namespace',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL of the website to view the namespace',
|
||||||
|
},
|
||||||
|
'id': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'Trigger-internal ID of the namespace',
|
||||||
|
},
|
||||||
|
'title': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'Human-readable title of the namespace',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['personal', 'score', 'avatar_url', 'id', 'title'],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
BUILD_SOURCES_SCHEMA = {
|
||||||
|
'type': 'array',
|
||||||
|
'items': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'name': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'The name of the repository, without its namespace',
|
||||||
|
},
|
||||||
|
'full_name': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'The name of the repository, with its namespace',
|
||||||
|
},
|
||||||
|
'description': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'The description of the repository. May be an empty string',
|
||||||
|
},
|
||||||
|
'last_updated': {
|
||||||
|
'type': 'number',
|
||||||
|
'description': 'The date/time when the repository was last updated, since epoch in UTC',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'The URL at which to view the repository in the browser',
|
||||||
|
},
|
||||||
|
'has_admin_permissions': {
|
||||||
|
'type': 'boolean',
|
||||||
|
'description': 'True if the current user has admin permissions on the repository',
|
||||||
|
},
|
||||||
|
'private': {
|
||||||
|
'type': 'boolean',
|
||||||
|
'description': 'True if the repository is private',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['name', 'full_name', 'description', 'last_updated',
|
||||||
|
'has_admin_permissions', 'private'],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
METADATA_SCHEMA = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'commit': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'first 7 characters of the SHA-1 identifier for a git commit',
|
||||||
|
'pattern': '^([A-Fa-f0-9]{7,})$',
|
||||||
|
},
|
||||||
|
'git_url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'The GIT url to use for the checkout',
|
||||||
|
},
|
||||||
|
'ref': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'git reference for a git commit',
|
||||||
|
'pattern': r'^refs\/(heads|tags|remotes)\/(.+)$',
|
||||||
|
},
|
||||||
|
'default_branch': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'default branch of the git repository',
|
||||||
|
},
|
||||||
|
'commit_info': {
|
||||||
|
'type': 'object',
|
||||||
|
'description': 'metadata about a git commit',
|
||||||
|
'properties': {
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view a git commit',
|
||||||
|
},
|
||||||
|
'message': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'git commit message',
|
||||||
|
},
|
||||||
|
'date': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'timestamp for a git commit'
|
||||||
|
},
|
||||||
|
'author': {
|
||||||
|
'type': 'object',
|
||||||
|
'description': 'metadata about the author of a git commit',
|
||||||
|
'properties': {
|
||||||
|
'username': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'username of the author',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the profile of the author',
|
||||||
|
},
|
||||||
|
'avatar_url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the avatar of the author',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['username'],
|
||||||
|
},
|
||||||
|
'committer': {
|
||||||
|
'type': 'object',
|
||||||
|
'description': 'metadata about the committer of a git commit',
|
||||||
|
'properties': {
|
||||||
|
'username': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'username of the committer',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the profile of the committer',
|
||||||
|
},
|
||||||
|
'avatar_url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the avatar of the committer',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['username'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['message'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['commit', 'git_url'],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@add_metaclass(ABCMeta)
|
||||||
|
class BuildTriggerHandler(object):
|
||||||
|
def __init__(self, trigger, override_config=None):
|
||||||
|
self.trigger = trigger
|
||||||
|
self.config = override_config or get_trigger_config(trigger)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def auth_token(self):
|
||||||
|
""" Returns the auth token for the trigger. """
|
||||||
|
# NOTE: This check is for testing.
|
||||||
|
if isinstance(self.trigger.auth_token, str):
|
||||||
|
return self.trigger.auth_token
|
||||||
|
|
||||||
|
# TODO(remove-unenc): Remove legacy field.
|
||||||
|
if self.trigger.secure_auth_token is not None:
|
||||||
|
return self.trigger.secure_auth_token.decrypt()
|
||||||
|
|
||||||
|
if ActiveDataMigration.has_flag(ERTMigrationFlags.READ_OLD_FIELDS):
|
||||||
|
return self.trigger.auth_token
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def load_dockerfile_contents(self):
|
||||||
|
"""
|
||||||
|
Loads the Dockerfile found for the trigger's config and returns them or None if none could
|
||||||
|
be found/loaded.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def list_build_source_namespaces(self):
|
||||||
|
"""
|
||||||
|
Take the auth information for the specific trigger type and load the
|
||||||
|
list of namespaces that can contain build sources.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def list_build_sources_for_namespace(self, namespace):
|
||||||
|
"""
|
||||||
|
Take the auth information for the specific trigger type and load the
|
||||||
|
list of repositories under the given namespace.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def list_build_subdirs(self):
|
||||||
|
"""
|
||||||
|
Take the auth information and the specified config so far and list all of
|
||||||
|
the possible subdirs containing dockerfiles.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def handle_trigger_request(self, request):
|
||||||
|
"""
|
||||||
|
Transform the incoming request data into a set of actions. Returns a PreparedBuild.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def is_active(self):
|
||||||
|
"""
|
||||||
|
Returns True if the current build trigger is active. Inactive means further
|
||||||
|
setup is needed.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def activate(self, standard_webhook_url):
|
||||||
|
"""
|
||||||
|
Activates the trigger for the service, with the given new configuration.
|
||||||
|
Returns new public and private config that should be stored if successful.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def deactivate(self):
|
||||||
|
"""
|
||||||
|
Deactivates the trigger for the service, removing any hooks installed in
|
||||||
|
the remote service. Returns the new config that should be stored if this
|
||||||
|
trigger is going to be re-activated.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def manual_start(self, run_parameters=None):
|
||||||
|
"""
|
||||||
|
Manually creates a repository build for this trigger. Returns a PreparedBuild.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def list_field_values(self, field_name, limit=None):
|
||||||
|
"""
|
||||||
|
Lists all values for the given custom trigger field. For example, a trigger might have a
|
||||||
|
field named "branches", and this method would return all branches.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_repository_url(self):
|
||||||
|
""" Returns the URL of the current trigger's repository. Note that this operation
|
||||||
|
can be called in a loop, so it should be as fast as possible. """
|
||||||
|
pass
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def filename_is_dockerfile(cls, file_name):
|
||||||
|
""" Returns whether the file is named Dockerfile or follows the convention <name>.Dockerfile"""
|
||||||
|
return file_name.endswith(".Dockerfile") or u"Dockerfile" == file_name
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def service_name(cls):
|
||||||
|
"""
|
||||||
|
Particular service implemented by subclasses.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_handler(cls, trigger, override_config=None):
|
||||||
|
for subc in cls.__subclasses__():
|
||||||
|
if subc.service_name() == trigger.service.name:
|
||||||
|
return subc(trigger, override_config)
|
||||||
|
|
||||||
|
raise InvalidServiceException('Unable to find service: %s' % trigger.service.name)
|
||||||
|
|
||||||
|
def put_config_key(self, key, value):
|
||||||
|
""" Updates a config key in the trigger, saving it to the DB. """
|
||||||
|
self.config[key] = value
|
||||||
|
model.build.update_build_trigger(self.trigger, self.config)
|
||||||
|
|
||||||
|
def set_auth_token(self, auth_token):
|
||||||
|
""" Sets the auth token for the trigger, saving it to the DB. """
|
||||||
|
model.build.update_build_trigger(self.trigger, self.config, auth_token=auth_token)
|
||||||
|
|
||||||
|
def get_dockerfile_path(self):
|
||||||
|
""" Returns the normalized path to the Dockerfile found in the subdirectory
|
||||||
|
in the config. """
|
||||||
|
dockerfile_path = self.config.get('dockerfile_path') or 'Dockerfile'
|
||||||
|
if dockerfile_path[0] == '/':
|
||||||
|
dockerfile_path = dockerfile_path[1:]
|
||||||
|
return dockerfile_path
|
||||||
|
|
||||||
|
def prepare_build(self, metadata, is_manual=False):
|
||||||
|
# Ensure that the metadata meets the scheme.
|
||||||
|
validate(metadata, METADATA_SCHEMA)
|
||||||
|
|
||||||
|
config = self.config
|
||||||
|
ref = metadata.get('ref', None)
|
||||||
|
commit_sha = metadata['commit']
|
||||||
|
default_branch = metadata.get('default_branch', None)
|
||||||
|
prepared = PreparedBuild(self.trigger)
|
||||||
|
prepared.name_from_sha(commit_sha)
|
||||||
|
prepared.subdirectory = config.get('dockerfile_path', None)
|
||||||
|
prepared.context = config.get('context', None)
|
||||||
|
prepared.is_manual = is_manual
|
||||||
|
prepared.metadata = metadata
|
||||||
|
|
||||||
|
if ref is not None:
|
||||||
|
prepared.tags_from_ref(ref, default_branch)
|
||||||
|
else:
|
||||||
|
prepared.tags = [commit_sha[:7]]
|
||||||
|
|
||||||
|
return prepared
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def build_sources_response(cls, sources):
|
||||||
|
validate(sources, BUILD_SOURCES_SCHEMA)
|
||||||
|
return sources
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def build_namespaces_response(cls, namespaces_dict):
|
||||||
|
namespaces = list(namespaces_dict.values())
|
||||||
|
validate(namespaces, NAMESPACES_SCHEMA)
|
||||||
|
return namespaces
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_parent_directory_mappings(cls, dockerfile_path, current_paths=None):
|
||||||
|
""" Returns a map of dockerfile_paths to it's possible contexts. """
|
||||||
|
if dockerfile_path == "":
|
||||||
|
return {}
|
||||||
|
|
||||||
|
if dockerfile_path[0] != os.path.sep:
|
||||||
|
dockerfile_path = os.path.sep + dockerfile_path
|
||||||
|
|
||||||
|
dockerfile_path = os.path.normpath(dockerfile_path)
|
||||||
|
all_paths = set()
|
||||||
|
path, _ = os.path.split(dockerfile_path)
|
||||||
|
if path == "":
|
||||||
|
path = os.path.sep
|
||||||
|
|
||||||
|
all_paths.add(path)
|
||||||
|
for i in range(1, len(path.split(os.path.sep))):
|
||||||
|
path, _ = os.path.split(path)
|
||||||
|
all_paths.add(path)
|
||||||
|
|
||||||
|
if current_paths:
|
||||||
|
return dict({dockerfile_path: list(all_paths)}, **current_paths)
|
||||||
|
|
||||||
|
return {dockerfile_path: list(all_paths)}
|
||||||
545
buildtrigger/bitbuckethandler.py
Normal file
545
buildtrigger/bitbuckethandler.py
Normal file
|
|
@ -0,0 +1,545 @@
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from calendar import timegm
|
||||||
|
|
||||||
|
import dateutil.parser
|
||||||
|
from bitbucket import BitBucket
|
||||||
|
from jsonschema import validate
|
||||||
|
|
||||||
|
from app import app, get_app_url
|
||||||
|
from buildtrigger.basehandler import BuildTriggerHandler
|
||||||
|
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
||||||
|
TriggerDeactivationException, TriggerStartException,
|
||||||
|
InvalidPayloadException, TriggerProviderException,
|
||||||
|
SkipRequestException,
|
||||||
|
determine_build_ref, raise_if_skipped_build,
|
||||||
|
find_matching_branches)
|
||||||
|
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
||||||
|
from util.security.ssh import generate_ssh_keypair
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BITBUCKET_COMMIT_URL = 'https://bitbucket.org/%s/commits/%s'
|
||||||
|
_RAW_AUTHOR_REGEX = re.compile(r'.*<(.+)>')
|
||||||
|
|
||||||
|
BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'repository': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'full_name': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['full_name'],
|
||||||
|
}, # /Repository
|
||||||
|
'push': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'changes': {
|
||||||
|
'type': 'array',
|
||||||
|
'items': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'new': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'target': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'hash': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
'message': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
'date': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
'author': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'user': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'display_name': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'account_id': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'links': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'avatar': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'href': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['href'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['avatar'],
|
||||||
|
}, # /User
|
||||||
|
},
|
||||||
|
}, # /Author
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['hash', 'message', 'date'],
|
||||||
|
}, # /Target
|
||||||
|
},
|
||||||
|
'required': ['name', 'target'],
|
||||||
|
}, # /New
|
||||||
|
},
|
||||||
|
}, # /Changes item
|
||||||
|
}, # /Changes
|
||||||
|
},
|
||||||
|
'required': ['changes'],
|
||||||
|
}, # / Push
|
||||||
|
},
|
||||||
|
'actor': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'account_id': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'display_name': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'links': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'avatar': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'href': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['href'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['avatar'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, # /Actor
|
||||||
|
'required': ['push', 'repository'],
|
||||||
|
} # /Root
|
||||||
|
|
||||||
|
BITBUCKET_COMMIT_INFO_SCHEMA = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'node': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'message': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'timestamp': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'raw_author': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['node', 'message', 'timestamp']
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_transformed_commit_info(bb_commit, ref, default_branch, repository_name, lookup_author):
|
||||||
|
""" Returns the BitBucket commit information transformed into our own
|
||||||
|
payload format.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
validate(bb_commit, BITBUCKET_COMMIT_INFO_SCHEMA)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.exception('Exception when validating Bitbucket commit information: %s from %s', exc.message, bb_commit)
|
||||||
|
raise InvalidPayloadException(exc.message)
|
||||||
|
|
||||||
|
commit = JSONPathDict(bb_commit)
|
||||||
|
|
||||||
|
config = SafeDictSetter()
|
||||||
|
config['commit'] = commit['node']
|
||||||
|
config['ref'] = ref
|
||||||
|
config['default_branch'] = default_branch
|
||||||
|
config['git_url'] = 'git@bitbucket.org:%s.git' % repository_name
|
||||||
|
|
||||||
|
config['commit_info.url'] = _BITBUCKET_COMMIT_URL % (repository_name, commit['node'])
|
||||||
|
config['commit_info.message'] = commit['message']
|
||||||
|
config['commit_info.date'] = commit['timestamp']
|
||||||
|
|
||||||
|
match = _RAW_AUTHOR_REGEX.match(commit['raw_author'])
|
||||||
|
if match:
|
||||||
|
author = lookup_author(match.group(1))
|
||||||
|
author_info = JSONPathDict(author) if author is not None else None
|
||||||
|
if author_info:
|
||||||
|
config['commit_info.author.username'] = author_info['user.display_name']
|
||||||
|
config['commit_info.author.avatar_url'] = author_info['user.avatar']
|
||||||
|
|
||||||
|
return config.dict_value()
|
||||||
|
|
||||||
|
|
||||||
|
def get_transformed_webhook_payload(bb_payload, default_branch=None):
|
||||||
|
""" Returns the BitBucket webhook JSON payload transformed into our own payload
|
||||||
|
format. If the bb_payload is not valid, returns None.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
validate(bb_payload, BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.exception('Exception when validating Bitbucket webhook payload: %s from %s', exc.message,
|
||||||
|
bb_payload)
|
||||||
|
raise InvalidPayloadException(exc.message)
|
||||||
|
|
||||||
|
payload = JSONPathDict(bb_payload)
|
||||||
|
change = payload['push.changes[-1].new']
|
||||||
|
if not change:
|
||||||
|
raise SkipRequestException
|
||||||
|
|
||||||
|
is_branch = change['type'] == 'branch'
|
||||||
|
ref = 'refs/heads/' + change['name'] if is_branch else 'refs/tags/' + change['name']
|
||||||
|
|
||||||
|
repository_name = payload['repository.full_name']
|
||||||
|
target = change['target']
|
||||||
|
|
||||||
|
config = SafeDictSetter()
|
||||||
|
config['commit'] = target['hash']
|
||||||
|
config['ref'] = ref
|
||||||
|
config['default_branch'] = default_branch
|
||||||
|
config['git_url'] = 'git@bitbucket.org:%s.git' % repository_name
|
||||||
|
|
||||||
|
config['commit_info.url'] = target['links.html.href'] or ''
|
||||||
|
config['commit_info.message'] = target['message']
|
||||||
|
config['commit_info.date'] = target['date']
|
||||||
|
|
||||||
|
config['commit_info.author.username'] = target['author.user.display_name']
|
||||||
|
config['commit_info.author.avatar_url'] = target['author.user.links.avatar.href']
|
||||||
|
|
||||||
|
config['commit_info.committer.username'] = payload['actor.display_name']
|
||||||
|
config['commit_info.committer.avatar_url'] = payload['actor.links.avatar.href']
|
||||||
|
return config.dict_value()
|
||||||
|
|
||||||
|
|
||||||
|
class BitbucketBuildTrigger(BuildTriggerHandler):
|
||||||
|
"""
|
||||||
|
BuildTrigger for Bitbucket.
|
||||||
|
"""
|
||||||
|
@classmethod
|
||||||
|
def service_name(cls):
|
||||||
|
return 'bitbucket'
|
||||||
|
|
||||||
|
def _get_client(self):
|
||||||
|
""" Returns a BitBucket API client for this trigger's config. """
|
||||||
|
key = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_KEY', '')
|
||||||
|
secret = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_SECRET', '')
|
||||||
|
|
||||||
|
trigger_uuid = self.trigger.uuid
|
||||||
|
callback_url = '%s/oauth1/bitbucket/callback/trigger/%s' % (get_app_url(), trigger_uuid)
|
||||||
|
|
||||||
|
return BitBucket(key, secret, callback_url, timeout=15)
|
||||||
|
|
||||||
|
def _get_authorized_client(self):
|
||||||
|
""" Returns an authorized API client. """
|
||||||
|
base_client = self._get_client()
|
||||||
|
auth_token = self.auth_token or 'invalid:invalid'
|
||||||
|
token_parts = auth_token.split(':')
|
||||||
|
if len(token_parts) != 2:
|
||||||
|
token_parts = ['invalid', 'invalid']
|
||||||
|
|
||||||
|
(access_token, access_token_secret) = token_parts
|
||||||
|
return base_client.get_authorized_client(access_token, access_token_secret)
|
||||||
|
|
||||||
|
def _get_repository_client(self):
|
||||||
|
""" Returns an API client for working with this config's BB repository. """
|
||||||
|
source = self.config['build_source']
|
||||||
|
(namespace, name) = source.split('/')
|
||||||
|
bitbucket_client = self._get_authorized_client()
|
||||||
|
return bitbucket_client.for_namespace(namespace).repositories().get(name)
|
||||||
|
|
||||||
|
def _get_default_branch(self, repository, default_value='master'):
|
||||||
|
""" Returns the default branch for the repository or the value given. """
|
||||||
|
(result, data, _) = repository.get_main_branch()
|
||||||
|
if result:
|
||||||
|
return data['name']
|
||||||
|
|
||||||
|
return default_value
|
||||||
|
|
||||||
|
def get_oauth_url(self):
|
||||||
|
""" Returns the OAuth URL to authorize Bitbucket. """
|
||||||
|
bitbucket_client = self._get_client()
|
||||||
|
(result, data, err_msg) = bitbucket_client.get_authorization_url()
|
||||||
|
if not result:
|
||||||
|
raise TriggerProviderException(err_msg)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
def exchange_verifier(self, verifier):
|
||||||
|
""" Exchanges the given verifier token to setup this trigger. """
|
||||||
|
bitbucket_client = self._get_client()
|
||||||
|
access_token = self.config.get('access_token', '')
|
||||||
|
access_token_secret = self.auth_token
|
||||||
|
|
||||||
|
# Exchange the verifier for a new access token.
|
||||||
|
(result, data, _) = bitbucket_client.verify_token(access_token, access_token_secret, verifier)
|
||||||
|
if not result:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Save the updated access token and secret.
|
||||||
|
self.set_auth_token(data[0] + ':' + data[1])
|
||||||
|
|
||||||
|
# Retrieve the current authorized user's information and store the username in the config.
|
||||||
|
authorized_client = self._get_authorized_client()
|
||||||
|
(result, data, _) = authorized_client.get_current_user()
|
||||||
|
if not result:
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.put_config_key('account_id', data['user']['account_id'])
|
||||||
|
self.put_config_key('nickname', data['user']['nickname'])
|
||||||
|
return True
|
||||||
|
|
||||||
|
def is_active(self):
|
||||||
|
return 'webhook_id' in self.config
|
||||||
|
|
||||||
|
def activate(self, standard_webhook_url):
|
||||||
|
config = self.config
|
||||||
|
|
||||||
|
# Add a deploy key to the repository.
|
||||||
|
public_key, private_key = generate_ssh_keypair()
|
||||||
|
config['credentials'] = [
|
||||||
|
{
|
||||||
|
'name': 'SSH Public Key',
|
||||||
|
'value': public_key,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
repository = self._get_repository_client()
|
||||||
|
(result, created_deploykey, err_msg) = repository.deploykeys().create(
|
||||||
|
app.config['REGISTRY_TITLE'] + ' webhook key', public_key)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
msg = 'Unable to add deploy key to repository: %s' % err_msg
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
config['deploy_key_id'] = created_deploykey['pk']
|
||||||
|
|
||||||
|
# Add a webhook callback.
|
||||||
|
description = 'Webhook for invoking builds on %s' % app.config['REGISTRY_TITLE_SHORT']
|
||||||
|
webhook_events = ['repo:push']
|
||||||
|
(result, created_webhook, err_msg) = repository.webhooks().create(
|
||||||
|
description, standard_webhook_url, webhook_events)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
msg = 'Unable to add webhook to repository: %s' % err_msg
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
config['webhook_id'] = created_webhook['uuid']
|
||||||
|
self.config = config
|
||||||
|
return config, {'private_key': private_key}
|
||||||
|
|
||||||
|
def deactivate(self):
|
||||||
|
config = self.config
|
||||||
|
|
||||||
|
webhook_id = config.pop('webhook_id', None)
|
||||||
|
deploy_key_id = config.pop('deploy_key_id', None)
|
||||||
|
repository = self._get_repository_client()
|
||||||
|
|
||||||
|
# Remove the webhook.
|
||||||
|
if webhook_id is not None:
|
||||||
|
(result, _, err_msg) = repository.webhooks().delete(webhook_id)
|
||||||
|
if not result:
|
||||||
|
msg = 'Unable to remove webhook from repository: %s' % err_msg
|
||||||
|
raise TriggerDeactivationException(msg)
|
||||||
|
|
||||||
|
# Remove the public key.
|
||||||
|
if deploy_key_id is not None:
|
||||||
|
(result, _, err_msg) = repository.deploykeys().delete(deploy_key_id)
|
||||||
|
if not result:
|
||||||
|
msg = 'Unable to remove deploy key from repository: %s' % err_msg
|
||||||
|
raise TriggerDeactivationException(msg)
|
||||||
|
|
||||||
|
return config
|
||||||
|
|
||||||
|
def list_build_source_namespaces(self):
|
||||||
|
bitbucket_client = self._get_authorized_client()
|
||||||
|
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
|
||||||
|
if not result:
|
||||||
|
raise RepositoryReadException('Could not read repository list: ' + err_msg)
|
||||||
|
|
||||||
|
namespaces = {}
|
||||||
|
for repo in data:
|
||||||
|
owner = repo['owner']
|
||||||
|
|
||||||
|
if owner in namespaces:
|
||||||
|
namespaces[owner]['score'] = namespaces[owner]['score'] + 1
|
||||||
|
else:
|
||||||
|
namespaces[owner] = {
|
||||||
|
'personal': owner == self.config.get('nickname', self.config.get('username')),
|
||||||
|
'id': owner,
|
||||||
|
'title': owner,
|
||||||
|
'avatar_url': repo['logo'],
|
||||||
|
'url': 'https://bitbucket.org/%s' % (owner),
|
||||||
|
'score': 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
return BuildTriggerHandler.build_namespaces_response(namespaces)
|
||||||
|
|
||||||
|
def list_build_sources_for_namespace(self, namespace):
|
||||||
|
def repo_view(repo):
|
||||||
|
last_modified = dateutil.parser.parse(repo['utc_last_updated'])
|
||||||
|
|
||||||
|
return {
|
||||||
|
'name': repo['slug'],
|
||||||
|
'full_name': '%s/%s' % (repo['owner'], repo['slug']),
|
||||||
|
'description': repo['description'] or '',
|
||||||
|
'last_updated': timegm(last_modified.utctimetuple()),
|
||||||
|
'url': 'https://bitbucket.org/%s/%s' % (repo['owner'], repo['slug']),
|
||||||
|
'has_admin_permissions': repo['read_only'] is False,
|
||||||
|
'private': repo['is_private'],
|
||||||
|
}
|
||||||
|
|
||||||
|
bitbucket_client = self._get_authorized_client()
|
||||||
|
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
|
||||||
|
if not result:
|
||||||
|
raise RepositoryReadException('Could not read repository list: ' + err_msg)
|
||||||
|
|
||||||
|
repos = [repo_view(repo) for repo in data if repo['owner'] == namespace]
|
||||||
|
return BuildTriggerHandler.build_sources_response(repos)
|
||||||
|
|
||||||
|
def list_build_subdirs(self):
|
||||||
|
config = self.config
|
||||||
|
repository = self._get_repository_client()
|
||||||
|
|
||||||
|
# Find the first matching branch.
|
||||||
|
repo_branches = self.list_field_values('branch_name') or []
|
||||||
|
branches = find_matching_branches(config, repo_branches)
|
||||||
|
if not branches:
|
||||||
|
branches = [self._get_default_branch(repository)]
|
||||||
|
|
||||||
|
(result, data, err_msg) = repository.get_path_contents('', revision=branches[0])
|
||||||
|
if not result:
|
||||||
|
raise RepositoryReadException(err_msg)
|
||||||
|
|
||||||
|
files = set([f['path'] for f in data['files']])
|
||||||
|
return ["/" + file_path for file_path in files if self.filename_is_dockerfile(os.path.basename(file_path))]
|
||||||
|
|
||||||
|
def load_dockerfile_contents(self):
|
||||||
|
repository = self._get_repository_client()
|
||||||
|
path = self.get_dockerfile_path()
|
||||||
|
|
||||||
|
(result, data, err_msg) = repository.get_raw_path_contents(path, revision='master')
|
||||||
|
if not result:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
def list_field_values(self, field_name, limit=None):
|
||||||
|
if 'build_source' not in self.config:
|
||||||
|
return None
|
||||||
|
|
||||||
|
source = self.config['build_source']
|
||||||
|
(namespace, name) = source.split('/')
|
||||||
|
|
||||||
|
bitbucket_client = self._get_authorized_client()
|
||||||
|
repository = bitbucket_client.for_namespace(namespace).repositories().get(name)
|
||||||
|
|
||||||
|
if field_name == 'refs':
|
||||||
|
(result, data, _) = repository.get_branches_and_tags()
|
||||||
|
if not result:
|
||||||
|
return None
|
||||||
|
|
||||||
|
branches = [b['name'] for b in data['branches']]
|
||||||
|
tags = [t['name'] for t in data['tags']]
|
||||||
|
|
||||||
|
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||||
|
[{'kind': 'tag', 'name': tag} for tag in tags])
|
||||||
|
|
||||||
|
if field_name == 'tag_name':
|
||||||
|
(result, data, _) = repository.get_tags()
|
||||||
|
if not result:
|
||||||
|
return None
|
||||||
|
|
||||||
|
tags = list(data.keys())
|
||||||
|
if limit:
|
||||||
|
tags = tags[0:limit]
|
||||||
|
|
||||||
|
return tags
|
||||||
|
|
||||||
|
if field_name == 'branch_name':
|
||||||
|
(result, data, _) = repository.get_branches()
|
||||||
|
if not result:
|
||||||
|
return None
|
||||||
|
|
||||||
|
branches = list(data.keys())
|
||||||
|
if limit:
|
||||||
|
branches = branches[0:limit]
|
||||||
|
|
||||||
|
return branches
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_repository_url(self):
|
||||||
|
source = self.config['build_source']
|
||||||
|
(namespace, name) = source.split('/')
|
||||||
|
return 'https://bitbucket.org/%s/%s' % (namespace, name)
|
||||||
|
|
||||||
|
def handle_trigger_request(self, request):
|
||||||
|
payload = request.get_json()
|
||||||
|
if payload is None:
|
||||||
|
raise InvalidPayloadException('Missing payload')
|
||||||
|
|
||||||
|
logger.debug('Got BitBucket request: %s', payload)
|
||||||
|
|
||||||
|
repository = self._get_repository_client()
|
||||||
|
default_branch = self._get_default_branch(repository)
|
||||||
|
|
||||||
|
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch)
|
||||||
|
prepared = self.prepare_build(metadata)
|
||||||
|
|
||||||
|
# Check if we should skip this build.
|
||||||
|
raise_if_skipped_build(prepared, self.config)
|
||||||
|
return prepared
|
||||||
|
|
||||||
|
def manual_start(self, run_parameters=None):
|
||||||
|
run_parameters = run_parameters or {}
|
||||||
|
repository = self._get_repository_client()
|
||||||
|
bitbucket_client = self._get_authorized_client()
|
||||||
|
|
||||||
|
def get_branch_sha(branch_name):
|
||||||
|
# Lookup the commit SHA for the branch.
|
||||||
|
(result, data, _) = repository.get_branch(branch_name)
|
||||||
|
if not result:
|
||||||
|
raise TriggerStartException('Could not find branch in repository')
|
||||||
|
|
||||||
|
return data['target']['hash']
|
||||||
|
|
||||||
|
def get_tag_sha(tag_name):
|
||||||
|
# Lookup the commit SHA for the tag.
|
||||||
|
(result, data, _) = repository.get_tag(tag_name)
|
||||||
|
if not result:
|
||||||
|
raise TriggerStartException('Could not find tag in repository')
|
||||||
|
|
||||||
|
return data['target']['hash']
|
||||||
|
|
||||||
|
def lookup_author(email_address):
|
||||||
|
(result, data, _) = bitbucket_client.accounts().get_profile(email_address)
|
||||||
|
return data if result else None
|
||||||
|
|
||||||
|
# Find the branch or tag to build.
|
||||||
|
default_branch = self._get_default_branch(repository)
|
||||||
|
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
|
||||||
|
default_branch)
|
||||||
|
|
||||||
|
# Lookup the commit SHA in BitBucket.
|
||||||
|
(result, commit_info, _) = repository.changesets().get(commit_sha)
|
||||||
|
if not result:
|
||||||
|
raise TriggerStartException('Could not lookup commit SHA')
|
||||||
|
|
||||||
|
# Return a prepared build for the commit.
|
||||||
|
repository_name = '%s/%s' % (repository.namespace, repository.repository_name)
|
||||||
|
metadata = get_transformed_commit_info(commit_info, ref, default_branch,
|
||||||
|
repository_name, lookup_author)
|
||||||
|
|
||||||
|
return self.prepare_build(metadata, is_manual=True)
|
||||||
229
buildtrigger/customhandler.py
Normal file
229
buildtrigger/customhandler.py
Normal file
|
|
@ -0,0 +1,229 @@
|
||||||
|
import logging
|
||||||
|
import json
|
||||||
|
|
||||||
|
from jsonschema import validate, ValidationError
|
||||||
|
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
||||||
|
TriggerStartException, ValidationRequestException,
|
||||||
|
InvalidPayloadException,
|
||||||
|
SkipRequestException, raise_if_skipped_build,
|
||||||
|
find_matching_branches)
|
||||||
|
|
||||||
|
from buildtrigger.basehandler import BuildTriggerHandler
|
||||||
|
|
||||||
|
from buildtrigger.bitbuckethandler import (BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA as bb_schema,
|
||||||
|
get_transformed_webhook_payload as bb_payload)
|
||||||
|
|
||||||
|
from buildtrigger.githubhandler import (GITHUB_WEBHOOK_PAYLOAD_SCHEMA as gh_schema,
|
||||||
|
get_transformed_webhook_payload as gh_payload)
|
||||||
|
|
||||||
|
from buildtrigger.gitlabhandler import (GITLAB_WEBHOOK_PAYLOAD_SCHEMA as gl_schema,
|
||||||
|
get_transformed_webhook_payload as gl_payload)
|
||||||
|
|
||||||
|
from util.security.ssh import generate_ssh_keypair
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Defines an ordered set of tuples of the schemas and associated transformation functions
|
||||||
|
# for incoming webhook payloads.
|
||||||
|
SCHEMA_AND_HANDLERS = [
|
||||||
|
(gh_schema, gh_payload),
|
||||||
|
(bb_schema, bb_payload),
|
||||||
|
(gl_schema, gl_payload),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def custom_trigger_payload(metadata, git_url):
|
||||||
|
# First try the customhandler schema. If it matches, nothing more to do.
|
||||||
|
custom_handler_validation_error = None
|
||||||
|
try:
|
||||||
|
validate(metadata, CustomBuildTrigger.payload_schema)
|
||||||
|
except ValidationError as vex:
|
||||||
|
custom_handler_validation_error = vex
|
||||||
|
|
||||||
|
# Otherwise, try the defined schemas, in order, until we find a match.
|
||||||
|
for schema, handler in SCHEMA_AND_HANDLERS:
|
||||||
|
try:
|
||||||
|
validate(metadata, schema)
|
||||||
|
except ValidationError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
result = handler(metadata)
|
||||||
|
result['git_url'] = git_url
|
||||||
|
return result
|
||||||
|
|
||||||
|
# If we have reached this point and no other schemas validated, then raise the error for the
|
||||||
|
# custom schema.
|
||||||
|
if custom_handler_validation_error is not None:
|
||||||
|
raise InvalidPayloadException(custom_handler_validation_error.message)
|
||||||
|
|
||||||
|
metadata['git_url'] = git_url
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
|
class CustomBuildTrigger(BuildTriggerHandler):
|
||||||
|
payload_schema = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'commit': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'first 7 characters of the SHA-1 identifier for a git commit',
|
||||||
|
'pattern': '^([A-Fa-f0-9]{7,})$',
|
||||||
|
},
|
||||||
|
'ref': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'git reference for a git commit',
|
||||||
|
'pattern': '^refs\/(heads|tags|remotes)\/(.+)$',
|
||||||
|
},
|
||||||
|
'default_branch': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'default branch of the git repository',
|
||||||
|
},
|
||||||
|
'commit_info': {
|
||||||
|
'type': 'object',
|
||||||
|
'description': 'metadata about a git commit',
|
||||||
|
'properties': {
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view a git commit',
|
||||||
|
},
|
||||||
|
'message': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'git commit message',
|
||||||
|
},
|
||||||
|
'date': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'timestamp for a git commit'
|
||||||
|
},
|
||||||
|
'author': {
|
||||||
|
'type': 'object',
|
||||||
|
'description': 'metadata about the author of a git commit',
|
||||||
|
'properties': {
|
||||||
|
'username': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'username of the author',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the profile of the author',
|
||||||
|
},
|
||||||
|
'avatar_url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the avatar of the author',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['username', 'url', 'avatar_url'],
|
||||||
|
},
|
||||||
|
'committer': {
|
||||||
|
'type': 'object',
|
||||||
|
'description': 'metadata about the committer of a git commit',
|
||||||
|
'properties': {
|
||||||
|
'username': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'username of the committer',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the profile of the committer',
|
||||||
|
},
|
||||||
|
'avatar_url': {
|
||||||
|
'type': 'string',
|
||||||
|
'description': 'URL to view the avatar of the committer',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['username', 'url', 'avatar_url'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['url', 'message', 'date'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['commit', 'ref', 'default_branch'],
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def service_name(cls):
|
||||||
|
return 'custom-git'
|
||||||
|
|
||||||
|
def is_active(self):
|
||||||
|
return self.config.has_key('credentials')
|
||||||
|
|
||||||
|
def _metadata_from_payload(self, payload, git_url):
|
||||||
|
# Parse the JSON payload.
|
||||||
|
try:
|
||||||
|
metadata = json.loads(payload)
|
||||||
|
except ValueError as vex:
|
||||||
|
raise InvalidPayloadException(vex.message)
|
||||||
|
|
||||||
|
return custom_trigger_payload(metadata, git_url)
|
||||||
|
|
||||||
|
def handle_trigger_request(self, request):
|
||||||
|
payload = request.data
|
||||||
|
if not payload:
|
||||||
|
raise InvalidPayloadException('Missing expected payload')
|
||||||
|
|
||||||
|
logger.debug('Payload %s', payload)
|
||||||
|
|
||||||
|
metadata = self._metadata_from_payload(payload, self.config['build_source'])
|
||||||
|
prepared = self.prepare_build(metadata)
|
||||||
|
|
||||||
|
# Check if we should skip this build.
|
||||||
|
raise_if_skipped_build(prepared, self.config)
|
||||||
|
|
||||||
|
return prepared
|
||||||
|
|
||||||
|
def manual_start(self, run_parameters=None):
|
||||||
|
# commit_sha is the only required parameter
|
||||||
|
commit_sha = run_parameters.get('commit_sha')
|
||||||
|
if commit_sha is None:
|
||||||
|
raise TriggerStartException('missing required parameter')
|
||||||
|
|
||||||
|
config = self.config
|
||||||
|
metadata = {
|
||||||
|
'commit': commit_sha,
|
||||||
|
'git_url': config['build_source'],
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
return self.prepare_build(metadata, is_manual=True)
|
||||||
|
except ValidationError as ve:
|
||||||
|
raise TriggerStartException(ve.message)
|
||||||
|
|
||||||
|
def activate(self, standard_webhook_url):
|
||||||
|
config = self.config
|
||||||
|
public_key, private_key = generate_ssh_keypair()
|
||||||
|
config['credentials'] = [
|
||||||
|
{
|
||||||
|
'name': 'SSH Public Key',
|
||||||
|
'value': public_key,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Webhook Endpoint URL',
|
||||||
|
'value': standard_webhook_url,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
self.config = config
|
||||||
|
return config, {'private_key': private_key}
|
||||||
|
|
||||||
|
def deactivate(self):
|
||||||
|
config = self.config
|
||||||
|
config.pop('credentials', None)
|
||||||
|
self.config = config
|
||||||
|
return config
|
||||||
|
|
||||||
|
def get_repository_url(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def list_build_source_namespaces(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def list_build_sources_for_namespace(self, namespace):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def list_build_subdirs(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def list_field_values(self, field_name, limit=None):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def load_dockerfile_contents(self):
|
||||||
|
raise NotImplementedError
|
||||||
587
buildtrigger/githubhandler.py
Normal file
587
buildtrigger/githubhandler.py
Normal file
|
|
@ -0,0 +1,587 @@
|
||||||
|
import logging
|
||||||
|
import os.path
|
||||||
|
import base64
|
||||||
|
import re
|
||||||
|
|
||||||
|
from calendar import timegm
|
||||||
|
from functools import wraps
|
||||||
|
from ssl import SSLError
|
||||||
|
|
||||||
|
from github import (Github, UnknownObjectException, GithubException,
|
||||||
|
BadCredentialsException as GitHubBadCredentialsException)
|
||||||
|
|
||||||
|
from jsonschema import validate
|
||||||
|
|
||||||
|
from app import app, github_trigger
|
||||||
|
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
||||||
|
TriggerDeactivationException, TriggerStartException,
|
||||||
|
EmptyRepositoryException, ValidationRequestException,
|
||||||
|
SkipRequestException, InvalidPayloadException,
|
||||||
|
determine_build_ref, raise_if_skipped_build,
|
||||||
|
find_matching_branches)
|
||||||
|
from buildtrigger.basehandler import BuildTriggerHandler
|
||||||
|
from endpoints.exception import ExternalServiceError
|
||||||
|
from util.security.ssh import generate_ssh_keypair
|
||||||
|
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
GITHUB_WEBHOOK_PAYLOAD_SCHEMA = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'ref': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'head_commit': {
|
||||||
|
'type': ['object', 'null'],
|
||||||
|
'properties': {
|
||||||
|
'id': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'message': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'timestamp': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'author': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'username': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
'html_url': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
'avatar_url': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'committer': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'username': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
'html_url': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
'avatar_url': {
|
||||||
|
'type': 'string'
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['id', 'url', 'message', 'timestamp'],
|
||||||
|
},
|
||||||
|
'repository': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'ssh_url': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['ssh_url'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['ref', 'head_commit', 'repository'],
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_transformed_webhook_payload(gh_payload, default_branch=None, lookup_user=None):
|
||||||
|
""" Returns the GitHub webhook JSON payload transformed into our own payload
|
||||||
|
format. If the gh_payload is not valid, returns None.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
validate(gh_payload, GITHUB_WEBHOOK_PAYLOAD_SCHEMA)
|
||||||
|
except Exception as exc:
|
||||||
|
raise InvalidPayloadException(exc.message)
|
||||||
|
|
||||||
|
payload = JSONPathDict(gh_payload)
|
||||||
|
|
||||||
|
if payload['head_commit'] is None:
|
||||||
|
raise SkipRequestException
|
||||||
|
|
||||||
|
config = SafeDictSetter()
|
||||||
|
config['commit'] = payload['head_commit.id']
|
||||||
|
config['ref'] = payload['ref']
|
||||||
|
config['default_branch'] = payload['repository.default_branch'] or default_branch
|
||||||
|
config['git_url'] = payload['repository.ssh_url']
|
||||||
|
|
||||||
|
config['commit_info.url'] = payload['head_commit.url']
|
||||||
|
config['commit_info.message'] = payload['head_commit.message']
|
||||||
|
config['commit_info.date'] = payload['head_commit.timestamp']
|
||||||
|
|
||||||
|
config['commit_info.author.username'] = payload['head_commit.author.username']
|
||||||
|
config['commit_info.author.url'] = payload.get('head_commit.author.html_url')
|
||||||
|
config['commit_info.author.avatar_url'] = payload.get('head_commit.author.avatar_url')
|
||||||
|
|
||||||
|
config['commit_info.committer.username'] = payload.get('head_commit.committer.username')
|
||||||
|
config['commit_info.committer.url'] = payload.get('head_commit.committer.html_url')
|
||||||
|
config['commit_info.committer.avatar_url'] = payload.get('head_commit.committer.avatar_url')
|
||||||
|
|
||||||
|
# Note: GitHub doesn't always return the extra information for users, so we do the lookup
|
||||||
|
# manually if possible.
|
||||||
|
if (lookup_user and not payload.get('head_commit.author.html_url') and
|
||||||
|
payload.get('head_commit.author.username')):
|
||||||
|
author_info = lookup_user(payload['head_commit.author.username'])
|
||||||
|
if author_info:
|
||||||
|
config['commit_info.author.url'] = author_info['html_url']
|
||||||
|
config['commit_info.author.avatar_url'] = author_info['avatar_url']
|
||||||
|
|
||||||
|
if (lookup_user and
|
||||||
|
payload.get('head_commit.committer.username') and
|
||||||
|
not payload.get('head_commit.committer.html_url')):
|
||||||
|
committer_info = lookup_user(payload['head_commit.committer.username'])
|
||||||
|
if committer_info:
|
||||||
|
config['commit_info.committer.url'] = committer_info['html_url']
|
||||||
|
config['commit_info.committer.avatar_url'] = committer_info['avatar_url']
|
||||||
|
|
||||||
|
return config.dict_value()
|
||||||
|
|
||||||
|
|
||||||
|
def _catch_ssl_errors(func):
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
except SSLError as se:
|
||||||
|
msg = 'Request to the GitHub API failed: %s' % se.message
|
||||||
|
logger.exception(msg)
|
||||||
|
raise ExternalServiceError(msg)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
|
"""
|
||||||
|
BuildTrigger for GitHub that uses the archive API and buildpacks.
|
||||||
|
"""
|
||||||
|
def _get_client(self):
|
||||||
|
""" Returns an authenticated client for talking to the GitHub API. """
|
||||||
|
return Github(self.auth_token,
|
||||||
|
base_url=github_trigger.api_endpoint(),
|
||||||
|
client_id=github_trigger.client_id(),
|
||||||
|
client_secret=github_trigger.client_secret(),
|
||||||
|
timeout=5)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def service_name(cls):
|
||||||
|
return 'github'
|
||||||
|
|
||||||
|
def is_active(self):
|
||||||
|
return 'hook_id' in self.config
|
||||||
|
|
||||||
|
def get_repository_url(self):
|
||||||
|
source = self.config['build_source']
|
||||||
|
return github_trigger.get_public_url(source)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_error_message(ghe, default_msg):
|
||||||
|
if ghe.data.get('errors') and ghe.data['errors'][0].get('message'):
|
||||||
|
return ghe.data['errors'][0]['message']
|
||||||
|
|
||||||
|
return default_msg
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def activate(self, standard_webhook_url):
|
||||||
|
config = self.config
|
||||||
|
new_build_source = config['build_source']
|
||||||
|
gh_client = self._get_client()
|
||||||
|
|
||||||
|
# Find the GitHub repository.
|
||||||
|
try:
|
||||||
|
gh_repo = gh_client.get_repo(new_build_source)
|
||||||
|
except UnknownObjectException:
|
||||||
|
msg = 'Unable to find GitHub repository for source: %s' % new_build_source
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
# Add a deploy key to the GitHub repository.
|
||||||
|
public_key, private_key = generate_ssh_keypair()
|
||||||
|
config['credentials'] = [
|
||||||
|
{
|
||||||
|
'name': 'SSH Public Key',
|
||||||
|
'value': public_key,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
deploy_key = gh_repo.create_key('%s Builder' % app.config['REGISTRY_TITLE'],
|
||||||
|
public_key)
|
||||||
|
config['deploy_key_id'] = deploy_key.id
|
||||||
|
except GithubException as ghe:
|
||||||
|
default_msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
||||||
|
msg = GithubBuildTrigger._get_error_message(ghe, default_msg)
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
# Add the webhook to the GitHub repository.
|
||||||
|
webhook_config = {
|
||||||
|
'url': standard_webhook_url,
|
||||||
|
'content_type': 'json',
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
hook = gh_repo.create_hook('web', webhook_config)
|
||||||
|
config['hook_id'] = hook.id
|
||||||
|
config['master_branch'] = gh_repo.default_branch
|
||||||
|
except GithubException as ghe:
|
||||||
|
default_msg = 'Unable to create webhook on repository: %s' % new_build_source
|
||||||
|
msg = GithubBuildTrigger._get_error_message(ghe, default_msg)
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
return config, {'private_key': private_key}
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def deactivate(self):
|
||||||
|
config = self.config
|
||||||
|
gh_client = self._get_client()
|
||||||
|
|
||||||
|
# Find the GitHub repository.
|
||||||
|
try:
|
||||||
|
repo = gh_client.get_repo(config['build_source'])
|
||||||
|
except UnknownObjectException:
|
||||||
|
msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
|
||||||
|
raise TriggerDeactivationException(msg)
|
||||||
|
except GitHubBadCredentialsException:
|
||||||
|
msg = 'Unable to access repository to disable trigger'
|
||||||
|
raise TriggerDeactivationException(msg)
|
||||||
|
|
||||||
|
# If the trigger uses a deploy key, remove it.
|
||||||
|
try:
|
||||||
|
if config['deploy_key_id']:
|
||||||
|
deploy_key = repo.get_key(config['deploy_key_id'])
|
||||||
|
deploy_key.delete()
|
||||||
|
except KeyError:
|
||||||
|
# There was no config['deploy_key_id'], thus this is an old trigger without a deploy key.
|
||||||
|
pass
|
||||||
|
except GithubException as ghe:
|
||||||
|
default_msg = 'Unable to remove deploy key: %s' % config['deploy_key_id']
|
||||||
|
msg = GithubBuildTrigger._get_error_message(ghe, default_msg)
|
||||||
|
raise TriggerDeactivationException(msg)
|
||||||
|
|
||||||
|
# Remove the webhook.
|
||||||
|
if 'hook_id' in config:
|
||||||
|
try:
|
||||||
|
hook = repo.get_hook(config['hook_id'])
|
||||||
|
hook.delete()
|
||||||
|
except GithubException as ghe:
|
||||||
|
default_msg = 'Unable to remove hook: %s' % config['hook_id']
|
||||||
|
msg = GithubBuildTrigger._get_error_message(ghe, default_msg)
|
||||||
|
raise TriggerDeactivationException(msg)
|
||||||
|
|
||||||
|
config.pop('hook_id', None)
|
||||||
|
self.config = config
|
||||||
|
return config
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def list_build_source_namespaces(self):
|
||||||
|
gh_client = self._get_client()
|
||||||
|
usr = gh_client.get_user()
|
||||||
|
|
||||||
|
# Build the full set of namespaces for the user, starting with their own.
|
||||||
|
namespaces = {}
|
||||||
|
namespaces[usr.login] = {
|
||||||
|
'personal': True,
|
||||||
|
'id': usr.login,
|
||||||
|
'title': usr.name or usr.login,
|
||||||
|
'avatar_url': usr.avatar_url,
|
||||||
|
'url': usr.html_url,
|
||||||
|
'score': usr.plan.private_repos if usr.plan else 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for org in usr.get_orgs():
|
||||||
|
organization = org.login if org.login else org.name
|
||||||
|
|
||||||
|
# NOTE: We don't load the organization's html_url nor its plan, because doing
|
||||||
|
# so requires loading *each organization* via its own API call in this tight
|
||||||
|
# loop, which was massively slowing down the load time for users when setting
|
||||||
|
# up triggers.
|
||||||
|
namespaces[organization] = {
|
||||||
|
'personal': False,
|
||||||
|
'id': organization,
|
||||||
|
'title': organization,
|
||||||
|
'avatar_url': org.avatar_url,
|
||||||
|
'url': '',
|
||||||
|
'score': 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
return BuildTriggerHandler.build_namespaces_response(namespaces)
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def list_build_sources_for_namespace(self, namespace):
|
||||||
|
def repo_view(repo):
|
||||||
|
return {
|
||||||
|
'name': repo.name,
|
||||||
|
'full_name': repo.full_name,
|
||||||
|
'description': repo.description or '',
|
||||||
|
'last_updated': timegm(repo.pushed_at.utctimetuple()) if repo.pushed_at else 0,
|
||||||
|
'url': repo.html_url,
|
||||||
|
'has_admin_permissions': repo.permissions.admin,
|
||||||
|
'private': repo.private,
|
||||||
|
}
|
||||||
|
|
||||||
|
gh_client = self._get_client()
|
||||||
|
usr = gh_client.get_user()
|
||||||
|
if namespace == usr.login:
|
||||||
|
repos = [repo_view(repo) for repo in usr.get_repos(type='owner', sort='updated')]
|
||||||
|
return BuildTriggerHandler.build_sources_response(repos)
|
||||||
|
|
||||||
|
try:
|
||||||
|
org = gh_client.get_organization(namespace)
|
||||||
|
if org is None:
|
||||||
|
return []
|
||||||
|
except GithubException:
|
||||||
|
return []
|
||||||
|
|
||||||
|
repos = [repo_view(repo) for repo in org.get_repos(type='member')]
|
||||||
|
return BuildTriggerHandler.build_sources_response(repos)
|
||||||
|
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def list_build_subdirs(self):
|
||||||
|
config = self.config
|
||||||
|
gh_client = self._get_client()
|
||||||
|
source = config['build_source']
|
||||||
|
|
||||||
|
try:
|
||||||
|
repo = gh_client.get_repo(source)
|
||||||
|
|
||||||
|
# Find the first matching branch.
|
||||||
|
repo_branches = self.list_field_values('branch_name') or []
|
||||||
|
branches = find_matching_branches(config, repo_branches)
|
||||||
|
branches = branches or [repo.default_branch or 'master']
|
||||||
|
default_commit = repo.get_branch(branches[0]).commit
|
||||||
|
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
||||||
|
|
||||||
|
return [elem.path for elem in commit_tree.tree
|
||||||
|
if (elem.type == u'blob' and self.filename_is_dockerfile(os.path.basename(elem.path)))]
|
||||||
|
except GithubException as ghe:
|
||||||
|
message = ghe.data.get('message', 'Unable to list contents of repository: %s' % source)
|
||||||
|
if message == 'Branch not found':
|
||||||
|
raise EmptyRepositoryException()
|
||||||
|
|
||||||
|
raise RepositoryReadException(message)
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def load_dockerfile_contents(self):
|
||||||
|
config = self.config
|
||||||
|
gh_client = self._get_client()
|
||||||
|
source = config['build_source']
|
||||||
|
|
||||||
|
try:
|
||||||
|
repo = gh_client.get_repo(source)
|
||||||
|
except GithubException as ghe:
|
||||||
|
message = ghe.data.get('message', 'Unable to list contents of repository: %s' % source)
|
||||||
|
raise RepositoryReadException(message)
|
||||||
|
|
||||||
|
path = self.get_dockerfile_path()
|
||||||
|
if not path:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
file_info = repo.get_contents(path)
|
||||||
|
# TypeError is needed because directory inputs cause a TypeError
|
||||||
|
except (GithubException, TypeError) as ghe:
|
||||||
|
logger.error("got error from trying to find github file %s" % ghe)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if file_info is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if isinstance(file_info, list):
|
||||||
|
return None
|
||||||
|
|
||||||
|
content = file_info.content
|
||||||
|
if file_info.encoding == 'base64':
|
||||||
|
content = base64.b64decode(content)
|
||||||
|
return content
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def list_field_values(self, field_name, limit=None):
|
||||||
|
if field_name == 'refs':
|
||||||
|
branches = self.list_field_values('branch_name')
|
||||||
|
tags = self.list_field_values('tag_name')
|
||||||
|
|
||||||
|
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||||
|
[{'kind': 'tag', 'name': tag} for tag in tags])
|
||||||
|
|
||||||
|
config = self.config
|
||||||
|
source = config.get('build_source')
|
||||||
|
if source is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if field_name == 'tag_name':
|
||||||
|
try:
|
||||||
|
gh_client = self._get_client()
|
||||||
|
repo = gh_client.get_repo(source)
|
||||||
|
gh_tags = repo.get_tags()
|
||||||
|
if limit:
|
||||||
|
gh_tags = repo.get_tags()[0:limit]
|
||||||
|
|
||||||
|
return [tag.name for tag in gh_tags]
|
||||||
|
except GitHubBadCredentialsException:
|
||||||
|
return []
|
||||||
|
except GithubException:
|
||||||
|
logger.exception("Got GitHub Exception when trying to list tags for trigger %s",
|
||||||
|
self.trigger.id)
|
||||||
|
return []
|
||||||
|
|
||||||
|
if field_name == 'branch_name':
|
||||||
|
try:
|
||||||
|
gh_client = self._get_client()
|
||||||
|
repo = gh_client.get_repo(source)
|
||||||
|
gh_branches = repo.get_branches()
|
||||||
|
if limit:
|
||||||
|
gh_branches = repo.get_branches()[0:limit]
|
||||||
|
|
||||||
|
branches = [branch.name for branch in gh_branches]
|
||||||
|
|
||||||
|
if not repo.default_branch in branches:
|
||||||
|
branches.insert(0, repo.default_branch)
|
||||||
|
|
||||||
|
if branches[0] != repo.default_branch:
|
||||||
|
branches.remove(repo.default_branch)
|
||||||
|
branches.insert(0, repo.default_branch)
|
||||||
|
|
||||||
|
return branches
|
||||||
|
except GitHubBadCredentialsException:
|
||||||
|
return ['master']
|
||||||
|
except GithubException:
|
||||||
|
logger.exception("Got GitHub Exception when trying to list branches for trigger %s",
|
||||||
|
self.trigger.id)
|
||||||
|
return ['master']
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _build_metadata_for_commit(cls, commit_sha, ref, repo):
|
||||||
|
try:
|
||||||
|
commit = repo.get_commit(commit_sha)
|
||||||
|
except GithubException:
|
||||||
|
logger.exception('Could not load commit information from GitHub')
|
||||||
|
return None
|
||||||
|
|
||||||
|
commit_info = {
|
||||||
|
'url': commit.html_url,
|
||||||
|
'message': commit.commit.message,
|
||||||
|
'date': commit.last_modified
|
||||||
|
}
|
||||||
|
|
||||||
|
if commit.author:
|
||||||
|
commit_info['author'] = {
|
||||||
|
'username': commit.author.login,
|
||||||
|
'avatar_url': commit.author.avatar_url,
|
||||||
|
'url': commit.author.html_url
|
||||||
|
}
|
||||||
|
|
||||||
|
if commit.committer:
|
||||||
|
commit_info['committer'] = {
|
||||||
|
'username': commit.committer.login,
|
||||||
|
'avatar_url': commit.committer.avatar_url,
|
||||||
|
'url': commit.committer.html_url
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'commit': commit_sha,
|
||||||
|
'ref': ref,
|
||||||
|
'default_branch': repo.default_branch,
|
||||||
|
'git_url': repo.ssh_url,
|
||||||
|
'commit_info': commit_info
|
||||||
|
}
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def manual_start(self, run_parameters=None):
|
||||||
|
config = self.config
|
||||||
|
source = config['build_source']
|
||||||
|
|
||||||
|
try:
|
||||||
|
gh_client = self._get_client()
|
||||||
|
repo = gh_client.get_repo(source)
|
||||||
|
default_branch = repo.default_branch
|
||||||
|
except GithubException as ghe:
|
||||||
|
msg = GithubBuildTrigger._get_error_message(ghe, 'Unable to start build trigger')
|
||||||
|
raise TriggerStartException(msg)
|
||||||
|
|
||||||
|
def get_branch_sha(branch_name):
|
||||||
|
try:
|
||||||
|
branch = repo.get_branch(branch_name)
|
||||||
|
return branch.commit.sha
|
||||||
|
except GithubException:
|
||||||
|
raise TriggerStartException('Could not find branch in repository')
|
||||||
|
|
||||||
|
def get_tag_sha(tag_name):
|
||||||
|
tags = {tag.name: tag for tag in repo.get_tags()}
|
||||||
|
if not tag_name in tags:
|
||||||
|
raise TriggerStartException('Could not find tag in repository')
|
||||||
|
|
||||||
|
return tags[tag_name].commit.sha
|
||||||
|
|
||||||
|
# Find the branch or tag to build.
|
||||||
|
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
|
||||||
|
default_branch)
|
||||||
|
|
||||||
|
metadata = GithubBuildTrigger._build_metadata_for_commit(commit_sha, ref, repo)
|
||||||
|
return self.prepare_build(metadata, is_manual=True)
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def lookup_user(self, username):
|
||||||
|
try:
|
||||||
|
gh_client = self._get_client()
|
||||||
|
user = gh_client.get_user(username)
|
||||||
|
return {
|
||||||
|
'html_url': user.html_url,
|
||||||
|
'avatar_url': user.avatar_url
|
||||||
|
}
|
||||||
|
except GithubException:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
|
def handle_trigger_request(self, request):
|
||||||
|
# Check the payload to see if we should skip it based on the lack of a head_commit.
|
||||||
|
payload = request.get_json()
|
||||||
|
if payload is None:
|
||||||
|
raise InvalidPayloadException('Missing payload')
|
||||||
|
|
||||||
|
# This is for GitHub's probing/testing.
|
||||||
|
if 'zen' in payload:
|
||||||
|
raise SkipRequestException()
|
||||||
|
|
||||||
|
# Lookup the default branch for the repository.
|
||||||
|
if 'repository' not in payload:
|
||||||
|
raise InvalidPayloadException("Missing 'repository' on request")
|
||||||
|
|
||||||
|
if 'owner' not in payload['repository']:
|
||||||
|
raise InvalidPayloadException("Missing 'owner' on repository")
|
||||||
|
|
||||||
|
if 'name' not in payload['repository']['owner']:
|
||||||
|
raise InvalidPayloadException("Missing owner 'name' on repository")
|
||||||
|
|
||||||
|
if 'name' not in payload['repository']:
|
||||||
|
raise InvalidPayloadException("Missing 'name' on repository")
|
||||||
|
|
||||||
|
default_branch = None
|
||||||
|
lookup_user = None
|
||||||
|
try:
|
||||||
|
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||||
|
payload['repository']['name'])
|
||||||
|
|
||||||
|
gh_client = self._get_client()
|
||||||
|
repo = gh_client.get_repo(repo_full_name)
|
||||||
|
default_branch = repo.default_branch
|
||||||
|
lookup_user = self.lookup_user
|
||||||
|
except GitHubBadCredentialsException:
|
||||||
|
logger.exception('Got GitHub Credentials Exception; Cannot lookup default branch')
|
||||||
|
except GithubException:
|
||||||
|
logger.exception("Got GitHub Exception when trying to start trigger %s", self.trigger.id)
|
||||||
|
raise SkipRequestException()
|
||||||
|
|
||||||
|
logger.debug('GitHub trigger payload %s', payload)
|
||||||
|
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
|
||||||
|
lookup_user=lookup_user)
|
||||||
|
prepared = self.prepare_build(metadata)
|
||||||
|
|
||||||
|
# Check if we should skip this build.
|
||||||
|
raise_if_skipped_build(prepared, self.config)
|
||||||
|
return prepared
|
||||||
621
buildtrigger/gitlabhandler.py
Normal file
621
buildtrigger/gitlabhandler.py
Normal file
|
|
@ -0,0 +1,621 @@
|
||||||
|
import os.path
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from calendar import timegm
|
||||||
|
from functools import wraps
|
||||||
|
|
||||||
|
import dateutil.parser
|
||||||
|
import gitlab
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from jsonschema import validate
|
||||||
|
|
||||||
|
from app import app, gitlab_trigger
|
||||||
|
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
||||||
|
TriggerDeactivationException, TriggerStartException,
|
||||||
|
SkipRequestException, InvalidPayloadException,
|
||||||
|
TriggerAuthException,
|
||||||
|
determine_build_ref, raise_if_skipped_build,
|
||||||
|
find_matching_branches)
|
||||||
|
from buildtrigger.basehandler import BuildTriggerHandler
|
||||||
|
from endpoints.exception import ExternalServiceError
|
||||||
|
from util.security.ssh import generate_ssh_keypair
|
||||||
|
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
GITLAB_WEBHOOK_PAYLOAD_SCHEMA = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'ref': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'checkout_sha': {
|
||||||
|
'type': ['string', 'null'],
|
||||||
|
},
|
||||||
|
'repository': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'git_ssh_url': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['git_ssh_url'],
|
||||||
|
},
|
||||||
|
'commits': {
|
||||||
|
'type': 'array',
|
||||||
|
'items': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'id': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'url': {
|
||||||
|
'type': ['string', 'null'],
|
||||||
|
},
|
||||||
|
'message': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'timestamp': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
'author': {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'email': {
|
||||||
|
'type': 'string',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['email'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['id', 'message', 'timestamp'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'required': ['ref', 'checkout_sha', 'repository'],
|
||||||
|
}
|
||||||
|
|
||||||
|
_ACCESS_LEVEL_MAP = {
|
||||||
|
50: ("owner", True),
|
||||||
|
40: ("master", True),
|
||||||
|
30: ("developer", False),
|
||||||
|
20: ("reporter", False),
|
||||||
|
10: ("guest", False),
|
||||||
|
}
|
||||||
|
|
||||||
|
_PER_PAGE_COUNT = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _catch_timeouts_and_errors(func):
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
except requests.exceptions.Timeout:
|
||||||
|
msg = 'Request to the GitLab API timed out'
|
||||||
|
logger.exception(msg)
|
||||||
|
raise ExternalServiceError(msg)
|
||||||
|
except gitlab.GitlabError:
|
||||||
|
msg = 'GitLab API error. Please contact support.'
|
||||||
|
logger.exception(msg)
|
||||||
|
raise ExternalServiceError(msg)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def _paginated_iterator(func, exc, **kwargs):
|
||||||
|
""" Returns an iterator over invocations of the given function, automatically handling
|
||||||
|
pagination.
|
||||||
|
"""
|
||||||
|
page = 1
|
||||||
|
while True:
|
||||||
|
result = func(page=page, per_page=_PER_PAGE_COUNT, **kwargs)
|
||||||
|
if result is None or result is False:
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
counter = 0
|
||||||
|
for item in result:
|
||||||
|
yield item
|
||||||
|
counter = counter + 1
|
||||||
|
|
||||||
|
if counter < _PER_PAGE_COUNT:
|
||||||
|
break
|
||||||
|
|
||||||
|
page = page + 1
|
||||||
|
|
||||||
|
|
||||||
|
def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None,
|
||||||
|
lookup_commit=None):
|
||||||
|
""" Returns the Gitlab webhook JSON payload transformed into our own payload
|
||||||
|
format. If the gl_payload is not valid, returns None.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
validate(gl_payload, GITLAB_WEBHOOK_PAYLOAD_SCHEMA)
|
||||||
|
except Exception as exc:
|
||||||
|
raise InvalidPayloadException(exc.message)
|
||||||
|
|
||||||
|
payload = JSONPathDict(gl_payload)
|
||||||
|
|
||||||
|
if payload['object_kind'] != 'push' and payload['object_kind'] != 'tag_push':
|
||||||
|
# Unknown kind of webhook.
|
||||||
|
raise SkipRequestException
|
||||||
|
|
||||||
|
# Check for empty commits. The commits list will be empty if the branch is deleted.
|
||||||
|
commits = payload['commits']
|
||||||
|
if payload['object_kind'] == 'push' and not commits:
|
||||||
|
raise SkipRequestException
|
||||||
|
|
||||||
|
# Check for missing commit information.
|
||||||
|
commit_sha = payload['checkout_sha'] or payload['after']
|
||||||
|
if commit_sha is None or commit_sha == '0000000000000000000000000000000000000000':
|
||||||
|
raise SkipRequestException
|
||||||
|
|
||||||
|
config = SafeDictSetter()
|
||||||
|
config['commit'] = commit_sha
|
||||||
|
config['ref'] = payload['ref']
|
||||||
|
config['default_branch'] = default_branch
|
||||||
|
config['git_url'] = payload['repository.git_ssh_url']
|
||||||
|
|
||||||
|
found_commit = JSONPathDict({})
|
||||||
|
if payload['object_kind'] == 'push' or payload['object_kind'] == 'tag_push':
|
||||||
|
# Find the commit associated with the checkout_sha. Gitlab doesn't (necessary) send this in
|
||||||
|
# any order, so we cannot simply index into the commits list.
|
||||||
|
found_commit = None
|
||||||
|
if commits is not None:
|
||||||
|
for commit in commits:
|
||||||
|
if commit['id'] == payload['checkout_sha']:
|
||||||
|
found_commit = JSONPathDict(commit)
|
||||||
|
break
|
||||||
|
|
||||||
|
if found_commit is None and lookup_commit:
|
||||||
|
checkout_sha = payload['checkout_sha'] or payload['after']
|
||||||
|
found_commit_info = lookup_commit(payload['project_id'], checkout_sha)
|
||||||
|
found_commit = JSONPathDict(dict(found_commit_info) if found_commit_info else {})
|
||||||
|
|
||||||
|
if found_commit is None:
|
||||||
|
raise SkipRequestException
|
||||||
|
|
||||||
|
config['commit_info.url'] = found_commit['url']
|
||||||
|
config['commit_info.message'] = found_commit['message']
|
||||||
|
config['commit_info.date'] = found_commit['timestamp']
|
||||||
|
|
||||||
|
# Note: Gitlab does not send full user information with the payload, so we have to
|
||||||
|
# (optionally) look it up.
|
||||||
|
author_email = found_commit['author.email'] or found_commit['author_email']
|
||||||
|
if lookup_user and author_email:
|
||||||
|
author_info = lookup_user(author_email)
|
||||||
|
if author_info:
|
||||||
|
config['commit_info.author.username'] = author_info['username']
|
||||||
|
config['commit_info.author.url'] = author_info['html_url']
|
||||||
|
config['commit_info.author.avatar_url'] = author_info['avatar_url']
|
||||||
|
|
||||||
|
return config.dict_value()
|
||||||
|
|
||||||
|
|
||||||
|
class GitLabBuildTrigger(BuildTriggerHandler):
|
||||||
|
"""
|
||||||
|
BuildTrigger for GitLab.
|
||||||
|
"""
|
||||||
|
@classmethod
|
||||||
|
def service_name(cls):
|
||||||
|
return 'gitlab'
|
||||||
|
|
||||||
|
def _get_authorized_client(self):
|
||||||
|
auth_token = self.auth_token or 'invalid'
|
||||||
|
api_version = self.config.get('API_VERSION', '4')
|
||||||
|
client = gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=20,
|
||||||
|
api_version=api_version)
|
||||||
|
try:
|
||||||
|
client.auth()
|
||||||
|
except gitlab.GitlabGetError as ex:
|
||||||
|
raise TriggerAuthException(ex.message)
|
||||||
|
|
||||||
|
return client
|
||||||
|
|
||||||
|
def is_active(self):
|
||||||
|
return 'hook_id' in self.config
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def activate(self, standard_webhook_url):
|
||||||
|
config = self.config
|
||||||
|
new_build_source = config['build_source']
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
|
||||||
|
# Find the GitLab repository.
|
||||||
|
gl_project = gl_client.projects.get(new_build_source)
|
||||||
|
if not gl_project:
|
||||||
|
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
# Add a deploy key to the repository.
|
||||||
|
public_key, private_key = generate_ssh_keypair()
|
||||||
|
config['credentials'] = [
|
||||||
|
{
|
||||||
|
'name': 'SSH Public Key',
|
||||||
|
'value': public_key,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
key = gl_project.keys.create({
|
||||||
|
'title': '%s Builder' % app.config['REGISTRY_TITLE'],
|
||||||
|
'key': public_key,
|
||||||
|
})
|
||||||
|
|
||||||
|
if not key:
|
||||||
|
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
config['key_id'] = key.get_id()
|
||||||
|
|
||||||
|
# Add the webhook to the GitLab repository.
|
||||||
|
hook = gl_project.hooks.create({
|
||||||
|
'url': standard_webhook_url,
|
||||||
|
'push': True,
|
||||||
|
'tag_push': True,
|
||||||
|
'push_events': True,
|
||||||
|
'tag_push_events': True,
|
||||||
|
})
|
||||||
|
if not hook:
|
||||||
|
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
||||||
|
raise TriggerActivationException(msg)
|
||||||
|
|
||||||
|
config['hook_id'] = hook.get_id()
|
||||||
|
self.config = config
|
||||||
|
return config, {'private_key': private_key}
|
||||||
|
|
||||||
|
def deactivate(self):
|
||||||
|
config = self.config
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
|
||||||
|
# Find the GitLab repository.
|
||||||
|
try:
|
||||||
|
gl_project = gl_client.projects.get(config['build_source'])
|
||||||
|
if not gl_project:
|
||||||
|
config.pop('key_id', None)
|
||||||
|
config.pop('hook_id', None)
|
||||||
|
self.config = config
|
||||||
|
return config
|
||||||
|
except gitlab.GitlabGetError as ex:
|
||||||
|
if ex.response_code != 404:
|
||||||
|
raise
|
||||||
|
|
||||||
|
# Remove the webhook.
|
||||||
|
try:
|
||||||
|
gl_project.hooks.delete(config['hook_id'])
|
||||||
|
except gitlab.GitlabDeleteError as ex:
|
||||||
|
if ex.response_code != 404:
|
||||||
|
raise
|
||||||
|
|
||||||
|
config.pop('hook_id', None)
|
||||||
|
|
||||||
|
# Remove the key
|
||||||
|
try:
|
||||||
|
gl_project.keys.delete(config['key_id'])
|
||||||
|
except gitlab.GitlabDeleteError as ex:
|
||||||
|
if ex.response_code != 404:
|
||||||
|
raise
|
||||||
|
|
||||||
|
config.pop('key_id', None)
|
||||||
|
|
||||||
|
self.config = config
|
||||||
|
return config
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def list_build_source_namespaces(self):
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
current_user = gl_client.user
|
||||||
|
if not current_user:
|
||||||
|
raise RepositoryReadException('Unable to get current user')
|
||||||
|
|
||||||
|
namespaces = {}
|
||||||
|
for namespace in _paginated_iterator(gl_client.namespaces.list, RepositoryReadException):
|
||||||
|
namespace_id = namespace.get_id()
|
||||||
|
if namespace_id in namespaces:
|
||||||
|
namespaces[namespace_id]['score'] = namespaces[namespace_id]['score'] + 1
|
||||||
|
else:
|
||||||
|
owner = namespace.attributes['name']
|
||||||
|
namespaces[namespace_id] = {
|
||||||
|
'personal': namespace.attributes['kind'] == 'user',
|
||||||
|
'id': str(namespace_id),
|
||||||
|
'title': namespace.attributes['name'],
|
||||||
|
'avatar_url': namespace.attributes.get('avatar_url'),
|
||||||
|
'score': 1,
|
||||||
|
'url': namespace.attributes.get('web_url') or '',
|
||||||
|
}
|
||||||
|
|
||||||
|
return BuildTriggerHandler.build_namespaces_response(namespaces)
|
||||||
|
|
||||||
|
def _get_namespace(self, gl_client, gl_namespace, lazy=False):
|
||||||
|
try:
|
||||||
|
if gl_namespace.attributes['kind'] == 'group':
|
||||||
|
return gl_client.groups.get(gl_namespace.attributes['id'], lazy=lazy)
|
||||||
|
|
||||||
|
if gl_namespace.attributes['kind'] == 'user':
|
||||||
|
return gl_client.users.get(gl_client.user.attributes['id'], lazy=lazy)
|
||||||
|
|
||||||
|
# Note: This doesn't seem to work for IDs retrieved via the namespaces API; the IDs are
|
||||||
|
# different.
|
||||||
|
return gl_client.users.get(gl_namespace.attributes['id'], lazy=lazy)
|
||||||
|
except gitlab.GitlabGetError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def list_build_sources_for_namespace(self, namespace_id):
|
||||||
|
if not namespace_id:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def repo_view(repo):
|
||||||
|
# Because *anything* can be None in GitLab API!
|
||||||
|
permissions = repo.attributes.get('permissions') or {}
|
||||||
|
group_access = permissions.get('group_access') or {}
|
||||||
|
project_access = permissions.get('project_access') or {}
|
||||||
|
|
||||||
|
missing_group_access = permissions.get('group_access') is None
|
||||||
|
missing_project_access = permissions.get('project_access') is None
|
||||||
|
|
||||||
|
access_level = max(group_access.get('access_level') or 0,
|
||||||
|
project_access.get('access_level') or 0)
|
||||||
|
|
||||||
|
has_admin_permission = _ACCESS_LEVEL_MAP.get(access_level, ("", False))[1]
|
||||||
|
if missing_group_access or missing_project_access:
|
||||||
|
# Default to has permission if we cannot check the permissions. This will allow our users
|
||||||
|
# to select the repository and then GitLab's own checks will ensure that the webhook is
|
||||||
|
# added only if allowed.
|
||||||
|
# TODO: Do we want to display this differently in the UI?
|
||||||
|
has_admin_permission = True
|
||||||
|
|
||||||
|
view = {
|
||||||
|
'name': repo.attributes['path'],
|
||||||
|
'full_name': repo.attributes['path_with_namespace'],
|
||||||
|
'description': repo.attributes.get('description') or '',
|
||||||
|
'url': repo.attributes.get('web_url'),
|
||||||
|
'has_admin_permissions': has_admin_permission,
|
||||||
|
'private': repo.attributes.get('visibility') == 'private',
|
||||||
|
}
|
||||||
|
|
||||||
|
if repo.attributes.get('last_activity_at'):
|
||||||
|
try:
|
||||||
|
last_modified = dateutil.parser.parse(repo.attributes['last_activity_at'])
|
||||||
|
view['last_updated'] = timegm(last_modified.utctimetuple())
|
||||||
|
except ValueError:
|
||||||
|
logger.exception('Gitlab gave us an invalid last_activity_at: %s', last_modified)
|
||||||
|
|
||||||
|
return view
|
||||||
|
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
|
||||||
|
try:
|
||||||
|
gl_namespace = gl_client.namespaces.get(namespace_id)
|
||||||
|
except gitlab.GitlabGetError:
|
||||||
|
return []
|
||||||
|
|
||||||
|
namespace_obj = self._get_namespace(gl_client, gl_namespace, lazy=True)
|
||||||
|
repositories = _paginated_iterator(namespace_obj.projects.list, RepositoryReadException)
|
||||||
|
|
||||||
|
try:
|
||||||
|
return BuildTriggerHandler.build_sources_response([repo_view(repo) for repo in repositories])
|
||||||
|
except gitlab.GitlabGetError:
|
||||||
|
return []
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def list_build_subdirs(self):
|
||||||
|
config = self.config
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
new_build_source = config['build_source']
|
||||||
|
|
||||||
|
gl_project = gl_client.projects.get(new_build_source)
|
||||||
|
if not gl_project:
|
||||||
|
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
||||||
|
raise RepositoryReadException(msg)
|
||||||
|
|
||||||
|
repo_branches = gl_project.branches.list()
|
||||||
|
if not repo_branches:
|
||||||
|
msg = 'Unable to find GitLab branches for source: %s' % new_build_source
|
||||||
|
raise RepositoryReadException(msg)
|
||||||
|
|
||||||
|
branches = [branch.attributes['name'] for branch in repo_branches]
|
||||||
|
branches = find_matching_branches(config, branches)
|
||||||
|
branches = branches or [gl_project.attributes['default_branch'] or 'master']
|
||||||
|
|
||||||
|
repo_tree = gl_project.repository_tree(ref=branches[0])
|
||||||
|
if not repo_tree:
|
||||||
|
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
|
||||||
|
raise RepositoryReadException(msg)
|
||||||
|
|
||||||
|
return [node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def load_dockerfile_contents(self):
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
path = self.get_dockerfile_path()
|
||||||
|
|
||||||
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||||
|
if not gl_project:
|
||||||
|
return None
|
||||||
|
|
||||||
|
branches = self.list_field_values('branch_name')
|
||||||
|
branches = find_matching_branches(self.config, branches)
|
||||||
|
if branches == []:
|
||||||
|
return None
|
||||||
|
|
||||||
|
branch_name = branches[0]
|
||||||
|
if gl_project.attributes['default_branch'] in branches:
|
||||||
|
branch_name = gl_project.attributes['default_branch']
|
||||||
|
|
||||||
|
try:
|
||||||
|
return gl_project.files.get(path, branch_name).decode()
|
||||||
|
except gitlab.GitlabGetError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def list_field_values(self, field_name, limit=None):
|
||||||
|
if field_name == 'refs':
|
||||||
|
branches = self.list_field_values('branch_name')
|
||||||
|
tags = self.list_field_values('tag_name')
|
||||||
|
|
||||||
|
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||||
|
[{'kind': 'tag', 'name': t} for t in tags])
|
||||||
|
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||||
|
if not gl_project:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if field_name == 'tag_name':
|
||||||
|
tags = gl_project.tags.list()
|
||||||
|
if not tags:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if limit:
|
||||||
|
tags = tags[0:limit]
|
||||||
|
|
||||||
|
return [tag.attributes['name'] for tag in tags]
|
||||||
|
|
||||||
|
if field_name == 'branch_name':
|
||||||
|
branches = gl_project.branches.list()
|
||||||
|
if not branches:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if limit:
|
||||||
|
branches = branches[0:limit]
|
||||||
|
|
||||||
|
return [branch.attributes['name'] for branch in branches]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_repository_url(self):
|
||||||
|
return gitlab_trigger.get_public_url(self.config['build_source'])
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def lookup_commit(self, repo_id, commit_sha):
|
||||||
|
if repo_id is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
gl_project = gl_client.projects.get(self.config['build_source'], lazy=True)
|
||||||
|
commit = gl_project.commits.get(commit_sha)
|
||||||
|
if not commit:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return commit
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def lookup_user(self, email):
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
try:
|
||||||
|
result = gl_client.users.list(search=email)
|
||||||
|
if not result:
|
||||||
|
return None
|
||||||
|
|
||||||
|
[user] = result
|
||||||
|
return {
|
||||||
|
'username': user.attributes['username'],
|
||||||
|
'html_url': user.attributes['web_url'],
|
||||||
|
'avatar_url': user.attributes['avatar_url']
|
||||||
|
}
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def get_metadata_for_commit(self, commit_sha, ref, repo):
|
||||||
|
commit = self.lookup_commit(repo.get_id(), commit_sha)
|
||||||
|
if commit is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
metadata = {
|
||||||
|
'commit': commit.attributes['id'],
|
||||||
|
'ref': ref,
|
||||||
|
'default_branch': repo.attributes['default_branch'],
|
||||||
|
'git_url': repo.attributes['ssh_url_to_repo'],
|
||||||
|
'commit_info': {
|
||||||
|
'url': os.path.join(repo.attributes['web_url'], 'commit', commit.attributes['id']),
|
||||||
|
'message': commit.attributes['message'],
|
||||||
|
'date': commit.attributes['committed_date'],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
committer = None
|
||||||
|
if 'committer_email' in commit.attributes:
|
||||||
|
committer = self.lookup_user(commit.attributes['committer_email'])
|
||||||
|
|
||||||
|
author = None
|
||||||
|
if 'author_email' in commit.attributes:
|
||||||
|
author = self.lookup_user(commit.attributes['author_email'])
|
||||||
|
|
||||||
|
if committer is not None:
|
||||||
|
metadata['commit_info']['committer'] = {
|
||||||
|
'username': committer['username'],
|
||||||
|
'avatar_url': committer['avatar_url'],
|
||||||
|
'url': committer.get('http_url', ''),
|
||||||
|
}
|
||||||
|
|
||||||
|
if author is not None:
|
||||||
|
metadata['commit_info']['author'] = {
|
||||||
|
'username': author['username'],
|
||||||
|
'avatar_url': author['avatar_url'],
|
||||||
|
'url': author.get('http_url', ''),
|
||||||
|
}
|
||||||
|
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def manual_start(self, run_parameters=None):
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||||
|
if not gl_project:
|
||||||
|
raise TriggerStartException('Could not find repository')
|
||||||
|
|
||||||
|
def get_tag_sha(tag_name):
|
||||||
|
try:
|
||||||
|
tag = gl_project.tags.get(tag_name)
|
||||||
|
except gitlab.GitlabGetError:
|
||||||
|
raise TriggerStartException('Could not find tag in repository')
|
||||||
|
|
||||||
|
return tag.attributes['commit']['id']
|
||||||
|
|
||||||
|
def get_branch_sha(branch_name):
|
||||||
|
try:
|
||||||
|
branch = gl_project.branches.get(branch_name)
|
||||||
|
except gitlab.GitlabGetError:
|
||||||
|
raise TriggerStartException('Could not find branch in repository')
|
||||||
|
|
||||||
|
return branch.attributes['commit']['id']
|
||||||
|
|
||||||
|
# Find the branch or tag to build.
|
||||||
|
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
|
||||||
|
gl_project.attributes['default_branch'])
|
||||||
|
|
||||||
|
metadata = self.get_metadata_for_commit(commit_sha, ref, gl_project)
|
||||||
|
return self.prepare_build(metadata, is_manual=True)
|
||||||
|
|
||||||
|
@_catch_timeouts_and_errors
|
||||||
|
def handle_trigger_request(self, request):
|
||||||
|
payload = request.get_json()
|
||||||
|
if not payload:
|
||||||
|
raise InvalidPayloadException()
|
||||||
|
|
||||||
|
logger.debug('GitLab trigger payload %s', payload)
|
||||||
|
|
||||||
|
# Lookup the default branch.
|
||||||
|
gl_client = self._get_authorized_client()
|
||||||
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||||
|
if not gl_project:
|
||||||
|
logger.debug('Skipping GitLab build; project %s not found', self.config['build_source'])
|
||||||
|
raise InvalidPayloadException()
|
||||||
|
|
||||||
|
def lookup_commit(repo_id, commit_sha):
|
||||||
|
commit = self.lookup_commit(repo_id, commit_sha)
|
||||||
|
if commit is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return dict(commit.attributes)
|
||||||
|
|
||||||
|
default_branch = gl_project.attributes['default_branch']
|
||||||
|
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
|
||||||
|
lookup_user=self.lookup_user,
|
||||||
|
lookup_commit=lookup_commit)
|
||||||
|
prepared = self.prepare_build(metadata)
|
||||||
|
|
||||||
|
# Check if we should skip this build.
|
||||||
|
raise_if_skipped_build(prepared, self.config)
|
||||||
|
return prepared
|
||||||
0
buildtrigger/test/__init__.py
Normal file
0
buildtrigger/test/__init__.py
Normal file
159
buildtrigger/test/bitbucketmock.py
Normal file
159
buildtrigger/test/bitbucketmock.py
Normal file
|
|
@ -0,0 +1,159 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from mock import Mock
|
||||||
|
|
||||||
|
from buildtrigger.bitbuckethandler import BitbucketBuildTrigger
|
||||||
|
from util.morecollections import AttrDict
|
||||||
|
|
||||||
|
def get_bitbucket_trigger(dockerfile_path=''):
|
||||||
|
trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger'))
|
||||||
|
trigger = BitbucketBuildTrigger(trigger_obj, {
|
||||||
|
'build_source': 'foo/bar',
|
||||||
|
'dockerfile_path': dockerfile_path,
|
||||||
|
'nickname': 'knownuser',
|
||||||
|
'account_id': 'foo',
|
||||||
|
})
|
||||||
|
|
||||||
|
trigger._get_client = get_mock_bitbucket
|
||||||
|
return trigger
|
||||||
|
|
||||||
|
def get_repo_path_contents(path, revision):
|
||||||
|
data = {
|
||||||
|
'files': [{'path': 'Dockerfile'}],
|
||||||
|
}
|
||||||
|
|
||||||
|
return (True, data, None)
|
||||||
|
|
||||||
|
def get_raw_path_contents(path, revision):
|
||||||
|
if path == 'Dockerfile':
|
||||||
|
return (True, 'hello world', None)
|
||||||
|
|
||||||
|
if path == 'somesubdir/Dockerfile':
|
||||||
|
return (True, 'hi universe', None)
|
||||||
|
|
||||||
|
return (False, None, None)
|
||||||
|
|
||||||
|
def get_branches_and_tags():
|
||||||
|
data = {
|
||||||
|
'branches': [{'name': 'master'}, {'name': 'otherbranch'}],
|
||||||
|
'tags': [{'name': 'sometag'}, {'name': 'someothertag'}],
|
||||||
|
}
|
||||||
|
return (True, data, None)
|
||||||
|
|
||||||
|
def get_branches():
|
||||||
|
return (True, {'master': {}, 'otherbranch': {}}, None)
|
||||||
|
|
||||||
|
def get_tags():
|
||||||
|
return (True, {'sometag': {}, 'someothertag': {}}, None)
|
||||||
|
|
||||||
|
def get_branch(branch_name):
|
||||||
|
if branch_name != 'master':
|
||||||
|
return (False, None, None)
|
||||||
|
|
||||||
|
data = {
|
||||||
|
'target': {
|
||||||
|
'hash': 'aaaaaaa',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return (True, data, None)
|
||||||
|
|
||||||
|
def get_tag(tag_name):
|
||||||
|
if tag_name != 'sometag':
|
||||||
|
return (False, None, None)
|
||||||
|
|
||||||
|
data = {
|
||||||
|
'target': {
|
||||||
|
'hash': 'aaaaaaa',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return (True, data, None)
|
||||||
|
|
||||||
|
def get_changeset_mock(commit_sha):
|
||||||
|
if commit_sha != 'aaaaaaa':
|
||||||
|
return (False, None, 'Not found')
|
||||||
|
|
||||||
|
data = {
|
||||||
|
'node': 'aaaaaaa',
|
||||||
|
'message': 'some message',
|
||||||
|
'timestamp': 'now',
|
||||||
|
'raw_author': 'foo@bar.com',
|
||||||
|
}
|
||||||
|
|
||||||
|
return (True, data, None)
|
||||||
|
|
||||||
|
def get_changesets():
|
||||||
|
changesets_mock = Mock()
|
||||||
|
changesets_mock.get = Mock(side_effect=get_changeset_mock)
|
||||||
|
return changesets_mock
|
||||||
|
|
||||||
|
def get_deploykeys():
|
||||||
|
deploykeys_mock = Mock()
|
||||||
|
deploykeys_mock.create = Mock(return_value=(True, {'pk': 'someprivatekey'}, None))
|
||||||
|
deploykeys_mock.delete = Mock(return_value=(True, {}, None))
|
||||||
|
return deploykeys_mock
|
||||||
|
|
||||||
|
def get_webhooks():
|
||||||
|
webhooks_mock = Mock()
|
||||||
|
webhooks_mock.create = Mock(return_value=(True, {'uuid': 'someuuid'}, None))
|
||||||
|
webhooks_mock.delete = Mock(return_value=(True, {}, None))
|
||||||
|
return webhooks_mock
|
||||||
|
|
||||||
|
def get_repo_mock(name):
|
||||||
|
if name != 'bar':
|
||||||
|
return None
|
||||||
|
|
||||||
|
repo_mock = Mock()
|
||||||
|
repo_mock.get_main_branch = Mock(return_value=(True, {'name': 'master'}, None))
|
||||||
|
repo_mock.get_path_contents = Mock(side_effect=get_repo_path_contents)
|
||||||
|
repo_mock.get_raw_path_contents = Mock(side_effect=get_raw_path_contents)
|
||||||
|
repo_mock.get_branches_and_tags = Mock(side_effect=get_branches_and_tags)
|
||||||
|
repo_mock.get_branches = Mock(side_effect=get_branches)
|
||||||
|
repo_mock.get_tags = Mock(side_effect=get_tags)
|
||||||
|
repo_mock.get_branch = Mock(side_effect=get_branch)
|
||||||
|
repo_mock.get_tag = Mock(side_effect=get_tag)
|
||||||
|
|
||||||
|
repo_mock.changesets = Mock(side_effect=get_changesets)
|
||||||
|
repo_mock.deploykeys = Mock(side_effect=get_deploykeys)
|
||||||
|
repo_mock.webhooks = Mock(side_effect=get_webhooks)
|
||||||
|
return repo_mock
|
||||||
|
|
||||||
|
def get_repositories_mock():
|
||||||
|
repos_mock = Mock()
|
||||||
|
repos_mock.get = Mock(side_effect=get_repo_mock)
|
||||||
|
return repos_mock
|
||||||
|
|
||||||
|
def get_namespace_mock(namespace):
|
||||||
|
namespace_mock = Mock()
|
||||||
|
namespace_mock.repositories = Mock(side_effect=get_repositories_mock)
|
||||||
|
return namespace_mock
|
||||||
|
|
||||||
|
def get_repo(namespace, name):
|
||||||
|
return {
|
||||||
|
'owner': namespace,
|
||||||
|
'logo': 'avatarurl',
|
||||||
|
'slug': name,
|
||||||
|
'description': 'some %s repo' % (name),
|
||||||
|
'utc_last_updated': str(datetime.utcfromtimestamp(0)),
|
||||||
|
'read_only': namespace != 'knownuser',
|
||||||
|
'is_private': name == 'somerepo',
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_visible_repos():
|
||||||
|
repos = [
|
||||||
|
get_repo('knownuser', 'somerepo'),
|
||||||
|
get_repo('someorg', 'somerepo'),
|
||||||
|
get_repo('someorg', 'anotherrepo'),
|
||||||
|
]
|
||||||
|
return (True, repos, None)
|
||||||
|
|
||||||
|
def get_authed_mock(token, secret):
|
||||||
|
authed_mock = Mock()
|
||||||
|
authed_mock.for_namespace = Mock(side_effect=get_namespace_mock)
|
||||||
|
authed_mock.get_visible_repositories = Mock(side_effect=get_visible_repos)
|
||||||
|
return authed_mock
|
||||||
|
|
||||||
|
def get_mock_bitbucket():
|
||||||
|
bitbucket_mock = Mock()
|
||||||
|
bitbucket_mock.get_authorized_client = Mock(side_effect=get_authed_mock)
|
||||||
|
return bitbucket_mock
|
||||||
178
buildtrigger/test/githubmock.py
Normal file
178
buildtrigger/test/githubmock.py
Normal file
|
|
@ -0,0 +1,178 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from mock import Mock
|
||||||
|
|
||||||
|
from github import GithubException
|
||||||
|
|
||||||
|
from buildtrigger.githubhandler import GithubBuildTrigger
|
||||||
|
from util.morecollections import AttrDict
|
||||||
|
|
||||||
|
def get_github_trigger(dockerfile_path=''):
|
||||||
|
trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger'))
|
||||||
|
trigger = GithubBuildTrigger(trigger_obj, {'build_source': 'foo', 'dockerfile_path': dockerfile_path})
|
||||||
|
trigger._get_client = get_mock_github
|
||||||
|
return trigger
|
||||||
|
|
||||||
|
def get_mock_github():
|
||||||
|
def get_commit_mock(commit_sha):
|
||||||
|
if commit_sha == 'aaaaaaa':
|
||||||
|
commit_mock = Mock()
|
||||||
|
commit_mock.sha = commit_sha
|
||||||
|
commit_mock.html_url = 'http://url/to/commit'
|
||||||
|
commit_mock.last_modified = 'now'
|
||||||
|
|
||||||
|
commit_mock.commit = Mock()
|
||||||
|
commit_mock.commit.message = 'some cool message'
|
||||||
|
|
||||||
|
commit_mock.committer = Mock()
|
||||||
|
commit_mock.committer.login = 'someuser'
|
||||||
|
commit_mock.committer.avatar_url = 'avatarurl'
|
||||||
|
commit_mock.committer.html_url = 'htmlurl'
|
||||||
|
|
||||||
|
commit_mock.author = Mock()
|
||||||
|
commit_mock.author.login = 'someuser'
|
||||||
|
commit_mock.author.avatar_url = 'avatarurl'
|
||||||
|
commit_mock.author.html_url = 'htmlurl'
|
||||||
|
return commit_mock
|
||||||
|
|
||||||
|
raise GithubException(None, None)
|
||||||
|
|
||||||
|
def get_branch_mock(branch_name):
|
||||||
|
if branch_name == 'master':
|
||||||
|
branch_mock = Mock()
|
||||||
|
branch_mock.commit = Mock()
|
||||||
|
branch_mock.commit.sha = 'aaaaaaa'
|
||||||
|
return branch_mock
|
||||||
|
|
||||||
|
raise GithubException(None, None)
|
||||||
|
|
||||||
|
def get_repo_mock(namespace, name):
|
||||||
|
repo_mock = Mock()
|
||||||
|
repo_mock.owner = Mock()
|
||||||
|
repo_mock.owner.login = namespace
|
||||||
|
|
||||||
|
repo_mock.full_name = '%s/%s' % (namespace, name)
|
||||||
|
repo_mock.name = name
|
||||||
|
repo_mock.description = 'some %s repo' % (name)
|
||||||
|
|
||||||
|
if name != 'anotherrepo':
|
||||||
|
repo_mock.pushed_at = datetime.utcfromtimestamp(0)
|
||||||
|
else:
|
||||||
|
repo_mock.pushed_at = None
|
||||||
|
|
||||||
|
repo_mock.html_url = 'https://bitbucket.org/%s/%s' % (namespace, name)
|
||||||
|
repo_mock.private = name == 'somerepo'
|
||||||
|
repo_mock.permissions = Mock()
|
||||||
|
repo_mock.permissions.admin = namespace == 'knownuser'
|
||||||
|
return repo_mock
|
||||||
|
|
||||||
|
def get_user_repos_mock(type='all', sort='created'):
|
||||||
|
return [get_repo_mock('knownuser', 'somerepo')]
|
||||||
|
|
||||||
|
def get_org_repos_mock(type='all'):
|
||||||
|
return [get_repo_mock('someorg', 'somerepo'), get_repo_mock('someorg', 'anotherrepo')]
|
||||||
|
|
||||||
|
def get_orgs_mock():
|
||||||
|
return [get_org_mock('someorg')]
|
||||||
|
|
||||||
|
def get_user_mock(username='knownuser'):
|
||||||
|
if username == 'knownuser':
|
||||||
|
user_mock = Mock()
|
||||||
|
user_mock.name = username
|
||||||
|
user_mock.plan = Mock()
|
||||||
|
user_mock.plan.private_repos = 1
|
||||||
|
user_mock.login = username
|
||||||
|
user_mock.html_url = 'https://bitbucket.org/%s' % (username)
|
||||||
|
user_mock.avatar_url = 'avatarurl'
|
||||||
|
user_mock.get_repos = Mock(side_effect=get_user_repos_mock)
|
||||||
|
user_mock.get_orgs = Mock(side_effect=get_orgs_mock)
|
||||||
|
return user_mock
|
||||||
|
|
||||||
|
raise GithubException(None, None)
|
||||||
|
|
||||||
|
def get_org_mock(namespace):
|
||||||
|
if namespace == 'someorg':
|
||||||
|
org_mock = Mock()
|
||||||
|
org_mock.get_repos = Mock(side_effect=get_org_repos_mock)
|
||||||
|
org_mock.login = namespace
|
||||||
|
org_mock.html_url = 'https://bitbucket.org/%s' % (namespace)
|
||||||
|
org_mock.avatar_url = 'avatarurl'
|
||||||
|
org_mock.name = namespace
|
||||||
|
org_mock.plan = Mock()
|
||||||
|
org_mock.plan.private_repos = 2
|
||||||
|
return org_mock
|
||||||
|
|
||||||
|
raise GithubException(None, None)
|
||||||
|
|
||||||
|
def get_tags_mock():
|
||||||
|
sometag = Mock()
|
||||||
|
sometag.name = 'sometag'
|
||||||
|
sometag.commit = get_commit_mock('aaaaaaa')
|
||||||
|
|
||||||
|
someothertag = Mock()
|
||||||
|
someothertag.name = 'someothertag'
|
||||||
|
someothertag.commit = get_commit_mock('aaaaaaa')
|
||||||
|
return [sometag, someothertag]
|
||||||
|
|
||||||
|
def get_branches_mock():
|
||||||
|
master = Mock()
|
||||||
|
master.name = 'master'
|
||||||
|
master.commit = get_commit_mock('aaaaaaa')
|
||||||
|
|
||||||
|
otherbranch = Mock()
|
||||||
|
otherbranch.name = 'otherbranch'
|
||||||
|
otherbranch.commit = get_commit_mock('aaaaaaa')
|
||||||
|
return [master, otherbranch]
|
||||||
|
|
||||||
|
def get_contents_mock(filepath):
|
||||||
|
if filepath == 'Dockerfile':
|
||||||
|
m = Mock()
|
||||||
|
m.content = 'hello world'
|
||||||
|
return m
|
||||||
|
|
||||||
|
if filepath == 'somesubdir/Dockerfile':
|
||||||
|
m = Mock()
|
||||||
|
m.content = 'hi universe'
|
||||||
|
return m
|
||||||
|
|
||||||
|
raise GithubException(None, None)
|
||||||
|
|
||||||
|
def get_git_tree_mock(commit_sha, recursive=False):
|
||||||
|
first_file = Mock()
|
||||||
|
first_file.type = 'blob'
|
||||||
|
first_file.path = 'Dockerfile'
|
||||||
|
|
||||||
|
second_file = Mock()
|
||||||
|
second_file.type = 'other'
|
||||||
|
second_file.path = '/some/Dockerfile'
|
||||||
|
|
||||||
|
third_file = Mock()
|
||||||
|
third_file.type = 'blob'
|
||||||
|
third_file.path = 'somesubdir/Dockerfile'
|
||||||
|
|
||||||
|
t = Mock()
|
||||||
|
|
||||||
|
if commit_sha == 'aaaaaaa':
|
||||||
|
t.tree = [
|
||||||
|
first_file, second_file, third_file,
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
t.tree = []
|
||||||
|
|
||||||
|
return t
|
||||||
|
|
||||||
|
repo_mock = Mock()
|
||||||
|
repo_mock.default_branch = 'master'
|
||||||
|
repo_mock.ssh_url = 'ssh_url'
|
||||||
|
|
||||||
|
repo_mock.get_branch = Mock(side_effect=get_branch_mock)
|
||||||
|
repo_mock.get_tags = Mock(side_effect=get_tags_mock)
|
||||||
|
repo_mock.get_branches = Mock(side_effect=get_branches_mock)
|
||||||
|
repo_mock.get_commit = Mock(side_effect=get_commit_mock)
|
||||||
|
repo_mock.get_contents = Mock(side_effect=get_contents_mock)
|
||||||
|
repo_mock.get_git_tree = Mock(side_effect=get_git_tree_mock)
|
||||||
|
|
||||||
|
gh_mock = Mock()
|
||||||
|
gh_mock.get_repo = Mock(return_value=repo_mock)
|
||||||
|
gh_mock.get_user = Mock(side_effect=get_user_mock)
|
||||||
|
gh_mock.get_organization = Mock(side_effect=get_org_mock)
|
||||||
|
return gh_mock
|
||||||
598
buildtrigger/test/gitlabmock.py
Normal file
598
buildtrigger/test/gitlabmock.py
Normal file
|
|
@ -0,0 +1,598 @@
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
|
import gitlab
|
||||||
|
|
||||||
|
from httmock import urlmatch, HTTMock
|
||||||
|
|
||||||
|
from buildtrigger.gitlabhandler import GitLabBuildTrigger
|
||||||
|
from util.morecollections import AttrDict
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab')
|
||||||
|
def catchall_handler(url, request):
|
||||||
|
return {'status_code': 404}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/users$')
|
||||||
|
def users_handler(url, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
if url.query.find('knownuser') < 0:
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([]),
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"username": "knownuser",
|
||||||
|
"name": "Known User",
|
||||||
|
"state": "active",
|
||||||
|
"avatar_url": "avatarurl",
|
||||||
|
"web_url": "https://bitbucket.org/knownuser",
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/user$')
|
||||||
|
def user_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": 1,
|
||||||
|
"username": "john_smith",
|
||||||
|
"email": "john@example.com",
|
||||||
|
"name": "John Smith",
|
||||||
|
"state": "active",
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/foo%2Fbar$')
|
||||||
|
def project_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": 4,
|
||||||
|
"description": None,
|
||||||
|
"default_branch": "master",
|
||||||
|
"visibility": "private",
|
||||||
|
"path_with_namespace": "someorg/somerepo",
|
||||||
|
"ssh_url_to_repo": "git@example.com:someorg/somerepo.git",
|
||||||
|
"web_url": "http://example.com/someorg/somerepo",
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tree$')
|
||||||
|
def project_tree_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([
|
||||||
|
{
|
||||||
|
"id": "a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
|
||||||
|
"name": "Dockerfile",
|
||||||
|
"type": "tree",
|
||||||
|
"path": "files/Dockerfile",
|
||||||
|
"mode": "040000",
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tags$')
|
||||||
|
def project_tags_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([
|
||||||
|
{
|
||||||
|
'name': 'sometag',
|
||||||
|
'commit': {
|
||||||
|
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'someothertag',
|
||||||
|
'commit': {
|
||||||
|
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/branches$')
|
||||||
|
def project_branches_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([
|
||||||
|
{
|
||||||
|
'name': 'master',
|
||||||
|
'commit': {
|
||||||
|
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'otherbranch',
|
||||||
|
'commit': {
|
||||||
|
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/branches/master$')
|
||||||
|
def project_branch_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"name": "master",
|
||||||
|
"merged": True,
|
||||||
|
"protected": True,
|
||||||
|
"developers_can_push": False,
|
||||||
|
"developers_can_merge": False,
|
||||||
|
"commit": {
|
||||||
|
"author_email": "john@example.com",
|
||||||
|
"author_name": "John Smith",
|
||||||
|
"authored_date": "2012-06-27T05:51:39-07:00",
|
||||||
|
"committed_date": "2012-06-28T03:44:20-07:00",
|
||||||
|
"committer_email": "john@example.com",
|
||||||
|
"committer_name": "John Smith",
|
||||||
|
"id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||||
|
"short_id": "7b5c3cc",
|
||||||
|
"title": "add projects API",
|
||||||
|
"message": "add projects API",
|
||||||
|
"parent_ids": [
|
||||||
|
"4ad91d3c1144c406e50c7b33bae684bd6837faf8",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces/someorg$')
|
||||||
|
def namespace_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": 2,
|
||||||
|
"name": "someorg",
|
||||||
|
"path": "someorg",
|
||||||
|
"kind": "group",
|
||||||
|
"full_path": "someorg",
|
||||||
|
"parent_id": None,
|
||||||
|
"members_count_with_descendants": 2
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces/knownuser$')
|
||||||
|
def user_namespace_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": 1,
|
||||||
|
"name": "knownuser",
|
||||||
|
"path": "knownuser",
|
||||||
|
"kind": "user",
|
||||||
|
"full_path": "knownuser",
|
||||||
|
"parent_id": None,
|
||||||
|
"members_count_with_descendants": 2
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces(/)?$')
|
||||||
|
def namespaces_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([{
|
||||||
|
"id": 2,
|
||||||
|
"name": "someorg",
|
||||||
|
"path": "someorg",
|
||||||
|
"kind": "group",
|
||||||
|
"full_path": "someorg",
|
||||||
|
"parent_id": None,
|
||||||
|
"web_url": "http://gitlab.com/groups/someorg",
|
||||||
|
"members_count_with_descendants": 2
|
||||||
|
}]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_projects_handler(add_permissions_block):
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/groups/2/projects$')
|
||||||
|
def projects_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
permissions_block = {
|
||||||
|
"project_access": {
|
||||||
|
"access_level": 10,
|
||||||
|
"notification_level": 3
|
||||||
|
},
|
||||||
|
"group_access": {
|
||||||
|
"access_level": 20,
|
||||||
|
"notification_level": 3
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([{
|
||||||
|
"id": 4,
|
||||||
|
"name": "Some project",
|
||||||
|
"description": None,
|
||||||
|
"default_branch": "master",
|
||||||
|
"visibility": "private",
|
||||||
|
"path": "someproject",
|
||||||
|
"path_with_namespace": "someorg/someproject",
|
||||||
|
"last_activity_at": "2013-09-30T13:46:02Z",
|
||||||
|
"web_url": "http://example.com/someorg/someproject",
|
||||||
|
"permissions": permissions_block if add_permissions_block else None,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"name": "Another project",
|
||||||
|
"description": None,
|
||||||
|
"default_branch": "master",
|
||||||
|
"visibility": "public",
|
||||||
|
"path": "anotherproject",
|
||||||
|
"path_with_namespace": "someorg/anotherproject",
|
||||||
|
"last_activity_at": "2013-09-30T13:46:02Z",
|
||||||
|
"web_url": "http://example.com/someorg/anotherproject",
|
||||||
|
}]),
|
||||||
|
}
|
||||||
|
return projects_handler
|
||||||
|
|
||||||
|
|
||||||
|
def get_group_handler(null_avatar):
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/groups/2$')
|
||||||
|
def group_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": 1,
|
||||||
|
"name": "SomeOrg Group",
|
||||||
|
"path": "someorg",
|
||||||
|
"description": "An interesting group",
|
||||||
|
"visibility": "public",
|
||||||
|
"lfs_enabled": True,
|
||||||
|
"avatar_url": 'avatar_url' if not null_avatar else None,
|
||||||
|
"web_url": "http://gitlab.com/groups/someorg",
|
||||||
|
"request_access_enabled": False,
|
||||||
|
"full_name": "SomeOrg Group",
|
||||||
|
"full_path": "someorg",
|
||||||
|
"parent_id": None,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
return group_handler
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/files/Dockerfile$')
|
||||||
|
def dockerfile_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"file_name": "Dockerfile",
|
||||||
|
"file_path": "Dockerfile",
|
||||||
|
"size": 10,
|
||||||
|
"encoding": "base64",
|
||||||
|
"content": base64.b64encode('hello world'),
|
||||||
|
"ref": "master",
|
||||||
|
"blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
|
||||||
|
"commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
|
||||||
|
"last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d"
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/files/somesubdir%2FDockerfile$')
|
||||||
|
def sub_dockerfile_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"file_name": "Dockerfile",
|
||||||
|
"file_path": "somesubdir/Dockerfile",
|
||||||
|
"size": 10,
|
||||||
|
"encoding": "base64",
|
||||||
|
"content": base64.b64encode('hi universe'),
|
||||||
|
"ref": "master",
|
||||||
|
"blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
|
||||||
|
"commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
|
||||||
|
"last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d"
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tags/sometag$')
|
||||||
|
def tag_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"name": "sometag",
|
||||||
|
"message": "some cool message",
|
||||||
|
"target": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||||
|
"commit": {
|
||||||
|
"id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||||
|
"short_id": "60a8ff03",
|
||||||
|
"title": "Initial commit",
|
||||||
|
"created_at": "2017-07-26T11:08:53.000+02:00",
|
||||||
|
"parent_ids": [
|
||||||
|
"f61c062ff8bcbdb00e0a1b3317a91aed6ceee06b"
|
||||||
|
],
|
||||||
|
"message": "v5.0.0\n",
|
||||||
|
"author_name": "Arthur Verschaeve",
|
||||||
|
"author_email": "contact@arthurverschaeve.be",
|
||||||
|
"authored_date": "2015-02-01T21:56:31.000+01:00",
|
||||||
|
"committer_name": "Arthur Verschaeve",
|
||||||
|
"committer_email": "contact@arthurverschaeve.be",
|
||||||
|
"committed_date": "2015-02-01T21:56:31.000+01:00"
|
||||||
|
},
|
||||||
|
"release": None,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/foo%2Fbar/repository/commits/60a8ff033665e1207714d6670fcd7b65304ec02f$')
|
||||||
|
def commit_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||||
|
"short_id": "60a8ff03366",
|
||||||
|
"title": "Sanitize for network graph",
|
||||||
|
"author_name": "someguy",
|
||||||
|
"author_email": "some.guy@gmail.com",
|
||||||
|
"committer_name": "Some Guy",
|
||||||
|
"committer_email": "some.guy@gmail.com",
|
||||||
|
"created_at": "2012-09-20T09:06:12+03:00",
|
||||||
|
"message": "Sanitize for network graph",
|
||||||
|
"committed_date": "2012-09-20T09:06:12+03:00",
|
||||||
|
"authored_date": "2012-09-20T09:06:12+03:00",
|
||||||
|
"parent_ids": [
|
||||||
|
"ae1d9fb46aa2b07ee9836d49862ec4e2c46fbbba"
|
||||||
|
],
|
||||||
|
"last_pipeline" : {
|
||||||
|
"id": 8,
|
||||||
|
"ref": "master",
|
||||||
|
"sha": "2dc6aa325a317eda67812f05600bdf0fcdc70ab0",
|
||||||
|
"status": "created",
|
||||||
|
},
|
||||||
|
"stats": {
|
||||||
|
"additions": 15,
|
||||||
|
"deletions": 10,
|
||||||
|
"total": 25
|
||||||
|
},
|
||||||
|
"status": "running"
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/deploy_keys$', method='POST')
|
||||||
|
def create_deploykey_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": 1,
|
||||||
|
"title": "Public key",
|
||||||
|
"key": "ssh-rsa some stuff",
|
||||||
|
"created_at": "2013-10-02T10:12:29Z",
|
||||||
|
"can_push": False,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/hooks$', method='POST')
|
||||||
|
def create_hook_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({
|
||||||
|
"id": 1,
|
||||||
|
"url": "http://example.com/hook",
|
||||||
|
"project_id": 4,
|
||||||
|
"push_events": True,
|
||||||
|
"issues_events": True,
|
||||||
|
"confidential_issues_events": True,
|
||||||
|
"merge_requests_events": True,
|
||||||
|
"tag_push_events": True,
|
||||||
|
"note_events": True,
|
||||||
|
"job_events": True,
|
||||||
|
"pipeline_events": True,
|
||||||
|
"wiki_page_events": True,
|
||||||
|
"enable_ssl_verification": True,
|
||||||
|
"created_at": "2012-10-12T17:04:47Z",
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/hooks/1$', method='DELETE')
|
||||||
|
def delete_hook_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/deploy_keys/1$', method='DELETE')
|
||||||
|
def delete_deploykey_handker(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps({}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/users/1/projects$')
|
||||||
|
def user_projects_list_handler(_, request):
|
||||||
|
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||||
|
return {'status_code': 401}
|
||||||
|
|
||||||
|
return {
|
||||||
|
'status_code': 200,
|
||||||
|
'headers': {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
'content': json.dumps([
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"name": "Another project",
|
||||||
|
"description": None,
|
||||||
|
"default_branch": "master",
|
||||||
|
"visibility": "public",
|
||||||
|
"path": "anotherproject",
|
||||||
|
"path_with_namespace": "knownuser/anotherproject",
|
||||||
|
"last_activity_at": "2013-09-30T13:46:02Z",
|
||||||
|
"web_url": "http://example.com/knownuser/anotherproject",
|
||||||
|
}
|
||||||
|
]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def get_gitlab_trigger(dockerfile_path='', add_permissions=True, missing_avatar_url=False):
|
||||||
|
handlers = [user_handler, users_handler, project_branches_handler, project_tree_handler,
|
||||||
|
project_handler, get_projects_handler(add_permissions), tag_handler,
|
||||||
|
project_branch_handler, get_group_handler(missing_avatar_url), dockerfile_handler,
|
||||||
|
sub_dockerfile_handler, namespace_handler, user_namespace_handler, namespaces_handler,
|
||||||
|
commit_handler, create_deploykey_handler, delete_deploykey_handker,
|
||||||
|
create_hook_handler, delete_hook_handler, project_tags_handler,
|
||||||
|
user_projects_list_handler, catchall_handler]
|
||||||
|
|
||||||
|
with HTTMock(*handlers):
|
||||||
|
trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger'))
|
||||||
|
trigger = GitLabBuildTrigger(trigger_obj, {
|
||||||
|
'build_source': 'foo/bar',
|
||||||
|
'dockerfile_path': dockerfile_path,
|
||||||
|
'username': 'knownuser'
|
||||||
|
})
|
||||||
|
|
||||||
|
client = gitlab.Gitlab('http://fakegitlab', oauth_token='foobar', timeout=20, api_version=4)
|
||||||
|
client.auth()
|
||||||
|
|
||||||
|
trigger._get_authorized_client = lambda: client
|
||||||
|
yield trigger
|
||||||
55
buildtrigger/test/test_basehandler.py
Normal file
55
buildtrigger/test/test_basehandler.py
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from buildtrigger.basehandler import BuildTriggerHandler
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('input,output', [
|
||||||
|
("Dockerfile", True),
|
||||||
|
("server.Dockerfile", True),
|
||||||
|
(u"Dockerfile", True),
|
||||||
|
(u"server.Dockerfile", True),
|
||||||
|
("bad file name", False),
|
||||||
|
(u"bad file name", False),
|
||||||
|
])
|
||||||
|
def test_path_is_dockerfile(input, output):
|
||||||
|
assert BuildTriggerHandler.filename_is_dockerfile(input) == output
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('input,output', [
|
||||||
|
("", {}),
|
||||||
|
("/a", {"/a": ["/"]}),
|
||||||
|
("a", {"/a": ["/"]}),
|
||||||
|
("/b/a", {"/b/a": ["/b", "/"]}),
|
||||||
|
("b/a", {"/b/a": ["/b", "/"]}),
|
||||||
|
("/c/b/a", {"/c/b/a": ["/c/b", "/c", "/"]}),
|
||||||
|
("/a//b//c", {"/a/b/c": ["/", "/a", "/a/b"]}),
|
||||||
|
("/a", {"/a": ["/"]}),
|
||||||
|
])
|
||||||
|
def test_subdir_path_map_no_previous(input, output):
|
||||||
|
actual_mapping = BuildTriggerHandler.get_parent_directory_mappings(input)
|
||||||
|
for key in actual_mapping:
|
||||||
|
value = actual_mapping[key]
|
||||||
|
actual_mapping[key] = value.sort()
|
||||||
|
for key in output:
|
||||||
|
value = output[key]
|
||||||
|
output[key] = value.sort()
|
||||||
|
|
||||||
|
assert actual_mapping == output
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('new_path,original_dictionary,output', [
|
||||||
|
("/a", {}, {"/a": ["/"]}),
|
||||||
|
("b", {"/a": ["some_path", "another_path"]}, {"/a": ["some_path", "another_path"], "/b": ["/"]}),
|
||||||
|
("/a/b/c/d", {"/e": ["some_path", "another_path"]},
|
||||||
|
{"/e": ["some_path", "another_path"], "/a/b/c/d": ["/", "/a", "/a/b", "/a/b/c"]}),
|
||||||
|
])
|
||||||
|
def test_subdir_path_map(new_path, original_dictionary, output):
|
||||||
|
actual_mapping = BuildTriggerHandler.get_parent_directory_mappings(new_path, original_dictionary)
|
||||||
|
for key in actual_mapping:
|
||||||
|
value = actual_mapping[key]
|
||||||
|
actual_mapping[key] = value.sort()
|
||||||
|
for key in output:
|
||||||
|
value = output[key]
|
||||||
|
output[key] = value.sort()
|
||||||
|
|
||||||
|
assert actual_mapping == output
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Reference in a new issue