From fc7301be0d5e7dc290135dcf6673c6fe69f4a93f Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 28 Sep 2016 20:17:14 -0400 Subject: [PATCH] *: fix legacy imports This change reorganizes imports and renames the legacy flask extensions. --- app.py | 6 ++-- auth/auth.py | 15 +++++---- auth/permissions.py | 7 ++-- auth/registry_jwt_auth.py | 2 +- endpoints/api/__init__.py | 16 +++++---- endpoints/api/discovery.py | 10 +++--- endpoints/api/user.py | 20 ++++++----- endpoints/bitbuckettrigger.py | 10 +++--- endpoints/common.py | 22 ++++++------ endpoints/decorated.py | 10 +++--- endpoints/githubtrigger.py | 2 +- endpoints/gitlabtrigger.py | 7 ++-- endpoints/notificationmethod.py | 11 ++++-- endpoints/oauthlogin.py | 4 ++- endpoints/realtime.py | 7 ++-- endpoints/web.py | 6 ++-- test/registry_tests.py | 59 ++++++++++++++++----------------- test/test_auth.py | 17 +++++----- test/test_external_jwt_authn.py | 23 ++++++++----- test/test_keystone_auth.py | 12 ++++--- util/cache.py | 3 +- util/config/validator.py | 39 ++++++++++++---------- util/useremails.py | 10 ++++-- 23 files changed, 179 insertions(+), 139 deletions(-) diff --git a/app.py b/app.py index 579dd0229..c5eb2ad20 100644 --- a/app.py +++ b/app.py @@ -4,9 +4,9 @@ import json from functools import partial from flask import Flask, request, Request, _request_ctx_stack -from flask.ext.principal import Principal -from flask.ext.login import LoginManager, UserMixin -from flask.ext.mail import Mail +from flask_principal import Principal +from flask_login import LoginManager, UserMixin +from flask_mail import Mail from werkzeug.routing import BaseConverter from jwkest.jwk import RSAKey from Crypto.PublicKey import RSA diff --git a/auth/auth.py b/auth/auth.py index ac86a093a..a9910825f 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -3,20 +3,21 @@ import logging from functools import wraps from uuid import UUID from datetime import datetime -from flask import request, session -from flask.ext.principal import identity_changed, Identity -from flask.ext.login import current_user -from flask.sessions import SecureCookieSessionInterface, BadSignature from base64 import b64decode +from flask import request, session +from flask.sessions import SecureCookieSessionInterface, BadSignature +from flask_login import current_user +from flask_principal import identity_changed, Identity + import scopes -from data import model from app import app, authentication -from endpoints.exception import InvalidToken, ExpiredToken -from permissions import QuayDeferredPermissionUser from auth_context import (set_authenticated_user, set_validated_token, set_grant_context, set_validated_oauth_token) +from data import model +from endpoints.exception import InvalidToken, ExpiredToken +from permissions import QuayDeferredPermissionUser from util.http import abort diff --git a/auth/permissions.py b/auth/permissions.py index 2cf778d32..5412bc7b7 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -1,13 +1,14 @@ import logging -from flask.ext.principal import identity_loaded, Permission, Identity, identity_changed from collections import namedtuple, defaultdict from functools import partial -import scopes +from flask_principal import identity_loaded, Permission, Identity, identity_changed + -from data import model from app import app, superusers +from auth import scopes +from data import model logger = logging.getLogger(__name__) diff --git a/auth/registry_jwt_auth.py b/auth/registry_jwt_auth.py index a4c48d749..69cb1b6cb 100644 --- a/auth/registry_jwt_auth.py +++ b/auth/registry_jwt_auth.py @@ -4,7 +4,7 @@ from functools import wraps from jsonschema import validate, ValidationError from flask import request, url_for -from flask.ext.principal import identity_changed, Identity +from flask_principal import identity_changed, Identity from app import app, get_app_url, instance_keys from .auth_context import set_grant_context, get_grant_context diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 3f0241b7d..e93301f55 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -1,19 +1,19 @@ import logging import datetime import json -from enum import Enum -from app import app, metric_queue -from flask import Blueprint, Response, request, make_response, jsonify, session, url_for -from flask.ext.restful import Resource, abort, Api, reqparse -from flask.ext.restful.utils.cors import crossdomain from calendar import timegm from email.utils import formatdate from functools import partial, wraps + +from enum import Enum +from flask import Blueprint, Response, request, make_response, jsonify, session, url_for +from flask_restful import Resource, abort, Api, reqparse +from flask_restful.utils.cors import crossdomain from jsonschema import validate, ValidationError +from app import app, metric_queue from data import model -from util.names import parse_namespace_repository from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, AdministerRepositoryPermission, UserReadPermission, UserAdminPermission) @@ -21,9 +21,11 @@ from auth import scopes from auth.auth_context import get_authenticated_user, get_validated_oauth_token from auth.auth import process_oauth from endpoints.csrf import csrf_protect -from endpoints.exception import ApiException, Unauthorized, InvalidRequest, InvalidResponse, FreshLoginRequired +from endpoints.exception import (ApiException, Unauthorized, InvalidRequest, InvalidResponse, + FreshLoginRequired) from endpoints.decorators import check_anon_protection from util.metrics.metricqueue import time_decorator +from util.names import parse_namespace_repository from util.pagination import encrypt_page_token, decrypt_page_token logger = logging.getLogger(__name__) diff --git a/endpoints/api/discovery.py b/endpoints/api/discovery.py index 1e59debb6..a9faf9ae2 100644 --- a/endpoints/api/discovery.py +++ b/endpoints/api/discovery.py @@ -4,13 +4,15 @@ import re import logging import sys -from flask.ext.restful import reqparse +from collections import OrderedDict + +from flask_restful import reqparse -from endpoints.api import (ApiResource, resource, method_metadata, nickname, truthy_bool, - parse_args, query_param) from app import app from auth import scopes -from collections import OrderedDict +from endpoints.api import (ApiResource, resource, method_metadata, nickname, truthy_bool, + parse_args, query_param) + logger = logging.getLogger(__name__) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 3680f841c..7547d014a 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -4,13 +4,19 @@ import logging import json from flask import request, abort -from flask.ext.login import logout_user -from flask.ext.principal import identity_changed, AnonymousIdentity +from flask_login import logout_user +from flask_principal import identity_changed, AnonymousIdentity from peewee import IntegrityError import features from app import app, billing as stripe, authentication, avatar +from auth import scopes +from auth.auth_context import get_authenticated_user +from auth.permissions import (AdministerOrganizationPermission, CreateRepositoryPermission, + UserAdminPermission, UserReadPermission, SuperUserPermission) +from data import model +from data.billing import get_plan from data.database import Repository as RepositoryTable from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, log_action, internal_only, require_user_admin, parse_args, @@ -21,21 +27,17 @@ from endpoints.exception import NotFound, InvalidToken from endpoints.api.subscribe import subscribe from endpoints.common import common_login from endpoints.decorators import anon_allowed - -from data import model -from data.billing import get_plan -from auth.permissions import (AdministerOrganizationPermission, CreateRepositoryPermission, - UserAdminPermission, UserReadPermission, SuperUserPermission) -from auth.auth_context import get_authenticated_user -from auth import scopes from util.useremails import (send_confirmation_email, send_recovery_email, send_change_email, send_password_changed, send_org_recovery_email) from util.names import parse_single_urn + REPOS_PER_PAGE = 100 + logger = logging.getLogger(__name__) + def handle_invite_code(invite_code, user): """ Checks that the given invite code matches the specified user's e-mail address. If so, the user is marked as having a verified e-mail address and this method returns True. diff --git a/endpoints/bitbuckettrigger.py b/endpoints/bitbuckettrigger.py index b3a9ca63a..f0111b0f6 100644 --- a/endpoints/bitbuckettrigger.py +++ b/endpoints/bitbuckettrigger.py @@ -1,15 +1,15 @@ import logging from flask import request, redirect, url_for, Blueprint -from flask.ext.login import current_user +from flask_login import current_user +from app import app +from auth.auth import require_session_login from buildtrigger.basehandler import BuildTriggerHandler from buildtrigger.bitbuckethandler import BitbucketBuildTrigger -from endpoints.common import route_show_if -from app import app from data import model +from endpoints.common import route_show_if from util.http import abort -from auth.auth import require_session_login import features @@ -44,4 +44,4 @@ def attach_bitbucket_build_trigger(trigger_uuid): trigger.uuid) logger.debug('Redirecting to full url: %s', full_url) - return redirect(full_url) \ No newline at end of file + return redirect(full_url) diff --git a/endpoints/common.py b/endpoints/common.py index 5de9c3ee8..791f987f8 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -10,20 +10,21 @@ from functools import wraps from cachetools import lru_cache from flask import make_response, render_template, request, abort, session -from flask.ext.login import login_user -from flask.ext.principal import identity_changed - -from app import app, oauth_apps, LoginWrappedDBUser -from auth.permissions import QuayDeferredPermissionUser -from auth import scopes -from config import frontend_visible_config -from external_libraries import get_external_javascript, get_external_css -from util.secscan import PRIORITY_LEVELS -from util.names import parse_namespace_repository +from flask_login import login_user +from flask_principal import identity_changed import endpoints.decorated # Register the various exceptions via decorators. import features +from app import app, oauth_apps, LoginWrappedDBUser +from auth import scopes +from auth.permissions import QuayDeferredPermissionUser +from config import frontend_visible_config +from external_libraries import get_external_javascript, get_external_css +from util.names import parse_namespace_repository +from util.secscan import PRIORITY_LEVELS + + logger = logging.getLogger(__name__) route_data = None @@ -31,6 +32,7 @@ route_data = None CACHE_BUSTERS_JSON = 'static/dist/cachebusters.json' CACHE_BUSTERS = None + def get_cache_busters(): """ Retrieves the cache busters hashes. """ global CACHE_BUSTERS diff --git a/endpoints/decorated.py b/endpoints/decorated.py index 2404f2b87..1e19996fe 100644 --- a/endpoints/decorated.py +++ b/endpoints/decorated.py @@ -1,15 +1,17 @@ import logging -import json from flask import make_response, jsonify +from flask_restful.utils.cors import crossdomain + from app import app -from util.useremails import CannotSendEmailException -from util.config.provider.baseprovider import CannotWriteConfigException -from flask.ext.restful.utils.cors import crossdomain from data import model +from util.config.provider.baseprovider import CannotWriteConfigException +from util.useremails import CannotSendEmailException + logger = logging.getLogger(__name__) + @app.errorhandler(model.DataModelException) def handle_dme(ex): logger.exception(ex) diff --git a/endpoints/githubtrigger.py b/endpoints/githubtrigger.py index 89c5f2841..dbd75c27d 100644 --- a/endpoints/githubtrigger.py +++ b/endpoints/githubtrigger.py @@ -1,7 +1,7 @@ import logging from flask import request, redirect, url_for, Blueprint -from flask.ext.login import current_user +from flask_login import current_user import features diff --git a/endpoints/gitlabtrigger.py b/endpoints/gitlabtrigger.py index 927d154ca..32499ca52 100644 --- a/endpoints/gitlabtrigger.py +++ b/endpoints/gitlabtrigger.py @@ -1,7 +1,9 @@ import logging from flask import Blueprint, request, redirect, url_for -from flask.ext.login import current_user +from flask_login import current_user + +import features from app import app, gitlab_trigger from auth.auth import require_session_login @@ -10,13 +12,12 @@ from data import model from endpoints.common import route_show_if from util.http import abort -import features - logger = logging.getLogger(__name__) client = app.config['HTTPCLIENT'] gitlabtrigger = Blueprint('gitlab', __name__) + @gitlabtrigger.route('/gitlab/callback/trigger', methods=['GET']) @route_show_if(features.GITLAB_BUILD) @require_session_login diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index d3e444fc1..18844715f 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -1,22 +1,28 @@ import logging import json -import requests import re -from flask.ext.mail import Message +import requests + +from flask_mail import Message + from app import mail, app, OVERRIDE_CONFIG_DIRECTORY from data import model from util.config.validator import SSL_FILENAMES from workers.queueworker import JobException + logger = logging.getLogger(__name__) + class InvalidNotificationMethodException(Exception): pass + class CannotValidateNotificationMethodException(Exception): pass + class NotificationMethodPerformException(JobException): pass @@ -26,6 +32,7 @@ if app.config['PREFERRED_URL_SCHEME'] == 'https': # TODO(jschorr): move this into the config provider library SSLClientCert = [OVERRIDE_CONFIG_DIRECTORY + f for f in SSL_FILENAMES] + class NotificationMethod(object): def __init__(self): pass diff --git a/endpoints/oauthlogin.py b/endpoints/oauthlogin.py index 21f13bbd9..23e5ed3e9 100644 --- a/endpoints/oauthlogin.py +++ b/endpoints/oauthlogin.py @@ -1,8 +1,9 @@ import logging + import requests from flask import request, redirect, url_for, Blueprint -from flask.ext.login import current_user +from flask_login import current_user from peewee import IntegrityError import features @@ -20,6 +21,7 @@ logger = logging.getLogger(__name__) client = app.config['HTTPCLIENT'] oauthlogin = Blueprint('oauthlogin', __name__) + def render_ologin_error(service_name, error_message='Could not load user data. The token may have expired.'): user_creation = features.USER_CREATION and features.DIRECT_LOGIN diff --git a/endpoints/realtime.py b/endpoints/realtime.py index b1e5151b2..38a53c327 100644 --- a/endpoints/realtime.py +++ b/endpoints/realtime.py @@ -2,13 +2,16 @@ import logging import json from flask import request, Blueprint, abort, Response -from flask.ext.login import current_user -from auth.auth import require_session_login +from flask_login import current_user + from app import userevents +from auth.auth import require_session_login from data.userevent import CannotReadUserEventsException + logger = logging.getLogger(__name__) + realtime = Blueprint('realtime', __name__) diff --git a/endpoints/web.py b/endpoints/web.py index 5fdbe6ee5..7b0b477e3 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -1,13 +1,12 @@ import json import logging -from urlparse import urlparse from datetime import timedelta from cachetools import lru_cache from flask import (abort, redirect, request, url_for, make_response, Response, render_template, - Blueprint, send_from_directory, jsonify, send_file) -from flask.ext.login import current_user + Blueprint, jsonify, send_file) +from flask_login import current_user import features @@ -38,7 +37,6 @@ from util.systemlogs import build_logs_archive from util.useremails import send_email_changed - @lru_cache(maxsize=1) def _get_route_data(): return swagger_route_data(include_internal=True, compact=True) diff --git a/test/registry_tests.py b/test/registry_tests.py index a5c42fb56..0fcd01600 100644 --- a/test/registry_tests.py +++ b/test/registry_tests.py @@ -1,49 +1,48 @@ -import unittest -import requests -import os -import math -import random -import string -import resumablehashlib import binascii -import uuid +import hashlib +import json +import logging +import math +import os +import random +import shutil +import string +import tarfile import time -import gpgme +import unittest +import uuid -import Crypto.Random +from cStringIO import StringIO +from tempfile import NamedTemporaryFile + +import bencode +import gpgme +import requests +import resumablehashlib + +from Crypto import Random +from Crypto.PublicKey import RSA from flask import request, jsonify from flask.blueprints import Blueprint -from flask.ext.testing import LiveServerTestCase +from flask_testing import LiveServerTestCase +from jwkest.jwk import RSAKey + +import endpoints.decorated # required for side effect from app import app, storage, instance_keys from data.database import close_db_filter, configure, DerivedStorageForImage, QueueItem, Image from data import model +from digest.checksums import compute_simple +from endpoints.api import api_bp +from endpoints.csrf import generate_csrf_token from endpoints.v1 import v1_bp from endpoints.v2 import v2_bp from endpoints.verbs import verbs -from endpoints.api import api_bp from image.docker.schema1 import DockerSchema1ManifestBuilder - from initdb import wipe_database, initialize_database, populate_database -from endpoints.csrf import generate_csrf_token -from tempfile import NamedTemporaryFile from jsonschema import validate as validate_schema from util.security.registry_jwt import decode_bearer_token -import endpoints.decorated -import json -import hashlib -import logging -import bencode - -import tarfile -import shutil - -from jwkest.jwk import RSAKey -from Crypto.PublicKey import RSA - -from cStringIO import StringIO -from digest.checksums import compute_simple try: app.register_blueprint(v1_bp, url_prefix='/v1') @@ -119,7 +118,7 @@ def reload_app(): configure(app.config) # Reload random after the process split, as it cannot be used uninitialized across forks. - Crypto.Random.atfork() + Random.atfork() return 'OK' app.register_blueprint(testbp, url_prefix='/__test') diff --git a/test/test_auth.py b/test/test_auth.py index 8bf78f897..3136cc7a4 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1,26 +1,27 @@ -import unittest import base64 +import unittest + from datetime import datetime, timedelta -from app import app -from data import model -from data.database import OAuthApplication, OAuthAccessToken from flask import g -from flask.ext.principal import identity_loaded +from flask_principal import identity_loaded +from app import app from auth.auth import _process_basic_auth from auth.scopes import (scopes_from_scope_string, is_subset_string, DIRECT_LOGIN, ADMIN_REPO, ALL_SCOPES) from auth.permissions import QuayDeferredPermissionUser -from endpoints.api import api_bp, api +from data import model +from data.database import OAuthApplication, OAuthAccessToken +from endpoints.api import api from endpoints.api.user import User, Signin - -import json as py_json from test.test_api_usage import ApiTestCase + ADMIN_ACCESS_USER = 'devtable' DISABLED_USER = 'disabled' + @identity_loaded.connect_via(app) def on_identity_loaded(sender, identity): g.identity = identity diff --git a/test/test_external_jwt_authn.py b/test/test_external_jwt_authn.py index 06627baf7..9b561d0fb 100644 --- a/test/test_external_jwt_authn.py +++ b/test/test_external_jwt_authn.py @@ -1,19 +1,24 @@ -import unittest -import requests -import jwt import base64 +import unittest + +from datetime import datetime, timedelta +from tempfile import NamedTemporaryFile + +import jwt +import requests + +from Crypto.PublicKey import RSA +from flask import Flask, jsonify, request, make_response +from flask_testing import LiveServerTestCase from app import app -from flask import Flask, jsonify, request, make_response -from flask.ext.testing import LiveServerTestCase -from initdb import setup_database_for_testing, finished_database_for_testing from data.users import ExternalJWTAuthN -from tempfile import NamedTemporaryFile -from Crypto.PublicKey import RSA -from datetime import datetime, timedelta +from initdb import setup_database_for_testing, finished_database_for_testing + _PORT_NUMBER = 5001 + class JWTAuthTestCase(LiveServerTestCase): maxDiff = None diff --git a/test/test_keystone_auth.py b/test/test_keystone_auth.py index 2a3eadf5f..0603c7f25 100644 --- a/test/test_keystone_auth.py +++ b/test/test_keystone_auth.py @@ -1,14 +1,18 @@ -import unittest -import requests -import os import json +import os +import unittest + +import requests from flask import Flask, request, abort -from flask.ext.testing import LiveServerTestCase +from flask_testing import LiveServerTestCase + from data.users.keystone import KeystoneUsers + _PORT_NUMBER = 5001 + class KeystoneAuthTests(LiveServerTestCase): maxDiff = None diff --git a/util/cache.py b/util/cache.py index 13c0949de..64f626539 100644 --- a/util/cache.py +++ b/util/cache.py @@ -1,5 +1,6 @@ from functools import wraps -from flask.ext.restful.utils import unpack + +from flask_restful.utils import unpack def cache_control(max_age=55): diff --git a/util/config/validator.py b/util/config/validator.py index dfe356d19..73696aa26 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -1,33 +1,36 @@ -import redis -import ldap -import peewee -import OpenSSL import logging -import time import subprocess +import time from StringIO import StringIO from fnmatch import fnmatch -from data.users import LDAP_CERT_FILENAME -from data.users.keystone import KeystoneUsers -from data.users.externaljwt import ExternalJWTAuthN -from data.users.externalldap import LDAPConnection, LDAPUsers + +import OpenSSL +import ldap +import peewee +import redis from flask import Flask -from flask.ext.mail import Mail, Message -from data.database import validate_database_url -from storage import get_storage_driver -from auth.auth_context import get_authenticated_user -from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig -from bitbucket import BitBucket -from util.security.signing import SIGNING_ENGINES -from util.secscan.api import SecurityScannerAPI -from boot import setup_jwt_proxy +from flask_mail import Mail, Message from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY +from auth.auth_context import get_authenticated_user +from bitbucket import BitBucket +from boot import setup_jwt_proxy +from data.database import validate_database_url +from data.users import LDAP_CERT_FILENAME +from data.users.externaljwt import ExternalJWTAuthN +from data.users.externalldap import LDAPConnection, LDAPUsers +from data.users.keystone import KeystoneUsers +from storage import get_storage_driver +from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig +from util.secscan.api import SecurityScannerAPI +from util.security.signing import SIGNING_ENGINES + logger = logging.getLogger(__name__) + # Note: Only add files required for HTTPS to the SSL_FILESNAMES list. SSL_FILENAMES = ['ssl.cert', 'ssl.key'] DB_SSL_FILENAMES = ['database.pem'] diff --git a/util/useremails.py b/util/useremails.py index 60ba7d9be..8bf8a3ae8 100644 --- a/util/useremails.py +++ b/util/useremails.py @@ -1,18 +1,22 @@ -import logging import json -import features +import logging -from flask.ext.mail import Message +from flask_mail import Message + +import features from app import mail, app, get_app_url from util.jinjautil import get_template_env + logger = logging.getLogger(__name__) template_env = get_template_env("emails") + class CannotSendEmailException(Exception): pass + class GmailAction(object): """ Represents an action that can be taken in Gmail in response to the email. """ def __init__(self, metadata):