quay/endpoints/api/mirror.py

# -*- coding: utf-8 -*-
import logging

from email.utils import parsedate_tz, mktime_tz
from datetime import datetime

from jsonschema import ValidationError
from flask import request

import features

from auth.auth_context import get_authenticated_user
from data import model
from data.database import RepoMirrorRuleType
from endpoints.api import (RepositoryParamResource, nickname, path_param, require_repo_admin,
                           resource, validate_json_request, define_json_response, show_if,
                           format_date)
from endpoints.exception import NotFound
from util.audit import track_and_log, wrap_repository
from util.names import parse_robot_username


common_properties = {
  'is_enabled': {
    'type': 'boolean',
    'description': 'Used to enable or disable synchronizations.',
  },
  'external_reference': {
    'type': 'string',
    'description': 'Location of the external repository.'
  },
  'external_registry_username': {
    'type': ['string', 'null'],
    'description': 'Username used to authenticate with external registry.',
  },
  'external_registry_password': {
    'type': ['string', 'null'],
    'description': 'Password used to authenticate with external registry.',
  },
  'sync_start_date': {
    'type': 'string',
    'description': 'Determines the next time this repository is ready for synchronization.',
  },
  'sync_interval': {
    'type': 'integer',
    'minimum': 0,
    'description': 'Number of seconds after next_start_date to begin synchronizing.'
  },
  'robot_username': {
    'type': 'string',
    'description': 'Username of robot which will be used for image pushes.'
  },
  'root_rule': {
    'type': 'object',
    'description': 'Tag mirror rule',
    'required': [
      'rule_kind',
      'rule_value'
    ],
    'properties': {
      'rule_kind': {
        'type': 'string',
        'description': 'The kind of rule type',
        'enum': ['tag_glob_csv'],
      },
      'rule_value': {
        'type': 'array',
        'description': 'Array of tag patterns',
        'items': {
          'type': 'string'
        }
      }
    },
    'description': 'A list of glob-patterns used to determine which tags should be synchronized.'
  },
  'external_registry_config': {
    'type': 'object',
    'properties': {
      'verify_tls': {
        'type': 'boolean',
        'description': (
          'Determines whether HTTPs is required and the certificate is verified when '
          'communicating with the external repository.'
        ),
      },
      'proxy': {
        'type': 'object',
        'description': 'Proxy configuration for use during synchronization.',
        'properties': {
          'https_proxy': {
            'type': ['string', 'null'],
            'description': 'Value for HTTPS_PROXY environment variable during sync.'
          },
          'http_proxy': {
            'type': ['string', 'null'],
            'description': 'Value for HTTP_PROXY environment variable during sync.'
          },
          'no_proxy': {
            'type': ['string', 'null'],
            'description': 'Value for NO_PROXY environment variable during sync.'
          }
        }
      }
    }
  }
}


@resource('/v1/repository/<apirepopath:repository>/mirror/sync-now')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@show_if(features.REPO_MIRROR)
class RepoMirrorSyncNowResource(RepositoryParamResource):
  """ A resource for managing RepoMirrorConfig.sync_status """

  @require_repo_admin
  @nickname('syncNow')
  def post(self, namespace_name, repository_name):
    """ Update the sync_status for a given Repository's mirroring configuration. """
    repo = model.repository.get_repository(namespace_name, repository_name)
    if not repo:
      raise NotFound()

    mirror = model.repo_mirror.get_mirror(repository=repo)
    if not mirror:
      raise NotFound()

    if mirror and model.repo_mirror.update_sync_status_to_sync_now(mirror):
      track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed="sync_status", to="SYNC_NOW")
      return '', 204

    raise NotFound()


@resource('/v1/repository/<apirepopath:repository>/mirror/sync-cancel')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@show_if(features.REPO_MIRROR)
class RepoMirrorSyncCancelResource(RepositoryParamResource):
  """ A resource for managing RepoMirrorConfig.sync_status """

  @require_repo_admin
  @nickname('syncCancel')
  def post(self, namespace_name, repository_name):
    """ Update the sync_status for a given Repository's mirroring configuration. """
    repo = model.repository.get_repository(namespace_name, repository_name)
    if not repo:
      raise NotFound()

    mirror = model.repo_mirror.get_mirror(repository=repo)
    if not mirror:
      raise NotFound()

    if mirror and model.repo_mirror.update_sync_status_to_cancel(mirror):
      track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed="sync_status", to="SYNC_CANCEL")
      return '', 204

    raise NotFound()


@resource('/v1/repository/<apirepopath:repository>/mirror')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@show_if(features.REPO_MIRROR)
class RepoMirrorResource(RepositoryParamResource):
  """
  Resource for managing repository mirroring.
  """
  schemas = {
    'CreateMirrorConfig': {
      'description': 'Create the repository mirroring configuration.',
      'type': 'object',
      'required': [
        'external_reference',
        'sync_interval',
        'sync_start_date',
        'root_rule'
      ],
      'properties': common_properties
    },
    'UpdateMirrorConfig': {
      'description': 'Update the repository mirroring configuration.',
      'type': 'object',
      'properties': common_properties
    },
    'ViewMirrorConfig': {
      'description': 'View the repository mirroring configuration.',
      'type': 'object',
      'required': [
        'is_enabled',
        'mirror_type',
        'external_reference',
        'external_registry_username',
        'external_registry_config',
        'sync_interval',
        'sync_start_date',
        'sync_expiration_date',
        'sync_retries_remaining',
        'sync_status',
        'root_rule',
        'robot_username',
      ],
      'properties': common_properties
    }
  }

  @require_repo_admin
  @define_json_response('ViewMirrorConfig')
  @nickname('getRepoMirrorConfig')
  def get(self, namespace_name, repository_name):
    """ Return the Mirror configuration for a given Repository. """
    repo = model.repository.get_repository(namespace_name, repository_name)
    if not repo:
      raise NotFound()

    mirror = model.repo_mirror.get_mirror(repo)
    if not mirror:
      raise NotFound()

    # Transformations
    rules = mirror.root_rule.rule_value
    username = self._decrypt_username(mirror.external_registry_username)
    sync_start_date = self._dt_to_string(mirror.sync_start_date)
    sync_expiration_date = self._dt_to_string(mirror.sync_expiration_date)
    robot = mirror.internal_robot.username if mirror.internal_robot is not None else None

    return {
      'is_enabled': mirror.is_enabled,
      'mirror_type': mirror.mirror_type.name,
      'external_reference': mirror.external_reference,
      'external_registry_username': username,
      'external_registry_config': mirror.external_registry_config or {},
      'sync_interval': mirror.sync_interval,
      'sync_start_date': sync_start_date,
      'sync_expiration_date': sync_expiration_date,
      'sync_retries_remaining': mirror.sync_retries_remaining,
      'sync_status': mirror.sync_status.name,
      'root_rule': {
        'rule_kind': 'tag_glob_csv',
        'rule_value': rules
      },
      'robot_username': robot,
    }

  @require_repo_admin
  @nickname('createRepoMirrorConfig')
  @validate_json_request('CreateMirrorConfig')
  def post(self, namespace_name, repository_name):
    """ Create a RepoMirrorConfig for a given Repository. """
    # TODO: Tidy up this function
    # TODO: Specify only the data we want to pass on when creating the RepoMirrorConfig. Avoid
    #       the possibility of data injection.

    repo = model.repository.get_repository(namespace_name, repository_name)
    if not repo:
      raise NotFound()

    if model.repo_mirror.get_mirror(repo):
      return {'detail': 'Mirror configuration already exits for repository %s/%s' % (
        namespace_name, repository_name)}, 409

    data = request.get_json()

    data['sync_start_date'] = self._string_to_dt(data['sync_start_date'])

    rule = model.repo_mirror.create_rule(repo, data['root_rule']['rule_value'])
    del data['root_rule']

    # Verify the robot is part of the Repository's namespace
    robot = self._setup_robot_for_mirroring(namespace_name, repository_name, data['robot_username'])
    del data['robot_username']

    mirror = model.repo_mirror.enable_mirroring_for_repository(repo, root_rule=rule,
                                                               internal_robot=robot, **data)
    if mirror:
      track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='external_reference', to=data['external_reference'])
      return '', 201
    else:
      # TODO: Determine appropriate Response
      return {'detail': 'RepoMirrorConfig already exists for this repository.'}, 409

  @require_repo_admin
  @validate_json_request('UpdateMirrorConfig')
  @nickname('changeRepoMirrorConfig')
  def put(self, namespace_name, repository_name):
    """ Allow users to modifying the repository's mirroring configuration. """
    values = request.get_json()

    repo = model.repository.get_repository(namespace_name, repository_name)
    if not repo:
      raise NotFound()

    mirror = model.repo_mirror.get_mirror(repo)
    if not mirror:
      raise NotFound()

    if 'is_enabled' in values:
      if values['is_enabled'] == True:
        if model.repo_mirror.enable_mirror(repo):
          track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='is_enabled', to=True)
      if values['is_enabled'] == False:
        if model.repo_mirror.disable_mirror(repo):
          track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='is_enabled', to=False)

    if 'external_reference' in values:
      if values['external_reference'] == '':
        return {'detail': 'Empty string is an invalid repository location.'}, 400
      if model.repo_mirror.change_remote(repo, values['external_reference']):
        track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='external_reference', to=values['external_reference'])

    if 'robot_username' in values:
      robot_username = values['robot_username']
      robot = self._setup_robot_for_mirroring(namespace_name, repository_name, robot_username)
      if model.repo_mirror.set_mirroring_robot(repo, robot):
        track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='robot_username', to=robot_username)

    if 'sync_start_date' in values:
      try:
        sync_start_date = self._string_to_dt(values['sync_start_date'])
      except ValueError as e:
        return {'detail': 'Incorrect DateTime format for sync_start_date.'}, 400
      if model.repo_mirror.change_sync_start_date(repo, sync_start_date):
        track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='sync_start_date', to=sync_start_date)

    if 'sync_interval' in values:
      if model.repo_mirror.change_sync_interval(repo, values['sync_interval']):
        track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='sync_interval', to=values['sync_interval'])

    if 'external_registry_username' in values and 'external_registry_password' in values:
      username = values['external_registry_username']
      password = values['external_registry_password']
      if username is None and password is not None:
        return {'detail': 'Unable to delete username while setting a password.'}, 400
      if model.repo_mirror.change_credentials(repo, username, password):
        track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='external_registry_username', to=username)
        if password is None:
          track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='external_registry_password', to=None)
        else:
          track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='external_registry_password', to="********")

    elif 'external_registry_username' in values:
      username = values['external_registry_username']
      if model.repo_mirror.change_username(repo, username):
        track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='external_registry_username', to=username)

    # Do not allow specifying a password without setting a username
    if 'external_registry_password' in values and 'external_registry_username' not in values:
      return {'detail': 'Unable to set a new password without also specifying a username.'}, 400

    if 'external_registry_config' in values:
      external_registry_config = values.get('external_registry_config', {})

      if 'verify_tls' in external_registry_config:
        updates = {'verify_tls': external_registry_config['verify_tls']}
        if model.repo_mirror.change_external_registry_config(repo, updates):
          track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='verify_tls', to=external_registry_config['verify_tls'])

      if 'proxy' in external_registry_config:
          proxy_values = external_registry_config.get('proxy', {})

          if 'http_proxy' in proxy_values:
            updates = {'proxy': {'http_proxy': proxy_values['http_proxy']}}
            if model.repo_mirror.change_external_registry_config(repo, updates):
              track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='http_proxy', to=proxy_values['http_proxy'])

          if 'https_proxy' in proxy_values:
            updates = {'proxy': {'https_proxy': proxy_values['https_proxy']}}
            if model.repo_mirror.change_external_registry_config(repo, updates):
              track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='https_proxy', to=proxy_values['https_proxy'])

          if 'no_proxy' in proxy_values:
            updates = {'proxy': {'no_proxy': proxy_values['no_proxy']}}
            if model.repo_mirror.change_external_registry_config(repo, updates):
              track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='no_proxy', to=proxy_values['no_proxy'])

    if 'root_rule' in values:

      if values['root_rule']['rule_kind'] != "tag_glob_csv":
        raise ValidationError('validation failed: rule_kind must be "tag_glob_csv"')

      if model.repo_mirror.change_rule(repo, RepoMirrorRuleType.TAG_GLOB_CSV, values['root_rule']['rule_value']):
        track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed="mirror_rule", to=values['root_rule']['rule_value'])

    return '', 201

  def _setup_robot_for_mirroring(self, namespace_name, repo_name, robot_username):
    """ Validate robot exists and give write permissions. """
    robot = model.user.lookup_robot(robot_username)
    assert robot.robot

    namespace, _ = parse_robot_username(robot_username)
    if namespace != namespace_name:
      raise model.DataModelException('Invalid robot')

    # Ensure the robot specified has access to the repository. If not, grant it.
    permissions = model.permission.get_user_repository_permissions(robot, namespace_name, repo_name)
    if not permissions or permissions[0].role.name == 'read':
      model.permission.set_user_repo_permission(robot.username, namespace_name, repo_name, 'write')

    return robot

  def _string_to_dt(self, string):
    """ Convert String to correct DateTime format. """
    if string is None:
      return None

    """
    # TODO: Use RFC2822. This doesn't work consistently.
    # TODO: Move this to same module as `format_date` once fixed.
    tup = parsedate_tz(string)
    if len(tup) == 8:
      tup = tup + (0,)  # If TimeZone is omitted, assume UTC
    ts = mktime_tz(tup)
    dt = datetime.fromtimestamp(ts, pytz.UTC)
    return dt
    """
    assert isinstance(string, (str, unicode))
    dt = datetime.strptime(string, "%Y-%m-%dT%H:%M:%SZ")
    return dt

  def _dt_to_string(self, dt):
    """ Convert DateTime to correctly formatted String."""
    if dt is None:
      return None

    """
    # TODO: Use RFC2822. Need to make it work bi-directionally.
    return format_date(dt)
    """

    assert isinstance(dt, datetime)
    string = dt.isoformat() + 'Z'
    return string

  def _decrypt_username(self, username):
    if username is None:
      return None
    return username.decrypt()