initial import for Open Source 🎉

util/repomirror/api.py

@@ -0,0 +1,133 @@
+import os
+import logging
+
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
+import requests
+from util.abchelpers import nooper
+from util.repomirror.validator import RepoMirrorConfigValidator
+
+from _init import CONF_DIR
+TOKEN_VALIDITY_LIFETIME_S = 60  # Amount of time the repo mirror has to call the skopeo URL
+
+MITM_CERT_PATH = os.path.join(CONF_DIR, 'mitm.cert')
+
+DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
+
+logger = logging.getLogger(__name__)
+
+
+class RepoMirrorException(Exception):
+  """ Exception raised when a layer fails to analyze due to a request issue. """
+
+class RepoMirrorRetryException(Exception):
+  """ Exception raised when a layer fails to analyze due to a request issue, and the request should
+      be retried.
+  """
+
+class APIRequestFailure(Exception):
+  """ Exception raised when there is a failure to conduct an API request. """
+
+class Non200ResponseException(Exception):
+  """ Exception raised when the upstream API returns a non-200 HTTP status code. """
+  def __init__(self, response):
+    super(Non200ResponseException, self).__init__()
+    self.response = response
+
+
+_API_METHOD_GET_REPOSITORY = 'repository/%s'
+_API_METHOD_PING = 'metrics'
+
+
+class RepoMirrorAPI(object):
+  """ Helper class for talking to the Repository Mirror service (usually Skopeo). """
+  def __init__(self, config, server_hostname=None, skip_validation=False, instance_keys=None):
+    feature_enabled = config.get('FEATURE_REPO_MIRROR', False)
+    has_valid_config = skip_validation
+
+    if not skip_validation and feature_enabled:
+      config_validator = RepoMirrorConfigValidator(feature_enabled)
+      has_valid_config = config_validator.valid()
+
+    self.state = NoopRepoMirrorAPI()
+    if feature_enabled and has_valid_config:
+      self.state = ImplementedRepoMirrorAPI(config, server_hostname, instance_keys=instance_keys)
+
+  def __getattr__(self, name):
+    return getattr(self.state, name, None)
+
+
+@add_metaclass(ABCMeta)
+class RepoMirrorAPIInterface(object):
+  """ Helper class for talking to the Repository Mirror service (usually Skopeo Worker). """
+
+  @abstractmethod
+  def ping(self):
+    """ Calls GET on the metrics endpoint of the repo mirror to ensure it is running
+        and properly configured. Returns the HTTP response.
+    """
+    pass
+
+  @abstractmethod
+  def repository_mirror(self, repository):
+    """ Posts the given repository to the repo mirror for processing, blocking until complete.
+        Returns the analysis version on success or raises an exception deriving from
+        AnalyzeLayerException on failure. Callers should handle all cases of AnalyzeLayerException.
+    """
+    pass
+
+  @abstractmethod
+  def get_repository_data(self, repository):
+    """ Returns the layer data for the specified layer. On error, returns None. """
+    pass
+
+
+@nooper
+class NoopRepoMirrorAPI(RepoMirrorAPIInterface):
+  """ No-op version of the repo mirror API. """
+  pass
+
+
+class ImplementedRepoMirrorAPI(RepoMirrorAPIInterface):
+  """ Helper class for talking to the repo mirror service. """
+  def __init__(self, config, server_hostname, client=None, instance_keys=None):
+    self._config = config
+    self._instance_keys = instance_keys
+    self._client = client
+    self._server_hostname = server_hostname
+
+  def repository_mirror(self, repository):
+    """ Posts the given repository and config information to the mirror endpoint, blocking until complete.
+        Returns the results on success or raises an exception.
+    """
+    def _response_json(request, response):
+      try:
+        return response.json()
+      except ValueError:
+        logger.exception('Failed to decode JSON when analyzing layer %s', request['Layer']['Name'])
+        raise RepoMirrorException
+
+    return
+
+  def get_repository_data(self, repository):
+    """ Returns the layer data for the specified layer. On error, returns None. """
+    return None
+
+  def ping(self):
+    """ Calls GET on the metrics endpoint of the repository mirror to ensure it is running
+        and properly configured. Returns the HTTP response.
+    """
+    try:
+      return self._call('GET', _API_METHOD_PING)
+    except requests.exceptions.Timeout as tie:
+      logger.exception('Timeout when trying to connect to repository mirror endpoint')
+      msg = 'Timeout when trying to connect to repository mirror endpoint: %s' % tie.message
+      raise Exception(msg)
+    except requests.exceptions.ConnectionError as ce:
+      logger.exception('Connection error when trying to connect to repository mirror endpoint')
+      msg = 'Connection error when trying to connect to repository mirror endpoint: %s' % ce.message
+      raise Exception(msg)
+    except (requests.exceptions.RequestException, ValueError) as ve:
+      logger.exception('Exception when trying to connect to repository mirror endpoint')
+      msg = 'Exception when trying to connect to repository mirror endpoint: %s' % ve
+      raise Exception(msg)

util/repomirror/skopeomirror.py

@@ -0,0 +1,113 @@
+import os
+import json
+import logging
+import subprocess
+from pipes import quote
+from collections import namedtuple
+
+logger = logging.getLogger(__name__)
+
+
+# success: True or False whether call was successful
+# tags: list of tags or empty list
+# stdout: stdout from skopeo subprocess
+# stderr: stderr from skopeo subprocess
+SkopeoResults = namedtuple("SkopeoCopyResults", "success tags stdout stderr")
+
+
+class SkopeoMirror(object):
+
+  # No DB calls here: This will be called from a separate worker that has no connection except
+  # to/from the mirror worker
+  def copy(self, src_image, dest_image,
+           src_tls_verify=True, dest_tls_verify=True,
+           src_username=None, src_password=None,
+           dest_username=None, dest_password=None,
+           proxy=None, verbose_logs=False):
+
+    args = ["/usr/bin/skopeo"]
+    if verbose_logs:
+      args = args + ["--debug"]
+    args = args + ["copy",
+            "--src-tls-verify=%s" % src_tls_verify,
+            "--dest-tls-verify=%s" % dest_tls_verify
+    ]
+    args = args + self.external_registry_credentials("--dest-creds", dest_username, dest_password)
+    args = args + self.external_registry_credentials("--src-creds", src_username, src_password)
+    args = args + [quote(src_image), quote(dest_image)]
+
+    return self.run_skopeo(args, proxy)
+
+
+  def tags(self, repository, rule_value, username=None, password=None, tls_verify=True, proxy=None, verbose_logs=False):
+    """
+    Unless a specific tag is known, 'skopeo inspect' won't work. Here first 'latest' is checked
+    and then the tag expression, split at commas, is each checked until one works.
+    """
+
+    args = ["/usr/bin/skopeo"]
+    if verbose_logs:
+      args = args + ["--debug"]
+    args = args + ["inspect", "--tls-verify=%s" % tls_verify]
+    args = args + self.external_registry_credentials("--creds", username, password)
+
+    if not rule_value:
+      rule_value = []
+
+    all_tags = []
+    for tag in rule_value + ["latest"]:
+      result = self.run_skopeo(args + [quote("%s:%s" % (repository, tag))], proxy)
+
+      if result.success:
+        all_tags = json.loads(result.stdout)['RepoTags']
+        if all_tags is not []:
+          break
+
+    return SkopeoResults(result.success, all_tags, result.stdout, result.stderr)
+
+
+  def external_registry_credentials(self, arg, username, password):
+    credentials = []
+    if username is not None and username != '':
+      if password is not None and password != '':
+        creds = "%s:%s" % (username, password)
+      else:
+        creds = "%s" % username
+      credentials = [arg, creds]
+
+    return credentials
+
+
+  def setup_env(self, proxy):
+    env = os.environ.copy()
+
+    if proxy.get("http_proxy"):
+      env["HTTP_PROXY"] = proxy.get("http_proxy")
+    if proxy.get("https_proxy"):
+      env["HTTPS_PROXY"] = proxy.get("https_proxy")
+    if proxy.get("no_proxy"):
+      env["NO_PROXY"] = proxy.get("no_proxy")
+
+    return env
+
+
+  def run_skopeo(self, args, proxy):
+    job = subprocess.Popen(args, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                           env=self.setup_env(proxy), close_fds=True)
+
+    # Poll process for new output until finished
+    stdout = ""
+    stderr = ""
+    while True:
+      stdout_nextline = job.stdout.readline()
+      stdout = stdout + stdout_nextline
+      stderr_nextline = job.stderr.readline()
+      stderr = stderr + stderr_nextline
+      if stdout_nextline == "" and stderr_nextline == "" and job.poll() is not None:
+        break
+      logger.debug("Skopeo [STDERR]: %s" % stderr_nextline)
+      logger.debug("Skopeo [STDOUT]: %s" % stdout_nextline)
+
+    job.communicate()
+
+    return SkopeoResults(job.returncode == 0, [], stdout, stderr)

util/repomirror/validator.py

@@ -0,0 +1,16 @@
+import logging
+from util.config.validators import ConfigValidationException
+
+logger = logging.getLogger(__name__)
+
+
+class RepoMirrorConfigValidator(object):
+  """ Helper class for validating the repository mirror configuration. """
+  def __init__(self, feature_repo_mirror):
+    self._feature_repo_mirror = feature_repo_mirror
+
+
+  def valid(self):
+    if not self._feature_repo_mirror:
+      raise ConfigValidationException('REPO_MIRROR feature not enabled')
+    return True