initial import for Open Source 🎉

workers/repomirrorworker/__init__.py

@@ -0,0 +1,280 @@
+import os
+import re
+import traceback
+import fnmatch
+import logging.config
+
+import features
+
+from app import app, prometheus
+from data import database
+from data.model.repo_mirror import claim_mirror, release_mirror
+from data.logs_model import logs_model
+from data.registry_model import registry_model
+from data.database import RepoMirrorStatus
+from data.model.oci.tag import delete_tag, retarget_tag, lookup_alive_tags_shallow
+from notifications import spawn_notification
+from util.audit import wrap_repository
+
+
+from workers.repomirrorworker.repo_mirror_model import repo_mirror_model as model
+
+logger = logging.getLogger(__name__)
+unmirrored_repositories_gauge = prometheus.create_gauge('unmirrored_repositories',
+                                                        'Number of repositories that need to be scanned.')
+
+class PreemptedException(Exception):
+  """ Exception raised if another worker analyzed the image before this worker was able to do so.
+  """
+
+class RepoMirrorSkopeoException(Exception):
+  """ Exception from skopeo
+  """
+  def __init__(self, message, stdout, stderr):
+    self.message = message
+    self.stdout = stdout
+    self.stderr = stderr
+
+
+def process_mirrors(skopeo, token=None):
+  """ Performs mirroring of repositories whose last sync time is greater than sync interval.
+      If a token is provided, scanning will begin where the token indicates it previously completed.
+  """
+
+  if not features.REPO_MIRROR:
+    logger.debug('Repository mirror disabled; skipping RepoMirrorWorker process_mirrors')
+    return None
+
+  iterator, next_token = model.repositories_to_mirror(start_token=token)
+  if iterator is None:
+    logger.debug('Found no additional repositories to mirror')
+    return next_token
+
+  with database.UseThenDisconnect(app.config):
+    for mirror, abt, num_remaining in iterator:
+      try:
+        perform_mirror(skopeo, mirror)
+      except PreemptedException:
+        logger.info('Another repository mirror worker pre-empted us for repository: %s', mirror.id)
+        abt.set()
+      except Exception as e:  # TODO: define exceptions
+        logger.exception('Repository Mirror service unavailable')
+        return None
+
+      unmirrored_repositories_gauge.Set(num_remaining)
+
+  return next_token
+
+
+def perform_mirror(skopeo, mirror):
+  """Run mirror on all matching tags of remote repository."""
+
+  if os.getenv('DEBUGLOG', 'false').lower() == 'true':
+    verbose_logs = True
+  else:
+    verbose_logs = False
+
+  mirror = claim_mirror(mirror)
+  if (mirror == None):
+    raise PreemptedException
+
+  emit_log(mirror, "repo_mirror_sync_started", "start", "'%s' with tag pattern '%s'" % (mirror.external_reference,
+                                                                  ",".join(mirror.root_rule.rule_value)))
+
+  # Fetch the tags to mirror, being careful to handle exceptions. The 'Exception' is safety net only, allowing
+  # easy communication by user through bug report.
+  tags = []
+  try:
+    tags = tags_to_mirror(skopeo, mirror)
+  except RepoMirrorSkopeoException as e:
+    emit_log(mirror, "repo_mirror_sync_failed", "end",
+             "'%s' with tag pattern '%s': %s" % (mirror.external_reference, ",".join(mirror.root_rule.rule_value), e.message),
+             tags=", ".join(tags), stdout=e.stdout, stderr=e.stderr)
+    release_mirror(mirror, RepoMirrorStatus.FAIL)
+    return
+  except Exception as e:
+    emit_log(mirror, "repo_mirror_sync_failed", "end",
+             "'%s' with tag pattern '%s': INTERNAL ERROR" % (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
+             tags=", ".join(tags), stdout="Not applicable", stderr=traceback.format_exc(e))
+    release_mirror(mirror, RepoMirrorStatus.FAIL)
+    return
+  if tags == []:
+    emit_log(mirror, "repo_mirror_sync_success", "end",
+             "'%s' with tag pattern '%s'" % (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
+             tags="No tags matched")
+    release_mirror(mirror, RepoMirrorStatus.SUCCESS)
+    return
+
+  # Sync tags
+  now_ms = database.get_epoch_timestamp_ms()
+  overall_status = RepoMirrorStatus.SUCCESS
+  try:
+    delete_obsolete_tags(mirror, tags)
+
+    username = (mirror.external_registry_username.decrypt()
+                if mirror.external_registry_username else None)
+    password = (mirror.external_registry_password.decrypt()
+                if mirror.external_registry_password else None)
+    dest_server = app.config.get('REPO_MIRROR_SERVER_HOSTNAME', None) or app.config['SERVER_HOSTNAME']
+
+    for tag in tags:
+      src_image = "docker://%s:%s" % (mirror.external_reference, tag)
+      dest_image = "docker://%s/%s/%s:%s" % (dest_server,
+                                             mirror.repository.namespace_user.username,
+                                             mirror.repository.name, tag)
+      with database.CloseForLongOperation(app.config):
+        result = skopeo.copy(src_image, dest_image,
+                    src_tls_verify=mirror.external_registry_config.get('tls_verify', True),
+                    dest_tls_verify=app.config.get('REPO_MIRROR_TLS_VERIFY', True),  # TODO: is this a config choice or something else?
+                    src_username=username,
+                    src_password=password,
+                    dest_username=mirror.internal_robot.username,
+                    dest_password=mirror.internal_robot.email,
+                    proxy=mirror.external_registry_config.get('proxy', {}),
+                    verbose_logs=verbose_logs)
+
+      if not result.success:
+        overall_status = RepoMirrorStatus.FAIL
+        emit_log(mirror, "repo_mirror_sync_tag_failed", "finish", "Source '%s' failed to sync" % src_image,
+                 tag=tag, stdout=result.stdout, stderr=result.stderr)
+        logger.info("Source '%s' failed to sync." % src_image)
+      else:
+        emit_log(mirror, "repo_mirror_sync_tag_success", "finish", "Source '%s' successful sync" % src_image,
+                 tag=tag, stdout=result.stdout, stderr=result.stderr)
+        logger.info("Source '%s' successful sync." % src_image)
+
+      mirror = claim_mirror(mirror)
+      if mirror is None:
+        emit_log(mirror, "repo_mirror_sync_failed", "lost",
+               "'%s' with tag pattern '%s'" % (mirror.external_reference, ",".join(mirror.root_rule.rule_value)))
+  except Exception as e:
+    overall_status = RepoMirrorStatus.FAIL
+    emit_log(mirror, "repo_mirror_sync_failed", "end",
+             "'%s' with tag pattern '%s': INTERNAL ERROR" % (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
+           tags=", ".join(tags), stdout="Not applicable", stderr=traceback.format_exc(e))
+    release_mirror(mirror, overall_status)
+    return
+  finally:
+    if overall_status == RepoMirrorStatus.FAIL:
+      emit_log(mirror, "repo_mirror_sync_failed", "lost",
+             "'%s' with tag pattern '%s'" % (mirror.external_reference, ",".join(mirror.root_rule.rule_value)))
+      rollback(mirror, now_ms)
+    else:
+      emit_log(mirror, "repo_mirror_sync_success", "end",
+               "'%s' with tag pattern '%s'" % (mirror.external_reference, ",".join(mirror.root_rule.rule_value)),
+             tags=", ".join(tags))
+    release_mirror(mirror, overall_status)
+
+  return overall_status
+
+
+def tags_to_mirror(skopeo, mirror):
+  all_tags = get_all_tags(skopeo, mirror)
+  if all_tags == []:
+    return []
+
+  matching_tags = []
+  for pattern in mirror.root_rule.rule_value:
+    matching_tags = matching_tags + filter(lambda tag: fnmatch.fnmatch(tag, pattern), all_tags)
+  matching_tags = list(set(matching_tags))
+  matching_tags.sort()
+  return matching_tags
+
+
+def get_all_tags(skopeo, mirror):
+  verbose_logs = os.getenv('DEBUGLOG', 'false').lower() == 'true'
+
+  username = (mirror.external_registry_username.decrypt()
+              if mirror.external_registry_username else None)
+  password = (mirror.external_registry_password.decrypt()
+              if mirror.external_registry_password else None)
+
+  with database.CloseForLongOperation(app.config):
+    result = skopeo.tags("docker://%s" % (mirror.external_reference),
+                         mirror.root_rule.rule_value,
+                         username=username,
+                         password=password,
+                         verbose_logs=verbose_logs,
+                         tls_verify=mirror.external_registry_config.get('tls_verify', True),
+                         proxy=mirror.external_registry_config.get('proxy', {}))
+
+  if not result.success:
+    raise RepoMirrorSkopeoException("skopeo inspect failed: %s" % _skopeo_inspect_failure(result),
+                                    result.stdout, result.stderr)
+
+  return result.tags
+
+
+def _skopeo_inspect_failure(result):
+  """
+  Custom processing of skopeo error messages for user friendly description
+
+  :param result: SkopeoResults object
+  :return: Message to display
+  """
+
+  lines = result.stderr.split("\n")
+  for line in lines:
+    if re.match('.*Error reading manifest.*', line):
+      return "No matching tags, including 'latest', to inspect for tags list"
+
+  return "See output"
+
+
+def rollback(mirror, since_ms):
+  """
+
+  :param mirror: Mirror to perform rollback on
+  :param start_time: Time mirror was started; all changes after will be undone
+  :return:
+  """
+
+  repository_ref = registry_model.lookup_repository(mirror.repository.namespace_user.username,
+                                                    mirror.repository.name)
+  tags, has_more = registry_model.list_repository_tag_history(repository_ref, 1, 100, since_time_ms=since_ms)
+  for tag in tags:
+    logger.debug("Repo mirroring rollback tag '%s'" % tag)
+
+    # If the tag has an end time, it was either deleted or moved.
+    if tag.lifetime_end_ms:
+      #  If a future entry exists with a start time equal to the end time for this tag,
+      # then the action was a move, rather than a delete and a create.
+      newer_tag = filter(lambda t: tag != t and tag.name == t.name and tag.lifetime_end_ms and
+                          t.lifetime_start_ms == tag.lifetime_end_ms, tags)[0]
+      if (newer_tag):
+        logger.debug("Repo mirroring rollback revert tag '%s'" % tag)
+        retarget_tag(tag.name, tag.manifest._db_id, is_reversion=True)
+      else:
+        logger.debug("Repo mirroring recreate tag '%s'" % tag)
+        retarget_tag(tag.name, tag.manifest._db_id, is_reversion=True)
+
+    #  If the tag has a start time, it was created.
+    elif tag.lifetime_start_ms:
+      logger.debug("Repo mirroring rollback delete tag '%s'" % tag)
+      delete_tag(mirror.repository, tag.name)
+
+
+def delete_obsolete_tags(mirror, tags):
+  existing_tags = lookup_alive_tags_shallow(mirror.repository.id)
+  obsolete_tags = list(filter(lambda tag: tag.name not in tags, existing_tags))
+
+  for tag in obsolete_tags:
+    delete_tag(mirror.repository, tag.name)
+
+  return obsolete_tags
+
+
+# TODO: better to call 'track_and_log()' https://jira.coreos.com/browse/QUAY-1821
+def emit_log(mirror, log_kind, verb, message, tag=None, tags=None, stdout=None, stderr=None):
+  logs_model.log_action(log_kind, namespace_name=mirror.repository.namespace_user.username,
+                        repository_name=mirror.repository.name,
+                        metadata={"verb": verb,
+                                  "namespace": mirror.repository.namespace_user.username,
+                                  "repo": mirror.repository.name,
+                                  "message": message,
+                                  "tag": tag,
+                                  "tags": tags,
+                                  "stdout": stdout, "stderr": stderr})
+
+  if log_kind in ("repo_mirror_sync_started", "repo_mirror_sync_failed", "repo_mirror_sync_success"):
+    spawn_notification(wrap_repository(mirror.repository), log_kind, {'message': message})

workers/repomirrorworker/models_interface.py

@@ -0,0 +1,25 @@
+from abc import ABCMeta, abstractmethod
+from collections import namedtuple
+
+from six import add_metaclass
+
+
+class RepoMirrorToken(namedtuple('NextRepoMirrorToken', ['min_id'])):
+  """
+  RepoMirrorToken represents an opaque token that can be passed between runs of the repository
+  mirror worker to continue mirroring whereever the previous run left off. Note that the data of the
+  token is *opaque* to the repository mirror worker, and the worker should *not* pull any data out
+  or modify the token in any way.
+  """
+
+@add_metaclass(ABCMeta)
+class RepoMirrorWorkerDataInterface(object):
+
+  @abstractmethod
+  def repositories_to_mirror(self, target_time, start_token=None):
+    """
+    Returns a tuple consisting of an iterator of all the candidates to scan and a NextScanToken.
+    The iterator returns a tuple for each iteration consisting of the candidate Repository, the abort
+    signal, and the number of remaining candidates. If the iterator returned is None, there are
+    no candidates to process.
+    """

workers/repomirrorworker/repo_mirror_model.py

@@ -0,0 +1,42 @@
+from math import log10
+
+from data.model.repo_mirror import (get_eligible_mirrors, get_max_id_for_repo_mirror_config,
+                                    get_min_id_for_repo_mirror_config)
+from data.database import RepoMirrorConfig
+from util.migrate.allocator import yield_random_entries
+
+from workers.repomirrorworker.models_interface import (RepoMirrorToken, RepoMirrorWorkerDataInterface)
+
+
+class RepoMirrorModel(RepoMirrorWorkerDataInterface):
+  def repositories_to_mirror(self, start_token=None):
+    def batch_query():
+      return get_eligible_mirrors()
+
+    # Find the minimum ID.
+    if start_token is not None:
+      min_id = start_token.min_id
+    else:
+      min_id = get_min_id_for_repo_mirror_config()
+
+    # Get the ID of the last repository mirror config. Will be None if there are none in the database.
+    max_id = get_max_id_for_repo_mirror_config()
+    if max_id is None:
+      return (None, None)
+
+    if min_id is None or min_id > max_id:
+      return (None, None)
+
+    # 4^log10(total) gives us a scalable batch size into the billions.
+    batch_size = int(4**log10(max(10, max_id - min_id)))
+
+    iterator = yield_random_entries(
+      batch_query,
+      RepoMirrorConfig.id,
+      batch_size,
+      max_id,
+      min_id)
+
+    return (iterator, RepoMirrorToken(max_id + 1))
+
+repo_mirror_model = RepoMirrorModel()

workers/repomirrorworker/repomirrorworker.py

@@ -0,0 +1,60 @@
+import os
+import logging.config
+import time
+import argparse
+
+import features
+
+from app import app, repo_mirror_api
+from workers.worker import Worker
+from workers.repomirrorworker import process_mirrors
+from util.repomirror.validator import RepoMirrorConfigValidator
+from util.repomirror.skopeomirror import SkopeoMirror
+from util.log import logfile_path
+
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MIRROR_INTERVAL = 30
+
+
+class RepoMirrorWorker(Worker):
+  def __init__(self):
+    super(RepoMirrorWorker, self).__init__()
+    RepoMirrorConfigValidator(app.config.get('FEATURE_REPO_MIRROR', False)).valid()
+
+    self._mirrorer = SkopeoMirror()
+    self._next_token = None
+
+    interval = app.config.get('REPO_MIRROR_INTERVAL', DEFAULT_MIRROR_INTERVAL)
+    self.add_operation(self._process_mirrors, interval)
+
+  def _process_mirrors(self):
+    while True:
+      assert app.config.get('FEATURE_REPO_MIRROR', False)
+
+      self._next_token = process_mirrors(self._mirrorer, self._next_token)
+      if self._next_token is None:
+        break
+
+
+if __name__ == '__main__':
+  if os.getenv('PYDEV_DEBUG', None):
+    import pydevd
+    host, port = os.getenv('PYDEV_DEBUG').split(':')
+    pydevd.settrace(host, port=int(port), stdoutToServer=True, stderrToServer=True, suspend=False)
+
+  logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False)
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument('mode', metavar='MODE', type=str, nargs='?', default='',
+                      choices=['mirror', ''])
+  args = parser.parse_args()
+
+  if not features.REPO_MIRROR:
+    logger.debug('Repository mirror disabled; skipping RepoMirrorWorker')
+    while True:
+      time.sleep(100000)
+
+  worker = RepoMirrorWorker()
+  worker.start()

workers/repomirrorworker/test/test_repomirrorworker.py

@@ -0,0 +1,516 @@
+import pytest
+import mock
+import json
+from functools import wraps
+
+from app import storage
+from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
+from data.registry_model import registry_model
+from data.model.test.test_repo_mirroring import create_mirror_repo_robot
+from data.database import Manifest, RepoMirrorConfig, RepoMirrorStatus
+
+from workers.repomirrorworker import delete_obsolete_tags
+from workers.repomirrorworker.repomirrorworker import RepoMirrorWorker
+from io import BytesIO
+from data.model.image import find_create_or_link_image
+from data.model.tag import create_or_update_tag_for_repo
+from util.repomirror.skopeomirror import SkopeoResults, SkopeoMirror
+
+from test.fixtures import *
+
+
+def disable_existing_mirrors(func):
+  @wraps(func)
+  def wrapper(*args, **kwargs):
+    for mirror in RepoMirrorConfig.select():
+      mirror.is_enabled = False
+      mirror.save()
+
+    func(*args, **kwargs)
+
+    for mirror in RepoMirrorConfig.select():
+      mirror.is_enabled = True
+      mirror.save()
+
+  return wrapper
+
+
+def _create_tag(repo, name):
+  repo_ref = registry_model.lookup_repository('mirror', 'repo')
+  with upload_blob(repo_ref, storage, BlobUploadSettings(500, 500, 500)) as upload:
+    app_config = {'TESTING': True}
+    config_json = json.dumps({
+      "config": {
+        "author": u"Repo Mirror",
+      },
+      "rootfs": {"type": "layers", "diff_ids": []},
+      "history": [
+        {
+          "created": "2019-07-30T18:37:09.284840891Z",
+          "created_by": "base",
+          "author": u"Repo Mirror",
+        },
+      ],
+    })
+    upload.upload_chunk(app_config, BytesIO(config_json))
+    blob = upload.commit_to_blob(app_config)
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(blob.digest, blob.compressed_size)
+  builder.add_layer('sha256:abcd', 1234, urls=['http://hello/world'])
+  manifest = builder.build()
+
+  manifest, tag = registry_model.create_manifest_and_retarget_tag(repo_ref, manifest,
+                                                                  name, storage)
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_successful_mirror(run_skopeo_mock, initialized_db, app):
+  """
+  Basic test of successful mirror
+  """
+
+  mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
+
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(True, [], '{"RepoTags": ["latest"]}', "")
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        u"docker://registry.example.com/namespace/repository:latest",
+        u"docker://localhost:5000/mirror/repo:latest"
+      ],
+      "results": SkopeoResults(True, [], "stdout", "stderr")
+    }
+  ]
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+
+  run_skopeo_mock.side_effect = skopeo_test
+
+  worker = RepoMirrorWorker()
+  worker._process_mirrors()
+
+  assert [] == skopeo_calls
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_successful_disabled_sync_now(run_skopeo_mock, initialized_db, app):
+  """
+  Disabled mirrors still allow "sync now"
+  """
+
+  mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
+  mirror.is_enabled = False
+  mirror.sync_status = RepoMirrorStatus.SYNC_NOW
+  mirror.save()
+
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(True, [], '{"RepoTags": ["latest"]}', "")
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        u"docker://registry.example.com/namespace/repository:latest",
+        u"docker://localhost:5000/mirror/repo:latest"
+      ],
+      "results": SkopeoResults(True, [], "stdout", "stderr")
+    }
+  ]
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+
+  run_skopeo_mock.side_effect = skopeo_test
+
+  worker = RepoMirrorWorker()
+  worker._process_mirrors()
+
+  assert [] == skopeo_calls
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_successful_mirror_verbose_logs(run_skopeo_mock, initialized_db, app, monkeypatch):
+  """
+  Basic test of successful mirror with verbose logs turned on
+  """
+
+  mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
+
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "--debug", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(True, [], '{"RepoTags": ["latest"]}', '')
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "--debug", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        u"docker://registry.example.com/namespace/repository:latest",
+        u"docker://localhost:5000/mirror/repo:latest"
+      ],
+      "results": SkopeoResults(True, [], 'Success', '')
+    }
+  ]
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+
+  run_skopeo_mock.side_effect = skopeo_test
+
+  monkeypatch.setenv('DEBUGLOG', 'true')
+  worker = RepoMirrorWorker()
+  worker._process_mirrors()
+
+  assert [] == skopeo_calls
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_rollback(run_skopeo_mock, initialized_db, app):
+  """
+  Tags in the repo:
+  "updated" - this tag will be updated during the mirror
+  "removed" - this tag will be removed during the mirror
+  "created" - this tag will be created during the mirror
+  """
+
+  mirror, repo = create_mirror_repo_robot(["updated", "created", "zzerror"])
+  _create_tag(repo, "updated")
+  _create_tag(repo, "deleted")
+
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:updated"],
+      "results": SkopeoResults(True, [], '{"RepoTags": ["latest", "updated", "created", "zzerror"]}', '')
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        u"docker://registry.example.com/namespace/repository:created",
+        u"docker://localhost:5000/mirror/repo:created"
+      ],
+      "results": SkopeoResults(True, [], 'Success', '')
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        u"docker://registry.example.com/namespace/repository:updated",
+        u"docker://localhost:5000/mirror/repo:updated"
+      ],
+      "results": SkopeoResults(True, [], 'Success', '')
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        u"docker://registry.example.com/namespace/repository:zzerror",
+        u"docker://localhost:5000/mirror/repo:zzerror"
+      ],
+      "results": SkopeoResults(False, [], '', 'ERROR')
+    }
+  ]
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      if args[1] == "copy" and args[6].endswith(":updated"):
+        _create_tag(repo, "updated")
+      elif args[1] == "copy" and args[6].endswith(":created"):
+        _create_tag(repo, "created")
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+
+  run_skopeo_mock.side_effect = skopeo_test
+
+  worker = RepoMirrorWorker()
+  worker._process_mirrors()
+
+  assert [] == skopeo_calls
+  # TODO: how to assert tag.retarget_tag() and tag.delete_tag() called?
+
+
+def test_remove_obsolete_tags(initialized_db):
+  """
+  As part of the mirror, the set of tags on the remote repository is compared to the local
+  existing tags. Those not present on the remote are removed locally.
+  """
+
+  mirror, repository = create_mirror_repo_robot(["updated", "created"], repo_name="removed")
+  manifest = Manifest.get()
+  image = find_create_or_link_image('removed', repository, None, {}, 'local_us')
+  tag = create_or_update_tag_for_repo(repository, 'oldtag', image.docker_image_id,
+                                      oci_manifest=manifest, reversion=True)
+
+  incoming_tags = ["one", "two"]
+  deleted_tags = delete_obsolete_tags(mirror, incoming_tags)
+
+  assert [tag.name for tag in deleted_tags] == [tag.name]
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_mirror_config_server_hostname(run_skopeo_mock, initialized_db, app, monkeypatch):
+  """
+  Set REPO_MIRROR_SERVER_HOSTNAME to override SERVER_HOSTNAME config
+  """
+
+  mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
+
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "--debug", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(True, [], '{"RepoTags": ["latest"]}', '')
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "--debug", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        u"docker://registry.example.com/namespace/repository:latest",
+        u"docker://config_server_hostname/mirror/repo:latest"
+      ],
+      "results": SkopeoResults(True, [], 'Success', '')
+    }
+  ]
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+
+  run_skopeo_mock.side_effect = skopeo_test
+
+  monkeypatch.setenv('DEBUGLOG', 'true')
+  with patch.dict('data.model.config.app_config', {'REPO_MIRROR_SERVER_HOSTNAME': 'config_server_hostname'}):
+    worker = RepoMirrorWorker()
+    worker._process_mirrors()
+
+  assert [] == skopeo_calls
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_quote_params(run_skopeo_mock, initialized_db, app):
+  """
+  Basic test of successful mirror
+  """
+
+  mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
+  mirror.external_reference = "& rm -rf /;/namespace/repository"
+  mirror.external_registry_username = "`rm -rf /`"
+  mirror.save()
+
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", "--creds", u"`rm -rf /`", u"'docker://& rm -rf /;/namespace/repository:latest'"],
+      "results": SkopeoResults(True, [], '{"RepoTags": ["latest"]}', "")
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", "%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        "--src-creds", u"`rm -rf /`",
+        u"'docker://& rm -rf /;/namespace/repository:latest'",
+        u"docker://localhost:5000/mirror/repo:latest"
+      ],
+      "results": SkopeoResults(True, [], "stdout", "stderr")
+    }
+  ]
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+
+  run_skopeo_mock.side_effect = skopeo_test
+
+  worker = RepoMirrorWorker()
+  worker._process_mirrors()
+
+  assert [] == skopeo_calls
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_quote_params_password(run_skopeo_mock, initialized_db, app):
+  """
+  Basic test of successful mirror
+  """
+
+  mirror, repo = create_mirror_repo_robot(["latest", "7.1"])
+  mirror.external_reference = "& rm -rf /;/namespace/repository"
+  mirror.external_registry_username = "`rm -rf /`"
+  mirror.external_registry_password = "\"\"$PATH\\\""
+  mirror.save()
+
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", "--creds", u"`rm -rf /`:\"\"$PATH\\\"", u"'docker://& rm -rf /;/namespace/repository:latest'"],
+      "results": SkopeoResults(True, [], '{"RepoTags": ["latest"]}', "")
+    }, {
+      "args": [
+        "/usr/bin/skopeo", "copy",
+        "--src-tls-verify=True", "--dest-tls-verify=True",
+        "--dest-creds", u"%s:%s" % (mirror.internal_robot.username, mirror.internal_robot.email),
+        "--src-creds", u"`rm -rf /`:\"\"$PATH\\\"",
+        u"'docker://& rm -rf /;/namespace/repository:latest'",
+        u"docker://localhost:5000/mirror/repo:latest"
+      ],
+      "results": SkopeoResults(True, [], "stdout", "stderr")
+    }
+  ]
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+
+  run_skopeo_mock.side_effect = skopeo_test
+
+  worker = RepoMirrorWorker()
+  worker._process_mirrors()
+
+  assert [] == skopeo_calls
+
+
+
+@disable_existing_mirrors
+@mock.patch('util.repomirror.skopeomirror.SkopeoMirror.run_skopeo')
+def test_inspect_error_mirror(run_skopeo_mock, initialized_db, app):
+  """
+  Test for no tag for skopeo inspect. The mirror is processed four times, asserting that the remaining syncs
+  decrement until next sync is bumped to the future, confirming the fourth is never processed.
+  """
+
+  def skopeo_test(args, proxy):
+    try:
+      skopeo_call = skopeo_calls.pop(0)
+      assert args == skopeo_call['args']
+      assert proxy == {}
+
+      return skopeo_call['results']
+    except Exception as e:
+      skopeo_calls.append(skopeo_call)
+      raise e
+  run_skopeo_mock.side_effect = skopeo_test
+  worker = RepoMirrorWorker()
+
+  mirror, repo = create_mirror_repo_robot(["7.1"])
+
+  # Call number 1
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:7.1"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    },
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    }
+  ]
+  worker._process_mirrors()
+  mirror = RepoMirrorConfig.get_by_id(mirror.id)
+  assert [] == skopeo_calls
+  assert 2 == mirror.sync_retries_remaining
+
+  # Call number 2
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:7.1"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    },
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    }
+  ]
+  worker._process_mirrors()
+  mirror = RepoMirrorConfig.get_by_id(mirror.id)
+  assert [] == skopeo_calls
+  assert 1 == mirror.sync_retries_remaining
+
+  # Call number 3
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:7.1"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    },
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    }
+  ]
+  worker._process_mirrors()
+  mirror = RepoMirrorConfig.get_by_id(mirror.id)
+  assert [] == skopeo_calls
+  assert 3 == mirror.sync_retries_remaining
+
+  # Call number 4
+  skopeo_calls = [
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:7.1"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest 7.1 in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    },
+    {
+      "args": ["/usr/bin/skopeo", "inspect", "--tls-verify=True", u"docker://registry.example.com/namespace/repository:latest"],
+      "results": SkopeoResults(False, [], "", 'time="2019-09-18T13:29:40Z" level=fatal msg="Error reading manifest latest in registry.example.com/namespace/repository: manifest unknown: manifest unknown"')
+    }
+  ]
+  worker._process_mirrors()
+  mirror = RepoMirrorConfig.get_by_id(mirror.id)
+  assert 2 == len(skopeo_calls)
+  assert 3 == mirror.sync_retries_remaining