Merge master into vulnerability-tool

static/css/pages/landing.css

@@ -85,16 +85,13 @@
 }
 
 .landing-content {
-  z-index: 4;
+  z-index: 1;
+  text-align: center;
   padding-left: 20px;
   padding-right: 20px;
   position: relative;
 }
 
-.landing-content {
-  text-align: center;
-}
-
 .landing-content .works-with img {
   max-height: 28px;
   margin-left: 30px;

static/css/pages/superuser.css

@@ -22,4 +22,30 @@
 
 .super-user .user-row.disabled .avatar {
   -webkit-filter: grayscale(100%);
-}
+}
+
+.super-user td .co-alert {
+  margin: 16px 0 0 0;
+}
+
+.super-user .add-storage-link {
+  margin-top: 5px;
+}
+
+.super-user .storage-config {
+  border-bottom: 1px solid #eee;
+  padding: 0 0 10px 0;
+  margin: 10px 0 0 0;
+}
+
+.super-user .last {
+  border-bottom: none;
+}
+
+.super-user .feature-storage-replication {
+  margin: 15px 0 10px 0;
+}
+
+.super-user .input-util {
+  margin-top: 10px;
+}

static/css/quay.css

@@ -1155,7 +1155,7 @@ i.toggle-icon:hover {
 
 form input.ng-invalid.ng-dirty,
 *[ng-form] input.ng-invalid.ng-dirty {
-  background-color: #FDD7D9;
+  background-color: #FDD7D9 !important;
 }
 
 form input.ng-valid.ng-dirty,

static/directives/config/config-setup-tool.html

@@ -194,54 +194,94 @@
             <strong>A remote storage system is required for high-avaliability systems.</strong>
           </p>
 
-          <table class="config-table">
-            <tr>
-              <td class="non-input">Storage Engine:</td>
-              <td>
-                <select ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[0]">
-                  <option value="LocalStorage">Locally mounted directory</option>
-                  <option value="S3Storage">Amazon S3</option>
-                  <option value="GoogleCloudStorage">Google Cloud Storage</option>
-                  <option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
-                  <option value="SwiftStorage">OpenStack Storage (Swift)</option>
-                </select>
-              </td>
-            </tr>
+          <div class="co-checkbox feature-storage-replication">
+            <input id="ftsr" type="checkbox" ng-model="config.FEATURE_STORAGE_REPLICATION">
+            <label for="ftsr">Enable Storage Replication</label>
+            <!-- TODO(josephschorr): add link to documentation -->
+            <div class="help-text">If enabled, replicates storage to other regions.</div>
+          </div>
 
-            <!-- Fields -->
-            <tr ng-repeat="field in STORAGE_CONFIG_FIELDS[config.DISTRIBUTED_STORAGE_CONFIG.local[0]]">
-              <td>{{ field.title }}:</td>
-              <td>
-                <span class="config-map-field"
-                      binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
-                      ng-if="field.kind == 'map'"
-                      keys="field.keys"></span>
-                <span class="config-string-field"
-                      binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
-                      placeholder="{{ field.placeholder }}"
-                      ng-if="field.kind == 'text'"
-                      is-optional="field.optional"></span>
-                <div class="co-checkbox" ng-if="field.kind == 'bool'">
-                  <input id="dsc-{{ field.name }}" type="checkbox"
-                         ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
-                  <label for="dsc-{{ field.name }}">{{ field.placeholder }}</label>
-                </div>
-                <div ng-if="field.kind == 'option'">
-                  <select ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
-                    <option ng-repeat="value in field.values" value="{{ value }}"
-                            ng-selected="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name] == value">{{ value }}</option>
+          <div class="storage-config" ng-class="$last ? 'last' : ''" ng-repeat="sc in storageConfig">
+            <table class="config-table">
+              <tr>
+                <td class="non-input">Location ID:</td>
+                <td>
+                  <input class="form-control" ng-if="allowChangeLocationStorageConfig(sc.location)" ng-class="storageConfigError[$index].location ? 'ng-invalid' : ''" ng-model="sc.location" ng-pattern="/^[a-zA-Z0-9_-]+$/" required>
+                  <div ng-if="!allowChangeLocationStorageConfig(sc.location)">
+                    {{ sc.location }}
+                  </div>
+                  <div class="co-alert co-alert-danger" ng-show="storageConfigError[$index].location">
+                    {{ storageConfigError[$index].location }}
+                  </div>
+                  <div class="input-util" ng-if="allowRemoveStorageConfig(sc.location)"><a href="javascript:void(0)" class="remove-link" ng-click="removeStorageConfig(sc)">Remove</a></div>
+                </td>
+              </tr>
+
+              <tr ng-if="config.FEATURE_STORAGE_REPLICATION">
+                <td class="non-input">Set Default:</td>
+                <td>
+                  <div class="co-checkbox">
+                    <input id="default-location-{{ $index }}" ng-model="sc.defaultLocation" type="checkbox">
+                    <label for="default-location-{{ $index }}">Replicate to storage engine by default</label>
+                  </div>
+                </td>
+              </tr>
+
+              <tr>
+                <td class="non-input">Storage Engine:</td>
+                <td>
+                  <select class="form-control" ng-class="storageConfigError[$index].engine ? 'ng-invalid' : ''" ng-model="sc.data[0]">
+                    <option value="LocalStorage">Locally mounted directory</option>
+                    <option value="S3Storage">Amazon S3</option>
+                    <option value="GoogleCloudStorage">Google Cloud Storage</option>
+                    <option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
+                    <option value="SwiftStorage">OpenStack Storage (Swift)</option>
                   </select>
-                </div>
-                <div class="help-text" ng-if="field.help_text">
-                  {{ field.help_text }}
-                </div>
-                <div class="help-text" ng-if="field.help_url">
-                  See <a href="{{ field.help_url }}" target="_blank">Documentation</a> for more information
-                </div>
-              </td>
-            </tr>
-          </table>
 
+                  <div class="co-alert co-alert-danger" ng-if="storageConfigError[$index].engine">
+                    {{ storageConfigError[$index].engine }}
+                  </div>
+                </td>
+              </tr>
+
+              <!-- Fields -->
+              <tr ng-repeat="field in STORAGE_CONFIG_FIELDS[sc.data[0]]">
+                <td>{{ field.title }}:</td>
+                <td>
+                  <span class="config-map-field"
+                        binding="sc.data[1][field.name]"
+                        ng-if="field.kind == 'map'"
+                        keys="field.keys"></span>
+                  <span class="config-string-field"
+                        binding="sc.data[1][field.name]"
+                        placeholder="{{ field.placeholder }}"
+                        ng-if="field.kind == 'text'"
+                        is-optional="field.optional"></span>
+                  <div class="co-checkbox" ng-if="field.kind == 'bool'">
+                    <input id="dsc-{{ field.name }}" type="checkbox"
+                           ng-model="sc.data[1][field.name]">
+                    <label for="dsc-{{ field.name }}">{{ field.placeholder }}</label>
+                  </div>
+                  <div ng-if="field.kind == 'option'">
+                    <select ng-model="sc.data[1][field.name]">
+                      <option ng-repeat="value in field.values" value="{{ value }}"
+                              ng-selected="sc.data[1][field.name] == value">{{ value }}</option>
+                    </select>
+                  </div>
+                  <div class="help-text" ng-if="field.help_text">
+                    {{ field.help_text }}
+                  </div>
+                  <div class="help-text" ng-if="field.help_url">
+                    See <a href="{{ field.help_url }}" target="_blank">Documentation</a> for more information
+                  </div>
+                </td>
+              </tr>
+            </table>
+          </div>
+
+          <div class="add-storage-link" ng-if="canAddStorageConfig()">
+            <a href="javascript:void(0)" ng-click="addStorageConfig()">Add Additional Storage Engine</a>
+          </div>
         </div>
       </div>
     </div>

static/directives/repo-view/repo-panel-tags.html

@@ -53,7 +53,7 @@
      </span>
 
      <span class="co-filter-box">
-      <span class="page-controls" total-count="tags.length" current-page="options.page" page-size="50"></span>
+      <span class="page-controls" total-count="tags.length" current-page="options.page" page-size="tagsPerPage"></span>
       <input class="form-control" type="text" ng-model="options.tagFilter" placeholder="Filter Tags...">
      </span>
    </div>

static/js/core-config-setup.js

@@ -185,12 +185,17 @@ angular.module("core-config-setup", ['angularFileUpload'])
 
         $scope.checkValidateAndSave = function() {
           if ($scope.configform.$valid) {
+            saveStorageConfig();
             $scope.validateAndSave();
             return;
           }
 
-          $element.find("input.ng-invalid:first")[0].scrollIntoView();
-          $element.find("input.ng-invalid:first").focus();
+          var query = $element.find("input.ng-invalid:first");
+
+          if (query && query.length) {
+            query[0].scrollIntoView();
+            query.focus();
+          }
         };
 
         $scope.validateAndSave = function() {
@@ -277,6 +282,99 @@ angular.module("core-config-setup", ['angularFileUpload'])
           }, ApiService.errorDisplay('Could not save configuration. Please report this error.'));
         };
 
+        // Convert storage config to an array
+        var initializeStorageConfig = function($scope) {
+          var config = $scope.config.DISTRIBUTED_STORAGE_CONFIG || {};
+          var defaultLocations = $scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS || [];
+          var preference = $scope.config.DISTRIBUTED_STORAGE_PREFERENCE || [];
+
+          $scope.serverStorageConfig = angular.copy(config);
+          $scope.storageConfig = [];
+
+          Object.keys(config).forEach(function(location) {
+            $scope.storageConfig.push({
+              location: location,
+              defaultLocation: defaultLocations.indexOf(location) >= 0,
+              data: angular.copy(config[location]),
+              error: {},
+            });
+          });
+
+          if (!$scope.storageConfig.length) {
+            $scope.addStorageConfig('default');
+            return;
+          }
+
+          // match DISTRIBUTED_STORAGE_PREFERENCE order first, remaining are
+          // ordered by unicode point value
+          $scope.storageConfig.sort(function(a, b) {
+            var indexA = preference.indexOf(a.location);
+            var indexB = preference.indexOf(b.location);
+
+            if (indexA > -1 && indexB > -1) return indexA < indexB ? -1 : 1;
+            if (indexA > -1) return -1;
+            if (indexB > -1) return 1;
+
+            return a.location < b.location ? -1 : 1;
+          });
+        };
+
+        $scope.allowChangeLocationStorageConfig = function(location) {
+          if (!$scope.serverStorageConfig[location]) { return true };
+
+          // allow user to change location ID if another exists with the same ID
+          return $scope.storageConfig.filter(function(sc) {
+            return sc.location === location;
+          }).length >= 2;
+        };
+
+        $scope.allowRemoveStorageConfig = function(location) {
+          return $scope.storageConfig.length > 1 && $scope.allowChangeLocationStorageConfig(location);
+        };
+
+        $scope.canAddStorageConfig = function() {
+          return $scope.config &&
+            $scope.config.FEATURE_STORAGE_REPLICATION &&
+            $scope.storageConfig &&
+            (!$scope.storageConfig.length || $scope.storageConfig.length < 10);
+        };
+
+        $scope.addStorageConfig = function(location) {
+          var storageType = 'LocalStorage';
+
+          // Use last storage type by default
+          if ($scope.storageConfig.length) {
+            storageType = $scope.storageConfig[$scope.storageConfig.length-1].data[0];
+          }
+
+          $scope.storageConfig.push({
+            location: location || '',
+            defaultLocation: false,
+            data: [storageType, {}],
+            error: {},
+          });
+        };
+
+        $scope.removeStorageConfig = function(sc) {
+          $scope.storageConfig.splice($scope.storageConfig.indexOf(sc), 1);
+        };
+
+        var saveStorageConfig = function() {
+          var config = {};
+          var defaultLocations = [];
+          var preference = [];
+
+          $scope.storageConfig.forEach(function(sc) {
+            config[sc.location] = sc.data;
+            if (sc.defaultLocation) defaultLocations.push(sc.location);
+            preference.push(sc.location);
+          });
+
+          $scope.config.DISTRIBUTED_STORAGE_CONFIG = config;
+          $scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS = defaultLocations;
+          $scope.config.DISTRIBUTED_STORAGE_PREFERENCE = preference;
+        };
+
         var gitlabSelector = function(key) {
           return function(value) {
             if (!value || !$scope.config) { return; }
@@ -378,18 +476,11 @@ angular.module("core-config-setup", ['angularFileUpload'])
         $scope.$watch('mapped.redis.port', redisSetter('port'));
         $scope.$watch('mapped.redis.password', redisSetter('password'));
 
-        // Add a watch to remove any fields not allowed by the current storage configuration.
-        // We have to do this otherwise extra fields (which are not allowed) can end up in the
-        // configuration.
-        $scope.$watch('config.DISTRIBUTED_STORAGE_CONFIG.local[0]', function(value) {
-          // Remove any fields not associated with the current kind.
-          if (!value || !$scope.STORAGE_CONFIG_FIELDS[value]
-              || !$scope.config.DISTRIBUTED_STORAGE_CONFIG
-              || !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local
-              || !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1]) { return; }
-
-          var allowedFields = $scope.STORAGE_CONFIG_FIELDS[value];
-          var configObject = $scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1];
+        // Remove extra extra fields (which are not allowed) from storage config.
+        var updateFields = function(sc) {
+          var type = sc.data[0];
+          var configObject = sc.data[1];
+          var allowedFields = $scope.STORAGE_CONFIG_FIELDS[type];
 
           // Remove any fields not allowed.
           for (var fieldName in configObject) {
@@ -412,8 +503,53 @@ angular.module("core-config-setup", ['angularFileUpload'])
               configObject[allowedFields[i].name] = configObject[allowedFields[i].name] || false;
             }
           }
+        };
+
+        // Validate and update storage config on update.
+        var refreshStorageConfig = function() {
+          if (!$scope.config || !$scope.storageConfig) return;
+
+          var locationCounts = {};
+          var errors = [];
+          var valid = true;
+
+          $scope.storageConfig.forEach(function(sc) {
+            // remove extra fields from storage config
+            updateFields(sc);
+
+            if (!locationCounts[sc.location]) locationCounts[sc.location] = 0;
+            locationCounts[sc.location]++;
+          });
+
+          // validate storage config
+          $scope.storageConfig.forEach(function(sc) {
+            var error = {};
+
+            if ($scope.config.FEATURE_STORAGE_REPLICATION && sc.data[0] === 'LocalStorage') {
+              error.engine = 'Replication to a locally mounted directory is unsupported as it is only accessible on a single machine.';
+              valid = false;
+            }
+
+            if (locationCounts[sc.location] > 1) {
+              error.location = 'Location ID must be unique.';
+              valid = false;
+            }
+
+            errors.push(error);
+          });
+
+          $scope.storageConfigError = errors;
+          $scope.configform.$setValidity('storageConfig', valid);
+        };
+
+        $scope.$watch('config.FEATURE_STORAGE_REPLICATION', function() {
+          refreshStorageConfig();
         });
 
+        $scope.$watch('storageConfig', function() {
+          refreshStorageConfig();
+        }, true);
+
         $scope.$watch('config', function(value) {
           $scope.mapped['$hasChanges'] = true;
         }, true);
@@ -424,6 +560,7 @@ angular.module("core-config-setup", ['angularFileUpload'])
           ApiService.scGetConfig().then(function(resp) {
             $scope.config = resp['config'] || {};
             initializeMappedLogic($scope.config);
+            initializeStorageConfig($scope);
             $scope.mapped['$hasChanges'] = false;
           }, ApiService.errorDisplay('Could not load config'));
         });
@@ -919,4 +1056,4 @@ angular.module("core-config-setup", ['angularFileUpload'])
       }
     };
     return directiveDefinitionObject;
-  });
+  });

static/js/directives/repo-view/repo-panel-tags.js

@@ -232,6 +232,7 @@ angular.module('quay').directive('repoPanelTags', function () {
       $scope.trackLineClass = function(index, track_info) {
         var startIndex = $.inArray(track_info.tags[0], $scope.tags);
         var endIndex = $.inArray(track_info.tags[track_info.tags.length - 1], $scope.tags);
+        index += $scope.options.page * $scope.tagsPerPage;
 
         if (index == startIndex) {
           return 'start';

static/js/services/datafile-service.js

@@ -55,6 +55,11 @@ angular.module('quay').factory('DataFileService', [function() {
       return;
     }
 
+    if (plain.byteLength == 0) {
+      failure();
+      return;
+    }
+
     dataFileService.tryAsTar_(plain, success, failure);
   };