Add robots.txt and update the sitemap.

Dockerfile

@@ -43,7 +43,6 @@ ADD screenshots screenshots
 ADD static static
 ADD storage storage
 ADD templates templates
-ADD test test
 ADD util util
 ADD workers workers
 
@@ -61,6 +60,8 @@ ADD conf/init/webhookworker.sh /etc/service/webhookworker/run
 RUN cd grunt && npm install
 RUN cd grunt && grunt
 
+# Add the tests last because they're prone to accidental changes, then run them
+ADD test test
 RUN TEST=true venv/bin/python -m unittest discover
 
 RUN rm -rf /conf/stack

endpoints/web.py

@@ -2,7 +2,7 @@ import logging
 import os
 
 from flask import (abort, redirect, request, url_for, make_response, Response,
-                   Blueprint)
+                   Blueprint, send_from_directory)
 from flask.ext.login import current_user
 from urlparse import urlparse
 
@@ -160,6 +160,12 @@ def privacy():
   return render_page_template('privacy.html')
 
 
+@web.route('/robots.txt', methods=['GET'])
+@no_cache
+def robots():
+  return send_from_directory('static', 'robots.txt')
+
+
 @web.route('/receipt', methods=['GET'])
 @route_show_if(features.BILLING)
 @require_session_login
@@ -307,7 +313,8 @@ def request_authorization_code():
       if not current_app:
         abort(404)
 
-      return provider._make_redirect_error_response(current_app.redirect_uri, 'redirect_uri_mismatch')
+      return provider._make_redirect_error_response(current_app.redirect_uri,
+                                                    'redirect_uri_mismatch')
 
     # Load the scope information.
     scope_info = scopes.get_scope_information(scope)
@@ -329,8 +336,9 @@ def request_authorization_code():
 
     # Show the authorization page.
     return render_page_template('oauthorize.html', scopes=scope_info, application=oauth_app_view,
-                                enumerate=enumerate, client_id=client_id, redirect_uri=redirect_uri,
-                                scope=scope, csrf_token_val=generate_csrf_token())
+                                enumerate=enumerate, client_id=client_id,
+                                redirect_uri=redirect_uri, scope=scope,
+                                csrf_token_val=generate_csrf_token())
 
   if response_type == 'token':
     return provider.get_token_response(response_type, client_id, redirect_uri, scope=scope)

static/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /api/

static/sitemap.xml

@@ -25,6 +25,11 @@
       <loc>https://quay.io/about/</loc>
       <changefreq>monthly</changefreq>
    </url>
+   <url>
+      <loc>https://quay.io/security/</loc>
+      <changefreq>monthly</changefreq>
+      <priority>0.4</priority>
+   </url>
    <url>
       <loc>https://quay.io/tos</loc>
       <changefreq>monthly</changefreq>