diff --git a/conf/init/copy_config_files.sh b/conf/init/copy_config_files.sh index 8849a97cf..ceeebdb37 100755 --- a/conf/init/copy_config_files.sh +++ b/conf/init/copy_config_files.sh @@ -2,7 +2,7 @@ if [ -e /conf/stack/robots.txt ] then - cp /conf/stack/robots.txt /static/robots.txt + cp /conf/stack/robots.txt /templates/robots.txt fi if [ -e /conf/stack/favicon.ico ] diff --git a/data/model/repository.py b/data/model/repository.py index 9ce184381..a5e0b2e89 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -405,3 +405,15 @@ def confirm_email_authorization_for_repo(code): return found +def list_popular_public_repos(action_count_threshold, time_span): + cutoff = datetime.now() - time_span + return (Repository + .select(Namespace.username, Repository.name) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(RepositoryActionCount) + .where(RepositoryActionCount.date >= cutoff, + Repository.visibility == get_public_repo_visibility()) + .group_by(RepositoryActionCount.repository) + .having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold) + .tuples()) diff --git a/endpoints/web.py b/endpoints/web.py index 952d77b57..bb8c1a0a2 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -2,15 +2,17 @@ import json import logging from urlparse import urlparse +from datetime import timedelta from cachetools import lru_cache -from flask import (abort, redirect, request, url_for, make_response, Response, +from flask import (abort, redirect, request, url_for, make_response, Response, render_template, Blueprint, send_from_directory, jsonify, send_file) from flask.ext.login import current_user import features -from app import app, billing as stripe, build_logs, avatar, signer, log_archive, config_provider +from app import (app, billing as stripe, build_logs, avatar, signer, log_archive, config_provider, + get_app_url) from auth import scopes from auth.auth import require_session_login, process_oauth, has_basic_auth, process_auth_or_cookie from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission, @@ -282,9 +284,19 @@ def disclaimer(): @web.route('/robots.txt', methods=['GET']) -@no_cache def robots(): - return send_from_directory('static', 'robots.txt') + robots_txt = make_response(render_template('robots.txt', baseurl=get_app_url())) + robots_txt.headers['Content-Type'] = 'text/plain' + return robots_txt + + +@web.route('/sitemap.xml', methods=['GET']) +def sitemap(): + popular_repo_tuples = model.repository.list_popular_public_repos(50, timedelta(weeks=1)) + xml = make_response(render_template('sitemap.xml', public_repos=popular_repo_tuples, + baseurl=get_app_url())) + xml.headers['Content-Type'] = 'application/xml' + return xml @web.route('/buildlogs/', methods=['GET']) diff --git a/initdb.py b/initdb.py index e8006e536..0e6fab37a 100644 --- a/initdb.py +++ b/initdb.py @@ -6,7 +6,7 @@ import calendar import os import argparse -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, savepoint_sqlite, savepoint) from itertools import count @@ -506,9 +506,9 @@ def populate_database(minimal=False, with_storage=False): (1, [(1, [], 'v5.0'), (1, [], 'v6.0')], None)], None)) - __generate_repository(with_storage, new_user_2, 'publicrepo', - 'Public repository pullable by the world.', True, - [], (10, [], 'latest')) + publicrepo = __generate_repository(with_storage, new_user_2, 'publicrepo', + 'Public repository pullable by the world.', True, + [], (10, [], 'latest')) __generate_repository(with_storage, outside_org, 'coolrepo', 'Some cool repo.', False, @@ -653,6 +653,7 @@ def populate_database(minimal=False, with_storage=False): week_ago = today - timedelta(6) six_ago = today - timedelta(5) four_ago = today - timedelta(4) + yesterday = datetime.combine(date.today(), datetime.min.time()) - timedelta(hours=6) __generate_service_key('kid1', 'somesamplekey', new_user_1, today, ServiceKeyApprovalType.SUPERUSER) @@ -740,6 +741,10 @@ def populate_database(minimal=False, with_storage=False): timestamp=today, metadata={'token_code': 'somecode', 'repo': 'orgrepo'}) + model.log.log_action('pull_repo', new_user_2.username, repository=publicrepo, + timestamp=yesterday, + metadata={'token_code': 'somecode', 'repo': 'publicrepo'}) + model.log.log_action('build_dockerfile', new_user_1.username, repository=building, timestamp=today, metadata={'repo': 'building', 'namespace': new_user_1.username, diff --git a/static/sitemap.xml b/static/sitemap.xml deleted file mode 100644 index edd8399ab..000000000 --- a/static/sitemap.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - https://quay.io/ - hourly - 0.8 - - - https://quay.io/plans/ - monthly - - - https://quay.io/organizations/ - weekly - - - https://quay.io/repository/ - always - - - https://quay.io/contact/ - monthly - - - https://quay.io/about/ - monthly - - - https://quay.io/security/ - monthly - 0.4 - - - https://quay.io/tos - monthly - 0.4 - - - https://quay.io/privacy - monthly - 0.4 - - diff --git a/static/robots.txt b/templates/robots.txt similarity index 100% rename from static/robots.txt rename to templates/robots.txt diff --git a/templates/sitemap.xml b/templates/sitemap.xml new file mode 100644 index 000000000..ca5061a13 --- /dev/null +++ b/templates/sitemap.xml @@ -0,0 +1,58 @@ + + + + {{ baseurl }}/ + hourly + 0.8 + + + {{ baseurl }}/plans/ + monthly + + + {{ baseurl }}/tour/ + weekly + + + {{ baseurl }}/tour/organizations + weekly + + + {{ baseurl }}/tour/features + weekly + + + {{ baseurl }}/tour/enterprise + weekly + + + {{ baseurl }}/contact/ + monthly + + + {{ baseurl }}/about/ + monthly + + + {{ baseurl }}/security/ + monthly + 0.4 + + + {{ baseurl }}/tos + monthly + 0.4 + + + {{ baseurl }}/privacy + monthly + 0.4 + + {% for namespace, reponame in public_repos -%} + + {{ baseurl }}/repository/{{ namespace }}/{{ reponame }} + daily + 0.3 + + {%- endfor %} + diff --git a/test/data/test.db b/test/data/test.db index 5e87c469f..6ceefdff8 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_endpoints.py b/test/test_endpoints.py index 461ec8ad2..d8430375b 100644 --- a/test/test_endpoints.py +++ b/test/test_endpoints.py @@ -111,6 +111,12 @@ class WebEndpointTestCase(EndpointTestCase): def test_index(self): self.getResponse('web.index') + def test_robots(self): + self.getResponse('web.robots') + + def test_sitemap(self): + self.getResponse('web.sitemap') + def test_repo_view(self): self.getResponse('web.repository', path='devtable/simple') diff --git a/test/test_repomodel.py b/test/test_repomodel.py new file mode 100644 index 000000000..419bfc644 --- /dev/null +++ b/test/test_repomodel.py @@ -0,0 +1,37 @@ +import unittest +from datetime import timedelta + +from app import app +from initdb import setup_database_for_testing, finished_database_for_testing +from data import model + + +PUBLIC_USERNAME = 'public' +PUBLIC_REPONAME = 'publicrepo' + + +class TestRepoModel(unittest.TestCase): + def setUp(self): + setup_database_for_testing(self) + + self.app = app.test_client() + self.ctx = app.test_request_context() + self.ctx.__enter__() + + def tearDown(self): + finished_database_for_testing(self) + self.ctx.__exit__(True, None, None) + + def test_popular_repo_list(self): + # Our repository action count table should have 1 event for the only public + # repo. + onlypublic = model.repository.list_popular_public_repos(0, timedelta(weeks=1)) + self.assertEquals(len(onlypublic), 1) + self.assertEquals(onlypublic[0], (PUBLIC_USERNAME, PUBLIC_REPONAME)) + + self.assertEquals(len(model.repository.list_popular_public_repos(1, timedelta(weeks=1))), 1) + self.assertEquals(len(model.repository.list_popular_public_repos(50, timedelta(weeks=1))), 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/workers/repositoryactioncounter.py b/workers/repositoryactioncounter.py index 34c2bf0b0..d1e951059 100644 --- a/workers/repositoryactioncounter.py +++ b/workers/repositoryactioncounter.py @@ -10,6 +10,10 @@ POLL_PERIOD_SECONDS = 10 logger = logging.getLogger(__name__) def count_repository_actions(): + """ Aggregates repository actions from the LogEntry table and writes them to + the RepositoryActionCount table. Returns the number of repositories for + which actions were logged. Returns 0 when there is no more work. + """ try: # Get a random repository to count. today = date.today() @@ -32,11 +36,14 @@ def count_repository_actions(): # Create the row. try: RepositoryActionCount.create(repository=to_count, date=yesterday, count=actions) + return 1 except: logger.exception('Exception when writing count') except Repository.DoesNotExist: logger.debug('No further repositories to count') + return 0 + class RepositoryActionCountWorker(Worker): def __init__(self):