Merge remote-tracking branch 'origin/master' into comewithmeifyouwanttowork

Conflicts:
	data/model/legacy.py

data/archivedlogs.py

@@ -0,0 +1,56 @@
+import logging
+
+from gzip import GzipFile
+from flask import send_file, abort
+from cStringIO import StringIO
+
+from data.userfiles import DelegateUserfiles, UserfilesHandlers
+
+
+JSON_MIMETYPE = 'application/json'
+
+
+logger = logging.getLogger(__name__)
+
+
+class LogArchiveHandlers(UserfilesHandlers):
+  def get(self, file_id):
+    path = self._files.get_file_id_path(file_id)
+    try:
+      with self._storage.stream_read_file(self._locations, path) as gzip_stream:
+        with GzipFile(fileobj=gzip_stream) as unzipped:
+          unzipped_buffer = StringIO(unzipped.read())
+          return send_file(unzipped_buffer, mimetype=JSON_MIMETYPE)
+    except IOError:
+      abort(404)
+
+
+class LogArchive(object):
+  def __init__(self, app=None, distributed_storage=None):
+    self.app = app
+    if app is not None:
+      self.state = self.init_app(app, distributed_storage)
+    else:
+      self.state = None
+
+  def init_app(self, app, distributed_storage):
+    location = app.config.get('LOG_ARCHIVE_LOCATION')
+    path = app.config.get('LOG_ARCHIVE_PATH', None)
+
+    handler_name = 'logarchive_handlers'
+
+    log_archive = DelegateUserfiles(app, distributed_storage, location, path, handler_name)
+
+    app.add_url_rule('/logarchive/<file_id>',
+                     view_func=LogArchiveHandlers.as_view(handler_name,
+                                                          distributed_storage=distributed_storage,
+                                                          location=location,
+                                                          files=log_archive))
+
+    # register extension with app
+    app.extensions = getattr(app, 'extensions', {})
+    app.extensions['log_archive'] = log_archive
+    return log_archive
+
+  def __getattr__(self, name):
+    return getattr(self.state, name, None)

data/billing.py

@@ -3,7 +3,7 @@ import stripe
 from datetime import datetime, timedelta
 from calendar import timegm
 
-from util.collections import AttrDict
+from util.morecollections import AttrDict
 
 PLANS = [
   # Deprecated Plans

data/buildlogs.py

@@ -2,6 +2,11 @@ import redis
 import json
 
 from util.dynamic import import_class
+from datetime import timedelta
+
+
+ONE_DAY = timedelta(days=1)
+
 
 class BuildStatusRetrievalError(Exception):
   pass
@@ -54,6 +59,13 @@ class RedisBuildLogs(object):
     except redis.ConnectionError:
       raise BuildStatusRetrievalError('Cannot retrieve build logs')
 
+  def expire_log_entries(self, build_id):
+    """
+    Sets the log entry to expire in 1 day.
+    """
+    self._redis.expire(self._logs_key(build_id), ONE_DAY)
+
+
   @staticmethod
   def _status_key(build_id):
     return 'builds/%s/status' % build_id

data/database.py

@@ -21,8 +21,24 @@ SCHEME_DRIVERS = {
   'postgresql+psycopg2': PostgresqlDatabase,
 }
 
+SCHEME_RANDOM_FUNCTION = {
+  'mysql': fn.Rand,
+  'mysql+pymysql': fn.Rand,
+  'sqlite': fn.Random,
+  'postgresql': fn.Random,
+  'postgresql+psycopg2': fn.Random,  
+}
+
+class CallableProxy(Proxy):
+  def __call__(self, *args, **kwargs):
+    if self.obj is None:
+      raise AttributeError('Cannot use uninitialized Proxy.')
+    return self.obj(*args, **kwargs)
+
 db = Proxy()
 read_slave = Proxy()
+db_random_func = CallableProxy()
+
 
 def _db_from_url(url, db_kwargs):
   parsed_url = make_url(url)
@@ -38,11 +54,15 @@ def _db_from_url(url, db_kwargs):
 
   return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs)
 
+
 def configure(config_object):
   db_kwargs = dict(config_object['DB_CONNECTION_ARGS'])
   write_db_uri = config_object['DB_URI']
   db.initialize(_db_from_url(write_db_uri, db_kwargs))
 
+  parsed_write_uri = make_url(write_db_uri)
+  db_random_func.initialize(SCHEME_RANDOM_FUNCTION[parsed_write_uri.drivername])
+
   read_slave_uri = config_object.get('DB_READ_SLAVE_URI', None)
   if read_slave_uri is not None:
     read_slave.initialize(_db_from_url(read_slave_uri, db_kwargs))
@@ -298,6 +318,16 @@ class RepositoryTag(BaseModel):
     )
 
 
+class BUILD_PHASE(object):
+  """ Build phases enum """
+  ERROR = 'error'
+  UNPACKING = 'unpacking'
+  PULLING = 'pulling'
+  BUILDING = 'building'
+  PUSHING = 'pushing'
+  COMPLETE = 'complete'
+
+
 class RepositoryBuild(BaseModel):
   uuid = CharField(default=uuid_generator, index=True)
   repository = ForeignKeyField(Repository, index=True)
@@ -309,6 +339,7 @@ class RepositoryBuild(BaseModel):
   display_name = CharField()
   trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
   pull_robot = ForeignKeyField(User, null=True, related_name='buildpullrobot')
+  logs_archived = BooleanField(default=False)
 
 
 class QueueItem(BaseModel):

data/migrations/env.py

@@ -8,7 +8,7 @@ from peewee import SqliteDatabase
 from data.database import all_models, db
 from app import app
 from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
-from util.collections import AttrDict
+from util.morecollections import AttrDict
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

data/migrations/versions/34fd69f63809_add_support_for_build_log_migration.py

@@ -0,0 +1,26 @@
+"""Add support for build log migration.
+
+Revision ID: 34fd69f63809
+Revises: 4a0c94399f38
+Create Date: 2014-09-12 11:50:09.217777
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '34fd69f63809'
+down_revision = '4a0c94399f38'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorybuild', sa.Column('logs_archived', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorybuild', 'logs_archived')
+    ### end Alembic commands ###

data/model/legacy.py

@@ -12,6 +12,7 @@ from util.backoff import exponential_backoff
 
 
 EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1)
+PRESUMED_DEAD_BUILD_AGE = timedelta(days=15)
 
 
 logger = logging.getLogger(__name__)
@@ -1896,6 +1897,16 @@ def get_active_users():
 def get_active_user_count():
   return get_active_users().count()
 
+
+def detach_external_login(user, service_name):
+  try:
+    service = LoginService.get(name = service_name)
+  except LoginService.DoesNotExist:
+    return
+
+  FederatedLogin.delete().where(FederatedLogin.user == user,
+                                FederatedLogin.service == service).execute()
+
 def delete_user(user):
   user.delete_instance(recursive=True, delete_nullable=True)
 
@@ -2006,3 +2017,10 @@ def confirm_team_invite(code, user):
   inviter = found.inviter
   found.delete_instance()
   return (team, inviter)
+
+def archivable_buildlogs_query():
+  presumed_dead_date = datetime.utcnow() - PRESUMED_DEAD_BUILD_AGE
+  return (RepositoryBuild.select()
+    .where((RepositoryBuild.phase == BUILD_PHASE.COMPLETE) |
+           (RepositoryBuild.phase == BUILD_PHASE.ERROR) |
+           (RepositoryBuild.started < presumed_dead_date), RepositoryBuild.logs_archived == False))

data/model/oauth.py

@@ -46,7 +46,7 @@ class DatabaseAuthorizationProvider(AuthorizationProvider):
   def validate_redirect_uri(self, client_id, redirect_uri):
     try:
       app = OAuthApplication.get(client_id=client_id)
-      if app.redirect_uri and redirect_uri.startswith(app.redirect_uri):
+      if app.redirect_uri and redirect_uri and redirect_uri.startswith(app.redirect_uri):
         return True
       return False
     except OAuthApplication.DoesNotExist:

data/userfiles.py

@@ -81,10 +81,13 @@ class DelegateUserfiles(object):
 
     return (url, file_id)
 
-  def store_file(self, file_like_obj, content_type):
-    file_id = str(uuid4())
+  def store_file(self, file_like_obj, content_type, content_encoding=None, file_id=None):
+    if file_id is None:
+      file_id = str(uuid4())
+
     path = self.get_file_id_path(file_id)
-    self._storage.stream_write(self._locations, path, file_like_obj, content_type)
+    self._storage.stream_write(self._locations, path, file_like_obj, content_type,
+                               content_encoding)
     return file_id
 
   def get_file_url(self, file_id, expires_in=300, requires_cors=False):