Add trending links (#16917)

* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests
This commit is contained in:
Eugen Rochko 2021-11-25 13:07:38 +01:00 committed by GitHub
parent 46e62fc4b3
commit 6e50134a42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
97 changed files with 2071 additions and 722 deletions

View file

@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index
end
field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } }
field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
end
end

View file

@ -4,7 +4,7 @@ module Admin
class DashboardController < BaseController
def index
@system_checks = Admin::SystemCheck.perform
@time_period = (1.month.ago.to_date...Time.now.utc.to_date)
@time_period = (29.days.ago.to_date...Time.now.utc.to_date)
@pending_users_count = User.pending.count
@pending_reports_count = Report.unresolved.count
@pending_tags_count = Tag.pending_review.count

View file

@ -2,38 +2,12 @@
module Admin
class TagsController < BaseController
before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all]
before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]
def index
authorize :tag, :index?
@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end
def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_tags_path(filter_params)
end
def approve_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save
redirect_to admin_tags_path(filter_params)
end
def reject_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save
redirect_to admin_tags_path(filter_params)
end
before_action :set_tag
def show
authorize @tag, :show?
@time_period = (6.days.ago.to_date...Time.now.utc.to_date)
end
def update
@ -52,52 +26,8 @@ module Admin
@tag = Tag.find(params[:id])
end
def set_usage_by_domain
@usage_by_domain = @tag.statuses
.with_public_visibility
.excluding_silenced_accounts
.where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day)))
.joins(:account)
.group('accounts.domain')
.reorder(statuses_count: :desc)
.pluck(Arel.sql('accounts.domain, count(*) AS statuses_count'))
end
def set_counters
@accounts_today = @tag.history.first[:accounts]
@accounts_week = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" })
end
def filtered_tags
TagFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end
def tag_params
params.require(:tag).permit(:name, :trendable, :usable, :listable)
end
def current_week_days
now = Time.now.utc.beginning_of_day.to_date
(Date.commercial(now.cwyear, now.cweek)..now).map do |date|
date.to_time(:utc).beginning_of_day.to_i
end
end
def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end
end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController
def index
authorize :preview_card_provider, :index?
@preview_card_providers = filtered_preview_card_providers.page(params[:page])
@form = Form::PreviewCardProviderBatch.new
end
def batch
@form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_preview_card_providers_path(filter_params)
end
private
def filtered_preview_card_providers
PreviewCardProviderFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
end
def form_preview_card_provider_batch_params
params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end

View file

@ -0,0 +1,45 @@
# frozen_string_literal: true
class Admin::Trends::LinksController < Admin::BaseController
def index
authorize :preview_card, :index?
@preview_cards = filtered_preview_cards.page(params[:page])
@form = Form::PreviewCardBatch.new
end
def batch
@form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_path(filter_params)
end
private
def filtered_preview_cards
PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
end
def filter_params
params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
end
def form_preview_card_batch_params
params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:approve_all]
'approve_all'
elsif params[:reject]
'reject'
elsif params[:reject_all]
'reject_all'
end
end
end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Trends::TagsController < Admin::BaseController
def index
authorize :tag, :index?
@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end
def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_tags_path(filter_params)
end
private
def filtered_tags
TagFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end
def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end

View file

@ -17,7 +17,8 @@ class Api::V1::Admin::DimensionsController < Api::BaseController
params[:keys],
params[:start_at],
params[:end_at],
params[:limit]
params[:limit],
params
)
end
end

View file

@ -16,7 +16,8 @@ class Api::V1::Admin::MeasuresController < Api::BaseController
@measures = Admin::Metrics::Measure.retrieve(
params[:keys],
params[:start_at],
params[:end_at]
params[:end_at],
params
)
end
end

View file

@ -0,0 +1,16 @@
# frozen_string_literal: true
class Api::V1::Admin::Trends::TagsController < Api::BaseController
before_action :require_staff!
before_action :set_tags
def index
render json: @tags, each_serializer: REST::Admin::TagSerializer
end
private
def set_tags
@tags = Trends.tags.get(false, limit_param(10))
end
end

View file

@ -1,16 +0,0 @@
# frozen_string_literal: true
class Api::V1::Admin::TrendsController < Api::BaseController
before_action :require_staff!
before_action :set_trends
def index
render json: @trends, each_serializer: REST::Admin::TagSerializer
end
private
def set_trends
@trends = TrendingTags.get(10, filtered: false)
end
end

View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
class Api::V1::Trends::LinksController < Api::BaseController
before_action :set_links
def index
render json: @links, each_serializer: REST::Trends::LinkSerializer
end
private
def set_links
@links = begin
if Setting.trends
Trends.links.get(true, limit_param(10))
else
[]
end
end
end
end

View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
class Api::V1::Trends::TagsController < Api::BaseController
before_action :set_tags
def index
render json: @tags, each_serializer: REST::TagSerializer
end
private
def set_tags
@tags = begin
if Setting.trends
Trends.tags.get(true, limit_param(10))
else
[]
end
end
end
end

View file

@ -1,15 +0,0 @@
# frozen_string_literal: true
class Api::V1::TrendsController < Api::BaseController
before_action :set_tags
def index
render json: @tags, each_serializer: REST::TagSerializer
end
private
def set_tags
@tags = TrendingTags.get(limit_param(10))
end
end

View file

@ -6,6 +6,8 @@ module Admin::FilterHelper
CustomEmojiFilter::KEYS,
ReportFilter::KEYS,
TagFilter::KEYS,
PreviewCardProviderFilter::KEYS,
PreviewCardFilter::KEYS,
InstanceFilter::KEYS,
InviteFilter::KEYS,
RelationshipFilter::KEYS,

View file

@ -0,0 +1,94 @@
# frozen_string_literal: true
module LanguagesHelper
HUMAN_LOCALES = {
af: 'Afrikaans',
ar: 'العربية',
ast: 'Asturianu',
bg: 'Български',
bn: 'বাংলা',
br: 'Breton',
ca: 'Català',
co: 'Corsu',
cs: 'Čeština',
cy: 'Cymraeg',
da: 'Dansk',
de: 'Deutsch',
el: 'Ελληνικά',
en: 'English',
eo: 'Esperanto',
'es-AR': 'Español (Argentina)',
'es-MX': 'Español (México)',
es: 'Español',
et: 'Eesti',
eu: 'Euskara',
fa: 'فارسی',
fi: 'Suomi',
fr: 'Français',
ga: 'Gaeilge',
gd: 'Gàidhlig',
gl: 'Galego',
he: 'עברית',
hi: 'हिन्दी',
hr: 'Hrvatski',
hu: 'Magyar',
hy: 'Հայերեն',
id: 'Bahasa Indonesia',
io: 'Ido',
is: 'Íslenska',
it: 'Italiano',
ja: '日本語',
ka: 'ქართული',
kab: 'Taqbaylit',
kk: 'Қазақша',
kmr: 'Kurmancî',
kn: 'ಕನ್ನಡ',
ko: '한국어',
ku: 'سۆرانی',
lt: 'Lietuvių',
lv: 'Latviešu',
mk: 'Македонски',
ml: 'മലയാളം',
mr: 'मराठी',
ms: 'Bahasa Melayu',
nl: 'Nederlands',
nn: 'Nynorsk',
no: 'Norsk',
oc: 'Occitan',
pl: 'Polski',
'pt-BR': 'Português (Brasil)',
'pt-PT': 'Português (Portugal)',
pt: 'Português',
ro: 'Română',
ru: 'Русский',
sa: 'संस्कृतम्',
sc: 'Sardu',
si: 'සිංහල',
sk: 'Slovenčina',
sl: 'Slovenščina',
sq: 'Shqip',
'sr-Latn': 'Srpski (latinica)',
sr: 'Српски',
sv: 'Svenska',
ta: 'தமிழ்',
te: 'తెలుగు',
th: 'ไทย',
tr: 'Türkçe',
uk: 'Українська',
ur: 'اُردُو',
vi: 'Tiếng Việt',
zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
'zh-CN': '简体中文',
'zh-HK': '繁體中文(香港)',
'zh-TW': '繁體中文(臺灣)',
zh: '中文',
}.freeze
def human_locale(locale)
if locale == 'und'
I18n.t('generic.none')
else
HUMAN_LOCALES[locale.to_sym] || locale
end
end
end

View file

@ -1,95 +1,8 @@
# frozen_string_literal: true
module SettingsHelper
HUMAN_LOCALES = {
af: 'Afrikaans',
ar: 'العربية',
ast: 'Asturianu',
bg: 'Български',
bn: 'বাংলা',
br: 'Breton',
ca: 'Català',
co: 'Corsu',
cs: 'Čeština',
cy: 'Cymraeg',
da: 'Dansk',
de: 'Deutsch',
el: 'Ελληνικά',
en: 'English',
eo: 'Esperanto',
'es-AR': 'Español (Argentina)',
'es-MX': 'Español (México)',
es: 'Español',
et: 'Eesti',
eu: 'Euskara',
fa: 'فارسی',
fi: 'Suomi',
fr: 'Français',
ga: 'Gaeilge',
gd: 'Gàidhlig',
gl: 'Galego',
he: 'עברית',
hi: 'हिन्दी',
hr: 'Hrvatski',
hu: 'Magyar',
hy: 'Հայերեն',
id: 'Bahasa Indonesia',
io: 'Ido',
is: 'Íslenska',
it: 'Italiano',
ja: '日本語',
ka: 'ქართული',
kab: 'Taqbaylit',
kk: 'Қазақша',
kmr: 'Kurmancî',
kn: 'ಕನ್ನಡ',
ko: '한국어',
ku: 'سۆرانی',
lt: 'Lietuvių',
lv: 'Latviešu',
mk: 'Македонски',
ml: 'മലയാളം',
mr: 'मराठी',
ms: 'Bahasa Melayu',
nl: 'Nederlands',
nn: 'Nynorsk',
no: 'Norsk',
oc: 'Occitan',
pl: 'Polski',
'pt-BR': 'Português (Brasil)',
'pt-PT': 'Português (Portugal)',
pt: 'Português',
ro: 'Română',
ru: 'Русский',
sa: 'संस्कृतम्',
sc: 'Sardu',
si: 'සිංහල',
sk: 'Slovenčina',
sl: 'Slovenščina',
sq: 'Shqip',
'sr-Latn': 'Srpski (latinica)',
sr: 'Српски',
sv: 'Svenska',
ta: 'தமிழ்',
te: 'తెలుగు',
th: 'ไทย',
tr: 'Türkçe',
uk: 'Українська',
ur: 'اُردُو',
vi: 'Tiếng Việt',
zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
'zh-CN': '简体中文',
'zh-HK': '繁體中文(香港)',
'zh-TW': '繁體中文(臺灣)',
zh: '中文',
}.freeze
def human_locale(locale)
HUMAN_LOCALES[locale]
end
def filterable_languages
LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?))
LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?))
end
def hash_to_object(hash)

View file

@ -32,6 +32,7 @@ export default class Counter extends React.PureComponent {
end_at: PropTypes.string.isRequired,
label: PropTypes.string.isRequired,
href: PropTypes.string,
params: PropTypes.object,
};
state = {
@ -40,9 +41,9 @@ export default class Counter extends React.PureComponent {
};
componentDidMount () {
const { measure, start_at, end_at } = this.props;
const { measure, start_at, end_at, params } = this.props;
api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at }).then(res => {
api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at, [measure]: params }).then(res => {
this.setState({
loading: false,
data: res.data,

View file

@ -13,6 +13,7 @@ export default class Dimension extends React.PureComponent {
end_at: PropTypes.string.isRequired,
limit: PropTypes.number.isRequired,
label: PropTypes.string.isRequired,
params: PropTypes.object,
};
state = {
@ -21,9 +22,9 @@ export default class Dimension extends React.PureComponent {
};
componentDidMount () {
const { start_at, end_at, dimension, limit } = this.props;
const { start_at, end_at, dimension, limit, params } = this.props;
api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit }).then(res => {
api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit, [dimension]: params }).then(res => {
this.setState({
loading: false,
data: res.data,

View file

@ -19,7 +19,7 @@ export default class Trends extends React.PureComponent {
componentDidMount () {
const { limit } = this.props;
api().get('/api/v1/admin/trends', { params: { limit } }).then(res => {
api().get('/api/v1/admin/trends/tags', { params: { limit } }).then(res => {
this.setState({
loading: false,
data: res.data,

View file

@ -325,3 +325,19 @@
margin-top: 10px;
}
}
.batch-table__row--muted .pending-account__header {
&,
a,
strong {
color: lighten($ui-base-color, 26%);
}
}
.batch-table__row--attention .pending-account__header {
&,
a,
strong {
color: $gold-star;
}
}

View file

@ -100,6 +100,16 @@
transition: all 200ms ease-out;
}
&.positive {
background: lighten($ui-base-color, 4%);
color: $valid-value-color;
}
&.negative {
background: lighten($ui-base-color, 4%);
color: $error-value-color;
}
span {
flex: 1 1 auto;
}

View file

@ -129,8 +129,6 @@ class ActivityPub::Activity
end
def crawl_links(status)
return if status.spoiler_text?
# Spread out crawling randomly to avoid DDoSing the link
LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
end

View file

@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
visibility: visibility_from_audience
)
original_status.tags.each do |tag|
tag.use!(@account)
end
Trends.tags.register(@status)
Trends.links.register(@status)
distribute(@status)
end

View file

@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def attach_tags(status)
@tags.each do |tag|
status.tags << tag
tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
end
# If we're processing an old status, this may register tags as being used now
# as opposed to when the status was really published, but this is probably
# not a big deal
Trends.tags.register(status)
@mentions.each do |mention|
mention.status = status
mention.save

View file

@ -7,9 +7,14 @@ class Admin::Metrics::Dimension
servers: Admin::Metrics::Dimension::ServersDimension,
space_usage: Admin::Metrics::Dimension::SpaceUsageDimension,
software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension,
tag_servers: Admin::Metrics::Dimension::TagServersDimension,
tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension,
}.freeze
def self.retrieve(dimension_keys, start_at, end_at, limit)
Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact
def self.retrieve(dimension_keys, start_at, end_at, limit, params)
Array(dimension_keys).map do |key|
klass = DIMENSIONS[key.to_sym]
klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end
end

View file

@ -1,10 +1,15 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::BaseDimension
def initialize(start_at, end_at, limit)
def self.with_params?
false
end
def initialize(start_at, end_at, limit, params)
@start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime
@limit = limit&.to_i
@params = params
end
def key
@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension
protected
def time_period
(@start_at...@end_at)
(@start_at..@end_at)
end
def params
raise NotImplementedError
end
end

View file

@ -1,6 +1,8 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def key
'languages'
end
@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } }
rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
end
end

View file

@ -0,0 +1,36 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def self.with_params?
true
end
def key
'tag_languages'
end
def data
sql = <<-SQL.squish
SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
FROM statuses
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY COALESCE(statuses.language, 'und')
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

View file

@ -0,0 +1,35 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension
def self.with_params?
true
end
def key
'tag_servers'
end
def data
sql = <<-SQL.squish
SELECT accounts.domain, count(*) AS value
FROM statuses
INNER JOIN accounts ON accounts.id = statuses.account_id
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY accounts.domain
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

View file

@ -7,9 +7,15 @@ class Admin::Metrics::Measure
interactions: Admin::Metrics::Measure::InteractionsMeasure,
opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure,
resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure,
tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure,
tag_uses: Admin::Metrics::Measure::TagUsesMeasure,
tag_servers: Admin::Metrics::Measure::TagServersMeasure,
}.freeze
def self.retrieve(measure_keys, start_at, end_at)
Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact
def self.retrieve(measure_keys, start_at, end_at, params)
Array(measure_keys).map do |key|
klass = MEASURES[key.to_sym]
klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end
end

View file

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
end
def time_period
(@start_at.to_date...@end_at.to_date)
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
end

View file

@ -1,9 +1,14 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::BaseMeasure
def initialize(start_at, end_at)
def self.with_params?
false
end
def initialize(start_at, end_at, params)
@start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime
@params = params
end
def key
@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure
protected
def time_period
(@start_at...@end_at)
(@start_at..@end_at)
end
def previous_time_period
((@start_at - length_of_period)...(@end_at - length_of_period))
((@start_at - length_of_period)..(@end_at - length_of_period))
end
def length_of_period
@length_of_period ||= @end_at - @start_at
end
def params
raise NotImplementedError
end
end

View file

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
end
def time_period
(@start_at.to_date...@end_at.to_date)
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_accounts'
end
def total
tag.history.aggregate(time_period).accounts
end
def previous_total
tag.history.aggregate(previous_time_period).accounts
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

View file

@ -0,0 +1,47 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_servers'
end
def total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def previous_total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def data
sql = <<-SQL.squish
SELECT axis.*, (
SELECT count(*) AS value
FROM statuses
WHERE statuses.id BETWEEN $1 AND $2
AND date_trunc('day', statuses.created_at)::date = axis.day
)
FROM (
SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day
) as axis
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]])
rows.map { |row| { date: row['day'], value: row['value'].to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def params
@params.permit(:id)
end
end

View file

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_uses'
end
def total
tag.history.aggregate(time_period).uses
end
def previous_total
tag.history.aggregate(previous_time_period).uses
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

View file

@ -4,6 +4,11 @@ class LinkDetailsExtractor
include ActionView::Helpers::TagHelper
class StructuredData
SUPPORTED_TYPES = %w(
NewsArticle
WebPage
).freeze
def initialize(data)
@data = data
end
@ -16,6 +21,14 @@ class LinkDetailsExtractor
json['description']
end
def language
json['inLanguage']
end
def type
json['@type']
end
def image
obj = first_of_value(json['image'])
@ -44,6 +57,10 @@ class LinkDetailsExtractor
publisher['name']
end
def publisher_logo
publisher.dig('logo', 'url')
end
private
def author
@ -58,8 +75,12 @@ class LinkDetailsExtractor
arr.is_a?(Array) ? arr.first : arr
end
def root_array(root)
root.is_a?(Array) ? root : [root]
end
def json
@json ||= first_of_value(Oj.load(@data))
@json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
end
end
@ -75,6 +96,7 @@ class LinkDetailsExtractor
description: description || '',
image_remote_url: image,
type: type,
link_type: link_type,
width: width || 0,
height: height || 0,
html: html || '',
@ -83,6 +105,7 @@ class LinkDetailsExtractor
author_name: author_name || '',
author_url: author_url || '',
embed_url: embed_url || '',
language: language,
}
end
@ -90,6 +113,14 @@ class LinkDetailsExtractor
player_url.present? ? :video : :link
end
def link_type
if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article'
:article
else
:unknown
end
end
def html
player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
end
@ -138,6 +169,14 @@ class LinkDetailsExtractor
valid_url_or_nil(opengraph_tag('twitter:player:stream'))
end
def language
valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
end
def icon
valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
end
private
def player_url
@ -162,6 +201,14 @@ class LinkDetailsExtractor
nil
end
def valid_locale_or_nil(str)
return nil if str.blank?
code, = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
locale = ISO_639.find(code)
locale&.alpha2
end
def link_tag(name)
document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
end

View file

@ -25,13 +25,25 @@ class AdminMailer < ApplicationMailer
end
end
def new_trending_tag(recipient, tag)
@tag = tag
@me = recipient
@instance = Rails.configuration.x.local_domain
def new_trending_tags(recipient, tags)
@tags = tags
@me = recipient
@instance = Rails.configuration.x.local_domain
@lowest_trending_tag = Trends.tags.get(true, Trends::Tags::REVIEW_THRESHOLD).last
locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance)
end
end
def new_trending_links(recipient, links)
@links = links
@me = recipient
@instance = Rails.configuration.x.local_domain
@lowest_trending_link = Trends.links.get(true, Trends::Links::REVIEW_THRESHOLD).last
locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance)
end
end
end

View file

@ -4,8 +4,8 @@
#
# Table name: account_statuses_cleanup_policies
#
# id :bigint not null, primary key
# account_id :bigint not null
# id :bigint(8) not null, primary key
# account_id :bigint(8) not null
# enabled :boolean default(TRUE), not null
# min_status_age :integer default(1209600), not null
# keep_direct :boolean default(TRUE), not null

View file

@ -0,0 +1,65 @@
# frozen_string_literal: true
class Form::PreviewCardBatch
include ActiveModel::Model
include Authorization
attr_accessor :preview_card_ids, :action, :current_account, :precision
def save
case action
when 'approve'
approve!
when 'approve_all'
approve_all!
when 'reject'
reject!
when 'reject_all'
reject_all!
end
end
private
def preview_cards
@preview_cards ||= PreviewCard.where(id: preview_card_ids)
end
def preview_card_providers
@preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) }
end
def approve!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: true)
end
def approve_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: true, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def reject!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: false)
end
def reject_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: false, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def action_time
@action_time ||= Time.now.utc
end
end

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
class Form::PreviewCardProviderBatch
include ActiveModel::Model
include Authorization
attr_accessor :preview_card_provider_ids, :action, :current_account
def save
case action
when 'approve'
approve!
when 'reject'
reject!
end
end
private
def preview_card_providers
PreviewCardProvider.where(id: preview_card_provider_ids)
end
def approve!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
end
def reject!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
end
end

View file

@ -23,11 +23,15 @@ class Form::TagBatch
def approve!
tags.each { |tag| authorize(tag, :update?) }
tags.update_all(trendable: true, reviewed_at: Time.now.utc)
tags.update_all(trendable: true, reviewed_at: action_time)
end
def reject!
tags.each { |tag| authorize(tag, :update?) }
tags.update_all(trendable: false, reviewed_at: Time.now.utc)
tags.update_all(trendable: false, reviewed_at: action_time)
end
def action_time
@action_time ||= Time.now.utc
end
end

View file

@ -24,6 +24,11 @@
# embed_url :string default(""), not null
# image_storage_schema_version :integer
# blurhash :string
# language :string
# max_score :float
# max_score_at :datetime
# trendable :boolean
# link_type :integer
#
class PreviewCard < ApplicationRecord
@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord
self.inheritance_column = false
enum type: [:link, :photo, :video, :rich]
enum link_type: [:unknown, :article]
has_and_belongs_to_many :statuses
@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord
before_save :extract_dimensions, if: :link?
def appropriate_for_trends?
link? && article? && title.present? && description.present? && image.present? && provider_name.present?
end
def domain
@domain ||= Addressable::URI.parse(url).normalized_host
end
def provider
@provider ||= PreviewCardProvider.matching_domain(domain)
end
def trendable?
if attributes['trendable'].nil?
provider&.trendable?
else
attributes['trendable']
end
end
def requires_review_notification?
attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?)
end
attr_writer :provider
def local?
false
end
@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord
save!
end
def history
@history ||= Trends::History.new('links', id)
end
class << self
private
# rubocop:disable Naming/MethodParameterName
def image_styles(f)
def image_styles(file)
styles = {
original: {
geometry: '400x400>',
@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord
},
}
styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif'
styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif'
styles
end
# rubocop:enable Naming/MethodParameterName
end
private

View file

@ -0,0 +1,53 @@
# frozen_string_literal: true
class PreviewCardFilter
KEYS = %i(
trending
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = PreviewCard.unscoped
params.each do |key, value|
next if key.to_s == 'page'
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope
end
private
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
ids = begin
case value.to_s
when 'allowed'
Trends.links.currently_trending_ids(true, -1)
else
Trends.links.currently_trending_ids(false, -1)
end
end
if ids.empty?
PreviewCard.none
else
PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
end
end
end

View file

@ -0,0 +1,57 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: preview_card_providers
#
# id :bigint(8) not null, primary key
# domain :string default(""), not null
# icon_file_name :string
# icon_content_type :string
# icon_file_size :bigint(8)
# icon_updated_at :datetime
# trendable :boolean
# reviewed_at :datetime
# requested_review_at :datetime
# created_at :datetime not null
# updated_at :datetime not null
#
class PreviewCardProvider < ApplicationRecord
include DomainNormalizable
include Attachmentable
ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze
LIMIT = 1.megabyte
validates :domain, presence: true, uniqueness: true, domain: true
has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
remotable_attachment :icon, LIMIT
scope :trendable, -> { where(trendable: true) }
scope :not_trendable, -> { where(trendable: false) }
scope :reviewed, -> { where.not(reviewed_at: nil) }
scope :pending_review, -> { where(reviewed_at: nil) }
def requires_review?
reviewed_at.nil?
end
def reviewed?
reviewed_at.present?
end
def requested_review?
requested_review_at.present?
end
def requires_review_notification?
requires_review? && !requested_review?
end
def self.matching_domain(domain)
segments = domain.split('.')
where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first
end
end

View file

@ -0,0 +1,49 @@
# frozen_string_literal: true
class PreviewCardProviderFilter
KEYS = %i(
status
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = PreviewCardProvider.unscoped
params.each do |key, value|
next if key.to_s == 'page'
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope.order(domain: :asc)
end
private
def scope_for(key, value)
case key.to_s
when 'status'
status_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def status_scope(value)
case value.to_s
when 'approved'
PreviewCardProvider.trendable
when 'rejected'
PreviewCardProvider.not_trendable
when 'pending_review'
PreviewCardProvider.pending_review
else
raise "Unknown status: #{value}"
end
end
end

View file

@ -36,6 +36,7 @@ class Tag < ApplicationRecord
scope :usable, -> { where(usable: [true, nil]) }
scope :listable, -> { where(listable: [true, nil]) }
scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
scope :not_trendable, -> { where(trendable: false) }
scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
@ -75,28 +76,12 @@ class Tag < ApplicationRecord
requested_review_at.present?
end
def use!(account, status: nil, at_time: Time.now.utc)
TrendingTags.record_use!(self, account, status: status, at_time: at_time)
end
def trending?
TrendingTags.trending?(self)
def requires_review_notification?
requires_review? && !requested_review?
end
def history
days = []
7.times do |i|
day = i.days.ago.beginning_of_day.to_i
days << {
day: day.to_s,
uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
}
end
days
@history ||= Trends::History.new('tags', id)
end
class << self

View file

@ -2,13 +2,8 @@
class TagFilter
KEYS = %i(
directory
reviewed
unreviewed
pending_review
popular
active
name
trending
status
).freeze
attr_reader :params
@ -18,7 +13,13 @@ class TagFilter
end
def results
scope = Tag.unscoped
scope = begin
if params[:status] == 'pending_review'
Tag.unscoped
else
trending_scope
end
end
params.each do |key, value|
next if key.to_s == 'page'
@ -26,27 +27,40 @@ class TagFilter
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope.order(id: :desc)
scope
end
private
def scope_for(key, value)
case key.to_s
when 'reviewed'
Tag.reviewed.order(reviewed_at: :desc)
when 'unreviewed'
Tag.unreviewed
when 'pending_review'
Tag.pending_review.order(requested_review_at: :desc)
when 'popular'
Tag.order('max_score DESC NULLS LAST')
when 'active'
Tag.order('last_status_at DESC NULLS LAST')
when 'name'
Tag.matches_name(value)
when 'status'
status_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope
ids = Trends.tags.currently_trending_ids(false, -1)
if ids.empty?
Tag.none
else
Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
end
end
def status_scope(value)
case value.to_s
when 'approved'
Tag.trendable
when 'rejected'
Tag.not_trendable
when 'pending_review'
Tag.pending_review
else
raise "Unknown status: #{value}"
end
end
end

View file

@ -1,128 +0,0 @@
# frozen_string_literal: true
class TrendingTags
KEY = 'trending_tags'
EXPIRE_HISTORY_AFTER = 7.days.seconds
EXPIRE_TRENDS_AFTER = 1.day.seconds
THRESHOLD = 5
LIMIT = 10
REVIEW_THRESHOLD = 3
MAX_SCORE_COOLDOWN = 2.days.freeze
MAX_SCORE_HALFLIFE = 2.hours.freeze
class << self
include Redisable
def record_use!(tag, account, status: nil, at_time: Time.now.utc)
return unless tag.usable? && !account.silenced?
# Even if a tag is not allowed to trend, we still need to
# record the stats since they can be displayed in other places
increment_historical_use!(tag.id, at_time)
increment_unique_use!(tag.id, account.id, at_time)
increment_use!(tag.id, at_time)
# Only update when the tag was last used once every 12 hours
# and only if a status is given (lets use ignore reblogs)
tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
end
def update!(at_time = Time.now.utc)
tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
tags = Tag.trendable.where(id: tag_ids.uniq)
# First pass to calculate scores and update the set
tags.each do |tag|
expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
expected = 1.0 if expected.zero?
observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
max_time = tag.max_score_at
max_score = tag.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
score = begin
if expected > observed || observed < THRESHOLD
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
tag.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
if decaying_score.zero?
redis.zrem(KEY, tag.id)
else
redis.zadd(KEY, decaying_score, tag.id)
end
end
users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
# Second pass to notify about previously unreviewed trends
tags.each do |tag|
current_rank = redis.zrevrank(KEY, tag.id)
needs_review_notification = tag.requires_review? && !tag.requested_review?
rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD
next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
tag.touch(:requested_review_at)
users_for_review.each do |user|
AdminMailer.new_trending_tag(user.account, tag).deliver_later!
end
end
# Trim older items
redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
redis.zremrangebyscore(KEY, '(0.3', '-inf')
end
def get(limit, filtered: true)
tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
tags = Tag.where(id: tag_ids)
tags = tags.trendable if filtered
tags = tags.index_by(&:id)
tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
end
def trending?(tag)
rank = redis.zrevrank(KEY, tag.id)
rank.present? && rank < LIMIT
end
private
def increment_historical_use!(tag_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
redis.incrby(key, 1)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
def increment_unique_use!(tag_id, account_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
redis.pfadd(key, account_id)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
def increment_use!(tag_id, at_time)
key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
redis.sadd(key, tag_id)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
end
end

27
app/models/trends.rb Normal file
View file

@ -0,0 +1,27 @@
# frozen_string_literal: true
module Trends
def self.table_name_prefix
'trends_'
end
def self.links
@links ||= Trends::Links.new
end
def self.tags
@tags ||= Trends::Tags.new
end
def self.refresh!
[links, tags].each(&:refresh)
end
def self.request_review!
[links, tags].each(&:request_review) if enabled?
end
def self.enabled?
Setting.trends
end
end

80
app/models/trends/base.rb Normal file
View file

@ -0,0 +1,80 @@
# frozen_string_literal: true
class Trends::Base
include Redisable
class_attribute :default_options
attr_reader :options
# @param [Hash] options
# @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score
# @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review
# @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from
# @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays
def initialize(options = {})
@options = self.class.default_options.merge(options)
end
def register(_status)
raise NotImplementedError
end
def add(*)
raise NotImplementedError
end
def refresh(*)
raise NotImplementedError
end
def request_review
raise NotImplementedError
end
def get(*)
raise NotImplementedError
end
def score(id)
redis.zscore("#{key_prefix}:all", id) || 0
end
def rank(id)
redis.zrevrank("#{key_prefix}:allowed", id)
end
def currently_trending_ids(allowed, limit)
redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i)
end
protected
def key_prefix
raise NotImplementedError
end
def recently_used_ids(at_time = Time.now.utc)
redis.smembers(used_key(at_time)).map(&:to_i)
end
def record_used_id(id, at_time = Time.now.utc)
redis.sadd(used_key(at_time), id)
redis.expire(used_key(at_time), 1.day.seconds)
end
def trim_older_items
redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1')
redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1')
end
def score_at_rank(rank)
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
end
private
def used_key(at_time)
"#{key_prefix}:used:#{at_time.beginning_of_day.to_i}"
end
end

View file

@ -0,0 +1,98 @@
# frozen_string_literal: true
class Trends::History
include Enumerable
class Aggregate
include Redisable
def initialize(prefix, id, date_range)
@days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) }
end
def uses
redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum
end
def accounts
redis.pfcount(*@days.map { |day| day.key_for(:accounts) })
end
end
class Day
include Redisable
EXPIRE_AFTER = 14.days.seconds
def initialize(prefix, id, day)
@prefix = prefix
@id = id
@day = day.beginning_of_day
end
attr_reader :day
def accounts
redis.pfcount(key_for(:accounts))
end
def uses
redis.get(key_for(:uses))&.to_i || 0
end
def add(account_id)
redis.pipelined do
redis.incrby(key_for(:uses), 1)
redis.pfadd(key_for(:accounts), account_id)
redis.expire(key_for(:uses), EXPIRE_AFTER)
redis.expire(key_for(:accounts), EXPIRE_AFTER)
end
end
def as_json
{ day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s }
end
def key_for(suffix)
case suffix
when :accounts
"#{key_prefix}:#{suffix}"
when :uses
key_prefix
end
end
def key_prefix
"activity:#{@prefix}:#{@id}:#{day.to_i}"
end
end
def initialize(prefix, id)
@prefix = prefix
@id = id
end
def get(date)
Day.new(@prefix, @id, date)
end
def add(account_id, at_time = Time.now.utc)
Day.new(@prefix, @id, at_time).add(account_id)
end
def aggregate(date_range)
Aggregate.new(@prefix, @id, date_range)
end
def each(&block)
if block_given?
(0...7).map { |i| block.call(get(i.days.ago)) }
else
to_enum(:each)
end
end
def as_json(*)
map(&:as_json)
end
end

117
app/models/trends/links.rb Normal file
View file

@ -0,0 +1,117 @@
# frozen_string_literal: true
class Trends::Links < Trends::Base
PREFIX = 'trending_links'
self.default_options = {
threshold: 15,
review_threshold: 10,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 8.hours.freeze,
}
def register(status, at_time = Time.now.utc)
original_status = status.reblog? ? status.reblog : status
return unless original_status.public_visibility? && status.public_visibility? &&
!original_status.account.silenced? && !status.account.silenced? &&
!original_status.spoiler_text?
original_status.preview_cards.each do |preview_card|
add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends?
end
end
def add(preview_card, account_id, at_time = Time.now.utc)
preview_card.history.add(account_id, at_time)
record_used_id(preview_card.id, at_time)
end
def get(allowed, limit)
preview_card_ids = currently_trending_ids(allowed, limit)
preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
preview_card_ids.map { |id| preview_cards[id] }.compact
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(preview_cards, at_time)
trim_older_items
end
def request_review
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
if preview_card.provider.nil?
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
else
preview_card.provider.touch(:requested_review_at)
end
preview_card
end
return if preview_cards_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
def key_prefix
PREFIX
end
private
def calculate_scores(preview_cards, at_time)
preview_cards.each do |preview_card|
expected = preview_card.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = preview_card.history.get(at_time).accounts.to_f
max_time = preview_card.max_score_at
max_score = preview_card.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
preview_card.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", preview_card.id)
redis.zrem("#{PREFIX}:allowed", preview_card.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
if preview_card.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
else
redis.zrem("#{PREFIX}:allowed", preview_card.id)
end
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

111
app/models/trends/tags.rb Normal file
View file

@ -0,0 +1,111 @@
# frozen_string_literal: true
class Trends::Tags < Trends::Base
PREFIX = 'trending_tags'
self.default_options = {
threshold: 15,
review_threshold: 10,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 4.hours.freeze,
}
def register(status, at_time = Time.now.utc)
original_status = status.reblog? ? status.reblog : status
return unless original_status.public_visibility? && status.public_visibility? &&
!original_status.account.silenced? && !status.account.silenced?
original_status.tags.each do |tag|
add(tag, status.account_id, at_time) if tag.usable?
end
end
def add(tag, account_id, at_time = Time.now.utc)
tag.history.add(account_id, at_time)
record_used_id(tag.id, at_time)
end
def refresh(at_time = Time.now.utc)
tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(tags, at_time)
trim_older_items
end
def get(allowed, limit)
tag_ids = currently_trending_ids(allowed, limit)
tags = Tag.where(id: tag_ids).index_by(&:id)
tag_ids.map { |id| tags[id] }.compact
end
def request_review
tags = Tag.where(id: currently_trending_ids(false, -1))
tags_requiring_review = tags.filter_map do |tag|
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
tag.touch(:requested_review_at)
tag
end
return if tags_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
def key_prefix
PREFIX
end
private
def calculate_scores(tags, at_time)
tags.each do |tag|
expected = tag.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = tag.history.get(at_time).accounts.to_f
max_time = tag.max_score_at
max_score = tag.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
tag.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", tag.id)
redis.zrem("#{PREFIX}:allowed", tag.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
if tag.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
else
redis.zrem("#{PREFIX}:allowed", tag.id)
end
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class PreviewCardPolicy < ApplicationPolicy
def index?
staff?
end
def update?
staff?
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class PreviewCardProviderPolicy < ApplicationPolicy
def index?
staff?
end
def update?
staff?
end
end

View file

@ -0,0 +1,5 @@
# frozen_string_literal: true
class REST::Trends::LinkSerializer < REST::PreviewCardSerializer
attributes :history
end

View file

@ -50,7 +50,7 @@ class FetchLinkCardService < BaseService
# We follow redirects, and ideally we want to save the preview card for
# the destination URL and not any link shortener in-between, so here
# we set the URL to the one of the last response in the redirect chain
@url = res.request.uri.to_s.to_s
@url = res.request.uri.to_s
@card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url
if res.code == 200 && res.mime_type == 'text/html'
@ -66,6 +66,7 @@ class FetchLinkCardService < BaseService
def attach_card
@status.preview_cards << @card
Rails.cache.delete(@status)
Trends.links.register(@status)
end
def parse_urls

View file

@ -91,7 +91,8 @@ class PostStatusService < BaseService
end
def postprocess_status!
LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text?
Trends.tags.register(@status)
LinkCrawlWorker.perform_async(@status.id)
DistributionWorker.perform_async(@status.id)
ActivityPub::DistributionWorker.perform_async(@status.id)
PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll

View file

@ -8,7 +8,7 @@ class ProcessHashtagsService < BaseService
Tag.find_or_create_by_names(tags) do |tag|
status.tags << tag
records << tag
tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility?
tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
end
return unless status.distributable?

View file

@ -30,12 +30,13 @@ class ReblogService < BaseService
reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit])
Trends.tags.register(reblog)
Trends.links.register(reblog)
DistributionWorker.perform_async(reblog.id)
ActivityPub::DistributionWorker.perform_async(reblog.id)
create_notification(reblog)
bump_potential_friendship(account, reblog)
record_use(account, reblog)
reblog
end
@ -60,16 +61,6 @@ class ReblogService < BaseService
PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
end
def record_use(account, reblog)
return unless reblog.public_visibility?
original_status = reblog.reblog
original_status.tags.each do |tag|
tag.use!(account)
end
end
def build_json(reblog)
Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
end

View file

@ -42,7 +42,7 @@
%span= t('admin.dashboard.pending_users_html', count: @pending_users_count)
= fa_icon 'chevron-right fw'
= link_to admin_tags_path(pending_review: '1'), class: 'dashboard__quick-access' do
= link_to admin_trends_tags_path(status: 'pending_review'), class: 'dashboard__quick-access' do
%span= t('admin.dashboard.pending_tags_html', count: @pending_tags_count)
= fa_icon 'chevron-right fw'

View file

@ -1,19 +0,0 @@
.batch-table__row
- if batch_available
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
.directory__tag
= link_to admin_tag_path(tag.id) do
%h4
= fa_icon 'hashtag'
= tag.name
%small
= t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
- if tag.trending?
= fa_icon 'fire fw'
= t('admin.tags.trending_right_now')
.trends__item__current= friendly_number_to_human tag.history.first[:uses]

View file

@ -1,74 +0,0 @@
- content_for :page_title do
= t('admin.tags.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.tags.review')
%ul
%li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil
%li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil
%li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil
.filter-subset
%strong= t('generic.order_by')
%ul
%li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil
%li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil
%li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do
.fields-group
- TagFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
- %i(name).each do |key|
.input.string.optional
= text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}")
.actions
%button.button= t('admin.accounts.search')
= link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative'
%hr.spacer/
= form_for(@form, url: batch_admin_tags_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- TagFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional
.batch-table__toolbar
- if params[:pending_review] == '1' || params[:unreviewed] == '1'
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
- else
.batch-table__toolbar__actions
%span.neutral-hint= t('generic.no_batch_actions_available')
.batch-table__body
- if @tags.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'tag', collection: @tags, locals: { f: f, batch_available: params[:pending_review] == '1' || params[:unreviewed] == '1' }
= paginate @tags
- if params[:pending_review] == '1' || params[:unreviewed] == '1'
%hr.spacer/
%div.action-buttons
%div
= link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button'
%div
= link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive'

View file

@ -1,15 +1,50 @@
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
- content_for :page_title do
= "##{@tag.name}"
.dashboard__counters
%div
= link_to tag_url(@tag), target: '_blank', rel: 'noopener noreferrer' do
.dashboard__counters__num= number_with_delimiter @accounts_today
.dashboard__counters__label= t 'admin.tags.accounts_today'
%div
%div
.dashboard__counters__num= number_with_delimiter @accounts_week
.dashboard__counters__label= t 'admin.tags.accounts_week'
- content_for :heading_actions do
= l(@time_period.first)
= ' - '
= l(@time_period.last)
.dashboard
.dashboard__item
= react_admin_component :counter, measure: 'tag_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_accounts_measure')
.dashboard__item
= react_admin_component :counter, measure: 'tag_uses', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_uses_measure')
.dashboard__item
= react_admin_component :counter, measure: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_servers_measure')
.dashboard__item
= react_admin_component :dimension, dimension: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_servers_dimension')
.dashboard__item
= react_admin_component :dimension, dimension: 'tag_languages', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_languages_dimension')
.dashboard__item
= link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.usable? ? 'positive' : 'negative'] do
- if @tag.usable?
%span= t('admin.trends.tags.usable')
= fa_icon 'check fw'
- else
%span= t('admin.trends.tags.not_usable')
= fa_icon 'lock fw'
= link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.trendable? ? 'positive' : 'negative'] do
- if @tag.trendable?
%span= t('admin.trends.tags.trendable')
= fa_icon 'check fw'
- else
%span= t('admin.trends.tags.not_trendable')
= fa_icon 'lock fw'
= link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.listable? ? 'positive' : 'negative'] do
- if @tag.listable?
%span= t('admin.trends.tags.listable')
= fa_icon 'check fw'
- else
%span= t('admin.trends.tags.not_listable')
= fa_icon 'lock fw'
%hr.spacer/
@ -26,18 +61,3 @@
.actions
= f.button :button, t('generic.save_changes'), type: :submit
%hr.spacer/
%h3= t 'admin.tags.breakdown'
.table-wrapper
%table.table
%tbody
- total = @usage_by_domain.sum(&:last).to_f
- @usage_by_domain.each do |(domain, count)|
%tr
%th= domain || site_hostname
%td= number_to_percentage((count / total) * 100, precision: 1)
%td= number_with_delimiter count

View file

@ -0,0 +1,30 @@
.batch-table__row{ class: [preview_card.provider&.requires_review? && 'batch-table__row--attention', !preview_card.provider&.requires_review? && !preview_card.trendable? && 'batch-table__row--muted'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :preview_card_ids, { multiple: true, include_hidden: false }, preview_card.id
.batch-table__row__content.pending-account
.pending-account__header
= link_to preview_card.title, preview_card.url
%br/
- if preview_card.provider_name.present?
= preview_card.provider_name
- if preview_card.language.present?
= human_locale(preview_card.language)
= t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts })
- if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id))
%abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
- if preview_card.max_score_at && preview_card.max_score_at >= Trends::Links::MAX_SCORE_COOLDOWN.ago && preview_card.max_score_at < 1.day.ago
= t('admin.trends.tags.peaked_on_and_decaying', date: l(preview_card.max_score_at.to_date, format: :short))
- elsif preview_card.provider&.requires_review?
= t('admin.trends.pending_review')

View file

@ -0,0 +1,41 @@
- content_for :page_title do
= t('admin.trends.links.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.trends.trending')
%ul
%li= filter_link_to t('generic.all'), trending: nil
%li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed'
.back-link
= link_to admin_trends_links_preview_card_providers_path do
= t('admin.trends.preview_card_providers.title')
= fa_icon 'chevron-right fw'
%hr.spacer/
= form_for(@form, url: batch_admin_trends_links_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- PreviewCardFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @preview_cards.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'preview_card', collection: @preview_cards, locals: { f: f }
= paginate @preview_cards

View file

@ -0,0 +1,16 @@
.batch-table__row{ class: [preview_card_provider.requires_review? && 'batch-table__row--attention', !preview_card_provider.requires_review? && !preview_card_provider.trendable? && 'batch-table__row--muted'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :preview_card_provider_ids, { multiple: true, include_hidden: false }, preview_card_provider.id
.batch-table__row__content.pending-account
.pending-account__header
%strong= preview_card_provider.domain
%br/
- if preview_card_provider.requires_review?
= t('admin.trends.pending_review')
- elsif preview_card_provider.trendable?
= t('admin.trends.preview_card_providers.allowed')
- else
= t('admin.trends.preview_card_providers.rejected')

View file

@ -0,0 +1,43 @@
- content_for :page_title do
= t('admin.trends.preview_card_providers.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.tags.review')
%ul
%li= filter_link_to t('generic.all'), status: nil
%li= filter_link_to t('admin.trends.approved'), status: 'approved'
%li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{PreviewCardProvider.pending_review.count})"], ' '), status: 'pending_review'
.back-link
= link_to admin_trends_links_path do
= fa_icon 'chevron-left fw'
= t('admin.trends.links.title')
%hr.spacer/
= form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- PreviewCardProviderFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @preview_card_providers.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'preview_card_provider', collection: @preview_card_providers, locals: { f: f }
= paginate @preview_card_providers

View file

@ -0,0 +1,24 @@
.batch-table__row{ class: [tag.requires_review? && 'batch-table__row--attention', !tag.requires_review? && !tag.trendable? && 'batch-table__row--muted'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
.batch-table__row__content.pending-account
.pending-account__header
= link_to admin_tag_path(tag.id) do
= fa_icon 'hashtag'
= tag.name
%br/
= t('admin.trends.tags.used_by_over_week', count: tag.history.reduce(0) { |sum, day| sum + day.accounts })
- if tag.trendable? && (rank = Trends.tags.rank(tag.id))
%abbr{ title: t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
- if tag.max_score_at && tag.max_score_at >= Trends::Tags::MAX_SCORE_COOLDOWN.ago && tag.max_score_at < 1.day.ago
= t('admin.trends.tags.peaked_on_and_decaying', date: l(tag.max_score_at.to_date, format: :short))
- elsif tag.requires_review?
= t('admin.trends.pending_review')

View file

@ -0,0 +1,38 @@
- content_for :page_title do
= t('admin.trends.tags.title')
- content_for :header_tags do
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
.filters
.filter-subset
%strong= t('admin.tags.review')
%ul
%li= filter_link_to t('generic.all'), status: nil
%li= filter_link_to t('admin.trends.approved'), status: 'approved'
%li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review'
%hr.spacer/
= form_for(@form, url: batch_admin_trends_tags_path) do |f|
= hidden_field_tag :page, params[:page] || 1
- TagFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
= f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @tags.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'tag', collection: @tags, locals: { f: f }
= paginate @tags

View file

@ -0,0 +1,16 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_links.body') %>
<% @links.each do |link| %>
- <%= link.title %> • <%= link.url %>
<%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %>
<% end %>
<% if @lowest_trending_link %>
<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %>
<% else %>
<%= t('admin_mailer.new_trending_links.no_approved_links') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %>

View file

@ -1,5 +0,0 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
<%= raw t('application_mailer.view')%> <%= admin_tags_url(pending_review: '1') %>

View file

@ -0,0 +1,16 @@
<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
<%= raw t('admin_mailer.new_trending_tags.body') %>
<% @tags.each do |tag| %>
- #<%= tag.name %>
<%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %>
<% end %>
<% if @lowest_trending_tag %>
<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %>
<% else %>
<%= t('admin_mailer.new_trending_tags.no_approved_tags') %>
<% end %>
<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %>

View file

@ -6,7 +6,7 @@
%p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html')
- if Setting.trends && !(user_signed_in? && !current_user.setting_trends)
- trends = TrendingTags.get(3)
- trends = Trends.tags.get(true, 3)
- unless trends.empty?
.endorsements-widget.trends-widget

View file

@ -1,11 +1,11 @@
# frozen_string_literal: true
class Scheduler::TrendingTagsScheduler
class Scheduler::Trends::RefreshScheduler
include Sidekiq::Worker
sidekiq_options retry: 0
def perform
TrendingTags.update! if Setting.trends
Trends.refresh!
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Scheduler::Trends::ReviewNotificationsScheduler
include Sidekiq::Worker
sidekiq_options retry: 0
def perform
Trends.request_review!
end
end

View file

@ -67,7 +67,7 @@
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/account.rb",
"line": 479,
"line": 484,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" WITH first_degree AS (\\n SELECT target_account_id\\n FROM follows\\n WHERE account_id = ?\\n UNION ALL\\n SELECT ?\\n )\\n SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n WHERE accounts.id IN (SELECT * FROM first_degree)\\n AND #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])",
"render_path": null,
@ -100,6 +100,26 @@
"confidence": "Weak",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "75fcd147b7611763ab6915faf8c5b0709e612b460f27c05c72d8b9bd0a6a77f8",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "lib/mastodon/snowflake.rb",
"line": 87,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "connection.execute(\"CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\nRETURNS bigint AS\\n$$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n$$ LANGUAGE plpgsql VOLATILE;\\n\")",
"render_path": null,
"location": {
"type": "method",
"class": "Mastodon::Snowflake",
"method": "define_timestamp_id"
},
"user_input": "SecureRandom.hex(16)",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Mass Assignment",
"warning_code": 105,
@ -140,6 +160,26 @@
"confidence": "High",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/preview_card_filter.rb",
"line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "PreviewCardFilter",
"method": "trending_scope"
},
"user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
@ -147,7 +187,7 @@
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/account.rb",
"line": 448,
"line": 453,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" SELECT\\n accounts.*,\\n ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
"render_path": null,
@ -160,26 +200,6 @@
"confidence": "Medium",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "9ccb9ba6a6947400e187d515e0bf719d22993d37cfc123c824d7fafa6caa9ac3",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "lib/mastodon/snowflake.rb",
"line": 87,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "connection.execute(\" CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n RETURNS bigint AS\\n $$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name ||\\n '#{SecureRandom.hex(16)}' ||\\n time_part::text\\n ),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n $$ LANGUAGE plpgsql VOLATILE;\\n\")",
"render_path": null,
"location": {
"type": "method",
"class": "Mastodon::Snowflake",
"method": "define_timestamp_id"
},
"user_input": "SecureRandom.hex(16)",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Redirect",
"warning_code": 18,
@ -201,23 +221,53 @@
"note": ""
},
{
"warning_type": "Redirect",
"warning_code": 18,
"fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33",
"check_name": "Redirect",
"message": "Possible unprotected redirect",
"file": "app/controllers/media_controller.rb",
"line": 20,
"link": "https://brakemanscanner.org/docs/warning_types/redirect/",
"code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))",
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/tag_filter.rb",
"line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "MediaController",
"method": "show"
"class": "TagFilter",
"method": "trending_scope"
},
"user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)",
"confidence": "High",
"user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Cross-Site Scripting",
"warning_code": 4,
"fingerprint": "cd5cfd7f40037fbfa753e494d7129df16e358bfc43ef0da3febafbf4ee1ed3ac",
"check_name": "LinkToHref",
"message": "Potentially unsafe model attribute in `link_to` href",
"file": "app/views/admin/trends/links/_preview_card.html.haml",
"line": 7,
"link": "https://brakemanscanner.org/docs/warning_types/link_to_href",
"code": "link_to((Unresolved Model).new.title, (Unresolved Model).new.url)",
"render_path": [
{
"type": "template",
"name": "admin/trends/links/index",
"line": 37,
"file": "app/views/admin/trends/links/index.html.haml",
"rendered": {
"name": "admin/trends/links/_preview_card",
"file": "app/views/admin/trends/links/_preview_card.html.haml"
}
}
],
"location": {
"type": "template",
"template": "admin/trends/links/_preview_card"
},
"user_input": "(Unresolved Model).new.url",
"confidence": "Weak",
"note": ""
},
{
@ -227,7 +277,7 @@
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/account.rb",
"line": 495,
"line": 500,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])",
"render_path": null,
@ -261,6 +311,6 @@
"note": ""
}
],
"updated": "2021-05-11 20:22:27 +0900",
"brakeman_version": "5.0.1"
"updated": "2021-11-14 05:26:09 +0100",
"brakeman_version": "5.1.2"
}

View file

@ -674,8 +674,8 @@ en:
desc_html: Affects hashtags that have not been previously disallowed
title: Allow hashtags to trend without prior review
trends:
desc_html: Publicly display previously reviewed hashtags that are currently trending
title: Trending hashtags
desc_html: Publicly display previously reviewed content that is currently trending
title: Trends
site_uploads:
delete: Delete uploaded file
destroyed_msg: Site upload successfully deleted!
@ -702,21 +702,51 @@ en:
sidekiq_process_check:
message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
tags:
accounts_today: Unique uses today
accounts_week: Unique uses this week
breakdown: Breakdown of today's usage by source
last_active: Recently used
most_popular: Most popular
most_recent: Recently created
name: Hashtag
review: Review status
reviewed: Reviewed
title: Hashtags
trending_right_now: Trending right now
unique_uses_today: "%{count} posting today"
unreviewed: Not reviewed
updated_msg: Hashtag settings updated successfully
title: Administration
trends:
allow: Allow
approved: Approved
disallow: Disallow
links:
allow: Allow link
allow_provider: Allow publisher
disallow: Disallow link
disallow_provider: Disallow publisher
shared_by_over_week:
one: Shared by one person over the last week
other: Shared by %{count} people over the last week
title: Trending links
usage_comparison: Shared %{today} times today, compared to %{yesterday} yesterday
pending_review: Pending review
preview_card_providers:
allowed: Links from this publisher can trend
rejected: Links from this publisher won't trend
title: Publishers
rejected: Rejected
tags:
current_score: Current score %{score}
dashboard:
tag_accounts_measure: unique uses
tag_languages_dimension: Top languages
tag_servers_dimension: Top servers
tag_servers_measure: different servers
tag_uses_measure: total uses
listable: Can be suggested
not_listable: Won't be suggested
not_trendable: Won't appear under trends
not_usable: Cannot be used
peaked_on_and_decaying: Peaked on %{date}, now decaying
title: Trending hashtags
trendable: Can appear under trends
trending_rank: 'Trending #%{rank}'
usable: Can be used
usage_comparison: Used %{today} times today, compared to %{yesterday} yesterday
used_by_over_week:
one: Used by one person over the last week
other: Used by %{count} people over the last week
title: Trends
warning_presets:
add_new: Add new
delete: Delete
@ -731,9 +761,16 @@ en:
body: "%{reporter} has reported %{target}"
body_remote: Someone from %{domain} has reported %{target}
subject: New report for %{instance} (#%{id})
new_trending_tag:
body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
subject: New hashtag up for review on %{instance} (#%{name})
new_trending_links:
body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated.
no_approved_links: There are currently no approved trending links.
requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.
subject: New trending links up for review on %{instance}
new_trending_tags:
body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:'
no_approved_tags: There are currently no approved trending hashtags.
requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.'
subject: New trending hashtags up for review on %{instance}
aliases:
add_new: Create alias
created_msg: Successfully created a new alias. You can now initiate the move from the old account.
@ -940,7 +977,7 @@ en:
changes_saved_msg: Changes successfully saved!
copy: Copy
delete: Delete
no_batch_actions_available: No batch actions available on this page
none: None
order_by: Order by
save_changes: Save changes
validation_errors:

View file

@ -204,8 +204,8 @@ en:
mention: Someone mentioned you
pending_account: New account needs review
reblog: Someone boosted your post
report: New report is submitted
trending_tag: An unreviewed hashtag is trending
report: A new report is submitted
trending_tag: A new trend requires approval
rule:
text: Rule
tag:

View file

@ -34,12 +34,16 @@ SimpleNavigation::Configuration.run do |navigation|
n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? }
n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? }
n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s|
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
end
n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s|
s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url
s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}
s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }

View file

@ -301,12 +301,27 @@ Rails.application.routes.draw do
resources :account_moderation_notes, only: [:create, :destroy]
resource :follow_recommendations, only: [:show, :update]
resources :tags, only: [:show, :update]
resources :tags, only: [:index, :show, :update] do
collection do
post :approve_all
post :reject_all
post :batch
namespace :trends do
resources :links, only: [:index] do
collection do
post :batch
end
end
resources :tags, only: [:index] do
collection do
post :batch
end
end
namespace :links do
resources :preview_card_providers, only: [:index], path: :publishers do
collection do
post :batch
end
end
end
end
end
@ -399,7 +414,7 @@ Rails.application.routes.draw do
resources :favourites, only: [:index]
resources :bookmarks, only: [:index]
resources :reports, only: [:create]
resources :trends, only: [:index]
resources :trends, only: [:index], controller: 'trends/tags'
resources :filters, only: [:index, :create, :show, :update, :destroy]
resources :endorsements, only: [:index]
resources :markers, only: [:index, :create]
@ -410,6 +425,11 @@ Rails.application.routes.draw do
resources :apps, only: [:create]
namespace :trends do
resources :links, only: [:index]
resources :tags, only: [:index]
end
namespace :emails do
resources :confirmations, only: [:create]
end
@ -512,7 +532,9 @@ Rails.application.routes.draw do
end
end
resources :trends, only: [:index]
namespace :trends do
resources :tags, only: [:index]
end
post :measures, to: 'measures#create'
post :dimensions, to: 'dimensions#create'

View file

@ -13,9 +13,13 @@
every: '5m'
class: Scheduler::ScheduledStatusesScheduler
queue: scheduler
trending_tags_scheduler:
trends_refresh_scheduler:
every: '5m'
class: Scheduler::TrendingTagsScheduler
class: Scheduler::Trends::RefreshScheduler
queue: scheduler
trends_review_notifications_scheduler:
every: '2h'
class: Scheduler::Trends::ReviewNotificationsScheduler
queue: scheduler
media_cleanup_scheduler:
cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'

View file

@ -0,0 +1,12 @@
class CreatePreviewCardProviders < ActiveRecord::Migration[6.1]
def change
create_table :preview_card_providers do |t|
t.string :domain, null: false, default: '', index: { unique: true }
t.attachment :icon
t.boolean :trendable
t.datetime :reviewed_at
t.datetime :requested_review_at
t.timestamps
end
end
end

View file

@ -0,0 +1,7 @@
class AddLanguageToPreviewCards < ActiveRecord::Migration[6.1]
def change
add_column :preview_cards, :language, :string
add_column :preview_cards, :max_score, :float
add_column :preview_cards, :max_score_at, :datetime
end
end

View file

@ -0,0 +1,5 @@
class AddTrendableToPreviewCards < ActiveRecord::Migration[6.1]
def change
add_column :preview_cards, :trendable, :boolean
end
end

View file

@ -0,0 +1,5 @@
class AddLinkTypeToPreviewCards < ActiveRecord::Migration[6.1]
def change
add_column :preview_cards, :link_type, :int
end
end

View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2021_08_08_071221) do
ActiveRecord::Schema.define(version: 2021_11_23_212714) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -689,6 +689,20 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
t.index ["status_id"], name: "index_polls_on_status_id"
end
create_table "preview_card_providers", force: :cascade do |t|
t.string "domain", default: "", null: false
t.string "icon_file_name"
t.string "icon_content_type"
t.bigint "icon_file_size"
t.datetime "icon_updated_at"
t.boolean "trendable"
t.datetime "reviewed_at"
t.datetime "requested_review_at"
t.datetime "created_at", precision: 6, null: false
t.datetime "updated_at", precision: 6, null: false
t.index ["domain"], name: "index_preview_card_providers_on_domain", unique: true
end
create_table "preview_cards", force: :cascade do |t|
t.string "url", default: "", null: false
t.string "title", default: "", null: false
@ -710,6 +724,11 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
t.string "embed_url", default: "", null: false
t.integer "image_storage_schema_version"
t.string "blurhash"
t.string "language"
t.float "max_score"
t.datetime "max_score_at"
t.boolean "trendable"
t.integer "link_type"
t.index ["url"], name: "index_preview_cards_on_url", unique: true
end

View file

@ -84,10 +84,7 @@ module Mastodon::Snowflake
-- Take the first two bytes (four hex characters)
substr(
-- Of the MD5 hash of the data we documented
md5(table_name ||
'#{SecureRandom.hex(16)}' ||
time_part::text
),
md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),
1, 4
)
-- And turn it into a bigint

View file

@ -96,7 +96,7 @@ namespace :repo do
end.uniq.compact
missing_available_locales = locales_in_files - I18n.available_locales
missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) }
missing_locale_names = I18n.available_locales.reject { |locale| LanguagesHelper::HUMAN_LOCALES.key?(locale) }
critical = false

View file

@ -9,18 +9,6 @@ RSpec.describe Admin::TagsController, type: :controller do
sign_in Fabricate(:user, admin: true)
end
describe 'GET #index' do
let!(:tag) { Fabricate(:tag) }
before do
get :index
end
it 'returns status 200' do
expect(response).to have_http_status(200)
end
end
describe 'GET #show' do
let!(:tag) { Fabricate(:tag) }

View file

@ -0,0 +1,22 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Api::V1::Trends::TagsController, type: :controller do
render_views
describe 'GET #index' do
before do
trending_tags = double()
allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag))
allow(Trends).to receive(:tags).and_return(trending_tags)
get :index
end
it 'returns http success' do
expect(response).to have_http_status(200)
end
end
end

View file

@ -1,18 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Api::V1::TrendsController, type: :controller do
render_views
describe 'GET #index' do
before do
allow(TrendingTags).to receive(:get).and_return(Fabricate.times(10, :tag))
get :index
end
it 'returns http success' do
expect(response).to have_http_status(200)
end
end
end

View file

@ -2,20 +2,15 @@
require 'rails_helper'
describe SettingsHelper do
describe LanguagesHelper do
describe 'the HUMAN_LOCALES constant' do
it 'includes all I18n locales' do
options = I18n.available_locales
expect(described_class::HUMAN_LOCALES.keys).to include(*options)
expect(described_class::HUMAN_LOCALES.keys).to include(*I18n.available_locales)
end
end
describe 'human_locale' do
it 'finds the human readable local description from a key' do
# Ensure the value is as we expect
expect(described_class::HUMAN_LOCALES[:en]).to eq('English')
expect(helper.human_locale(:en)).to eq('English')
end
end

View file

@ -5,4 +5,14 @@ class AdminMailerPreview < ActionMailer::Preview
def new_pending_account
AdminMailer.new_pending_account(Account.first, User.pending.first)
end
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags
def new_trending_tags
AdminMailer.new_trending_tags(Account.first, Tag.limit(3))
end
# Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links
def new_trending_links
AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3))
end
end

View file

@ -1,68 +0,0 @@
require 'rails_helper'
RSpec.describe TrendingTags do
describe '.record_use!' do
pending
end
describe '.update!' do
let!(:at_time) { Time.now.utc }
let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
before do
allow(Redis.current).to receive(:pfcount) do |key|
case key
when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
2
when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts"
16
when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
0
when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts"
4
when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
13
end
end
Redis.current.zadd('trending_tags', 0.9, tag3.id)
Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id])
tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours)
described_class.update!(at_time)
end
it 'calculates and re-calculates scores' do
expect(described_class.get(10, filtered: false)).to eq [tag1, tag3]
end
it 'omits hashtags below threshold' do
expect(described_class.get(10, filtered: false)).to_not include(tag2)
end
it 'decays scores' do
expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9
end
end
describe '.trending?' do
let(:tag) { Fabricate(:tag) }
before do
10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) }
end
it 'returns true if the hashtag is within limit' do
Redis.current.zadd('trending_tags', 11, tag.id)
expect(described_class.trending?(tag)).to be true
end
it 'returns false if the hashtag is outside the limit' do
Redis.current.zadd('trending_tags', 0, tag.id)
expect(described_class.trending?(tag)).to be false
end
end
end

View file

@ -0,0 +1,67 @@
require 'rails_helper'
RSpec.describe Trends::Tags do
subject { described_class.new(threshold: 5, review_threshold: 10) }
let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) }
describe '#add' do
let(:tag) { Fabricate(:tag) }
before do
subject.add(tag, 1, at_time)
end
it 'records history' do
expect(tag.history.get(at_time).accounts).to eq 1
end
it 'records use' do
expect(subject.send(:recently_used_ids, at_time)).to eq [tag.id]
end
end
describe '#get' do
pending
end
describe '#refresh' do
let!(:today) { at_time }
let!(:yesterday) { today - 1.day }
let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
before do
2.times { |i| subject.add(tag1, i, yesterday) }
13.times { |i| subject.add(tag3, i, yesterday) }
16.times { |i| subject.add(tag1, i, today) }
4.times { |i| subject.add(tag2, i, today) }
end
context do
before do
subject.refresh(yesterday + 12.hours)
subject.refresh(at_time)
end
it 'calculates and re-calculates scores' do
expect(subject.get(false, 10)).to eq [tag1, tag3]
end
it 'omits hashtags below threshold' do
expect(subject.get(false, 10)).to_not include(tag2)
end
end
it 'decays scores' do
subject.refresh(yesterday + 12.hours)
original_score = subject.score(tag3.id)
expect(original_score).to eq 144.0
subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife])
decayed_score = subject.score(tag3.id)
expect(decayed_score).to be <= original_score / 2
end
end
end