diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index e74e4755f..70afdedd7 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -27,7 +27,7 @@ module Admin @saml_enabled = ENV['SAML_ENABLED'] == 'true' @pam_enabled = ENV['PAM_ENABLED'] == 'true' @hidden_service = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true' - @trending_hashtags = TrendingTags.get(7) + @trending_hashtags = TrendingTags.get(10, filtered: false) @profile_directory = Setting.profile_directory @timeline_preview = Setting.timeline_preview @spam_check_enabled = Setting.spam_check_enabled diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb index e9f4f2cfa..0e9dda302 100644 --- a/app/controllers/admin/tags_controller.rb +++ b/app/controllers/admin/tags_controller.rb @@ -4,41 +4,49 @@ module Admin class TagsController < BaseController before_action :set_tags, only: :index before_action :set_tag, except: :index - before_action :set_filter_params def index authorize :tag, :index? end - def hide - authorize @tag, :hide? - @tag.account_tag_stat.update!(hidden: true) - redirect_to admin_tags_path(@filter_params) + def show + authorize @tag, :show? end - def unhide - authorize @tag, :unhide? - @tag.account_tag_stat.update!(hidden: false) - redirect_to admin_tags_path(@filter_params) + def update + authorize @tag, :update? + + if @tag.update(tag_params.merge(reviewed_at: Time.now.utc)) + redirect_to admin_tag_path(@tag.id) + else + render :show + end end private def set_tags - @tags = Tag.discoverable - @tags.merge!(Tag.hidden) if filter_params[:hidden] + @tags = filtered_tags.page(params[:page]) end def set_tag @tag = Tag.find(params[:id]) end - def set_filter_params - @filter_params = filter_params.to_hash.symbolize_keys + def filtered_tags + scope = Tag + scope = scope.discoverable if filter_params[:context] == 'directory' + scope = scope.reviewed if filter_params[:review] == 'reviewed' + scope = scope.pending_review if filter_params[:review] == 'pending_review' + scope.reorder(score: :desc) end def filter_params - params.permit(:hidden) + params.slice(:context, :review).permit(:context, :review) + end + + def tag_params + params.require(:tag).permit(:name, :trendable, :usable, :listable) end end end diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb new file mode 100644 index 000000000..bcea9857e --- /dev/null +++ b/app/controllers/api/v1/trends_controller.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class Api::V1::TrendsController < Api::BaseController + before_action :set_tags + + respond_to :json + + def index + render json: @tags, each_serializer: REST::TagSerializer + end + + private + + def set_tags + @tags = TrendingTags.get(limit_param(10)) + end +end diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb index 742c97cdb..d548072a8 100644 --- a/app/controllers/settings/preferences_controller.rb +++ b/app/controllers/settings/preferences_controller.rb @@ -56,7 +56,7 @@ class Settings::PreferencesController < Settings::BaseController :setting_advanced_layout, :setting_use_blurhash, :setting_use_pending_items, - notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account), + notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag), interactions: %i(must_be_follower must_be_following must_be_following_dm) ) end diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb index 0bda25974..506429e10 100644 --- a/app/helpers/admin/filter_helper.rb +++ b/app/helpers/admin/filter_helper.rb @@ -5,15 +5,16 @@ module Admin::FilterHelper REPORT_FILTERS = %i(resolved account_id target_account_id).freeze INVITE_FILTER = %i(available expired).freeze CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze - TAGS_FILTERS = %i(hidden).freeze + TAGS_FILTERS = %i(context review).freeze INSTANCES_FILTERS = %i(limited by_domain).freeze FOLLOWERS_FILTERS = %i(relationship status by_domain activity order).freeze FILTERS = ACCOUNT_FILTERS + REPORT_FILTERS + INVITE_FILTER + CUSTOM_EMOJI_FILTERS + TAGS_FILTERS + INSTANCES_FILTERS + FOLLOWERS_FILTERS def filter_link_to(text, link_to_params, link_class_params = link_to_params) - new_url = filtered_url_for(link_to_params) + new_url = filtered_url_for(link_to_params) new_class = filtered_url_for(link_class_params) + link_to text, new_url, class: filter_link_class(new_class) end diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb index 9ab3e2bbd..8abce5f05 100644 --- a/app/mailers/admin_mailer.rb +++ b/app/mailers/admin_mailer.rb @@ -24,4 +24,14 @@ class AdminMailer < ApplicationMailer mail to: @me.user_email, subject: I18n.t('admin_mailer.new_pending_account.subject', instance: @instance, username: @account.username) end end + + def new_trending_tag(recipient, tag) + @tag = tag + @me = recipient + @instance = Rails.configuration.x.local_domain + + locale_for_account(@me) do + mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name) + end + end end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 83134d41a..c1b873da6 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -2,5 +2,16 @@ class ApplicationRecord < ActiveRecord::Base self.abstract_class = true + include Remotable + + def boolean_with_default(key, default_value) + value = attributes[key] + + if value.nil? + default_value + else + value + end + end end diff --git a/app/models/tag.rb b/app/models/tag.rb index c7f0af86d..6a02581fa 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -3,11 +3,16 @@ # # Table name: tags # -# id :bigint(8) not null, primary key -# name :string default(""), not null -# created_at :datetime not null -# updated_at :datetime not null -# score :integer +# id :bigint(8) not null, primary key +# name :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# score :integer +# usable :boolean +# trendable :boolean +# listable :boolean +# reviewed_at :datetime +# requested_review_at :datetime # class Tag < ApplicationRecord @@ -22,16 +27,17 @@ class Tag < ApplicationRecord HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i } + validate :validate_name_change, if: -> { !new_record? && name_changed? } - scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) } - scope :hidden, -> { where(account_tag_stats: { hidden: true }) } + scope :reviewed, -> { where.not(reviewed_at: nil) } + scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) } + scope :discoverable, -> { where.not(listable: false).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) } scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) } delegate :accounts_count, :accounts_count=, :increment_count!, :decrement_count!, - :hidden?, to: :account_tag_stat after_save :save_account_tag_stat @@ -48,6 +54,40 @@ class Tag < ApplicationRecord name end + def usable + boolean_with_default('usable', true) + end + + alias usable? usable + + def listable + boolean_with_default('listable', true) + end + + alias listable? listable + + def trendable + boolean_with_default('trendable', false) + end + + alias trendable? trendable + + def requires_review? + reviewed_at.nil? + end + + def reviewed? + reviewed_at.present? + end + + def requested_review? + requested_review_at.present? + end + + def trending? + TrendingTags.trending?(self) + end + def history days = [] @@ -117,4 +157,8 @@ class Tag < ApplicationRecord return unless account_tag_stat&.changed? account_tag_stat.save end + + def validate_name_change + errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero? + end end diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb index 211c8f1dc..e9b9b25e3 100644 --- a/app/models/trending_tags.rb +++ b/app/models/trending_tags.rb @@ -10,20 +10,28 @@ class TrendingTags include Redisable def record_use!(tag, account, at_time = Time.now.utc) - return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot? + return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?) increment_historical_use!(tag.id, at_time) increment_unique_use!(tag.id, account.id, at_time) - increment_vote!(tag.id, at_time) + increment_vote!(tag, at_time) end - def get(limit) - key = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}" - tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i) - tags = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag } + def get(limit, filtered: true) + tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i) + + tags = Tag.where(id: tag_ids) + tags = tags.where(trendable: true) if filtered + tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag } + tag_ids.map { |tag_id| tags[tag_id] }.compact end + def trending?(tag) + rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id) + rank.present? && rank <= 10 + end + private def increment_historical_use!(tag_id, at_time) @@ -38,33 +46,27 @@ class TrendingTags redis.expire(key, EXPIRE_HISTORY_AFTER) end - def increment_vote!(tag_id, at_time) + def increment_vote!(tag, at_time) key = "#{KEY}:#{at_time.beginning_of_day.to_i}" - expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f + expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f + observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f if expected > observed || observed < THRESHOLD - redis.zrem(key, tag_id.to_s) + redis.zrem(key, tag.id) else - score = ((observed - expected)**2) / expected - added = redis.zadd(key, score, tag_id.to_s) - bump_tag_score!(tag_id) if added + score = ((observed - expected)**2) / expected + old_rank = redis.zrevrank(key, tag.id) + + redis.zadd(key, score, tag.id) + request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review? end redis.expire(key, EXPIRE_TRENDS_AFTER) end - def bump_tag_score!(tag_id) - Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1') - end - - def disallowed_hashtags - return @disallowed_hashtags if defined?(@disallowed_hashtags) - - @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags - @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String - @disallowed_hashtags = @disallowed_hashtags.map(&:downcase) + def request_review!(tag) + User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? } end end end diff --git a/app/models/user.rb b/app/models/user.rb index 6806c0362..b83e26af3 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -207,6 +207,10 @@ class User < ApplicationRecord settings.notification_emails['pending_account'] end + def allows_trending_tag_emails? + settings.notification_emails['trending_tag'] + end + def hides_network? @hides_network ||= settings.hide_network end diff --git a/app/policies/tag_policy.rb b/app/policies/tag_policy.rb index c63de01db..aaf70fcab 100644 --- a/app/policies/tag_policy.rb +++ b/app/policies/tag_policy.rb @@ -5,11 +5,11 @@ class TagPolicy < ApplicationPolicy staff? end - def hide? + def show? staff? end - def unhide? + def update? staff? end end diff --git a/app/validators/disallowed_hashtags_validator.rb b/app/validators/disallowed_hashtags_validator.rb index ee06b20f6..d745b767f 100644 --- a/app/validators/disallowed_hashtags_validator.rb +++ b/app/validators/disallowed_hashtags_validator.rb @@ -4,24 +4,7 @@ class DisallowedHashtagsValidator < ActiveModel::Validator def validate(status) return unless status.local? && !status.reblog? - @status = status - tags = select_tags - - status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size)) unless tags.empty? - end - - private - - def select_tags - tags = Extractor.extract_hashtags(@status.text) - tags.keep_if { |tag| disallowed_hashtags.include? tag.downcase } - end - - def disallowed_hashtags - return @disallowed_hashtags if @disallowed_hashtags - - @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags - @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String - @disallowed_hashtags = @disallowed_hashtags.map(&:downcase) + disallowed_hashtags = Tag.matching_name(Extractor.extract_hashtags(status.text)).reject(&:usable?) + status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_hashtags.map(&:name).join(', '), count: disallowed_hashtags.size)) unless disallowed_hashtags.empty? end end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 77cc1a2a0..910896075 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -107,5 +107,5 @@ %ul - @trending_hashtags.each do |tag| %li - = link_to "##{tag.name}", web_url("timelines/tag/#{tag.name}") + = link_to content_tag(:span, "##{tag.name}", class: !tag.trendable? && !tag.reviewed? ? 'warning-hint' : (!tag.trendable? ? 'negative-hint' : nil)), admin_tag_path(tag.id) %span.pull-right= number_with_delimiter(tag.history[0][:accounts].to_i) diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml index 961b83f93..91af8e492 100644 --- a/app/views/admin/tags/_tag.html.haml +++ b/app/views/admin/tags/_tag.html.haml @@ -1,12 +1,16 @@ -%tr - %td - = link_to explore_hashtag_path(tag) do +.directory__tag + = link_to admin_tag_path(tag.id) do + %h4 = fa_icon 'hashtag' = tag.name - %td - = t('directories.people', count: tag.accounts_count) - %td - - if tag.hidden? - = table_link_to 'eye', t('admin.tags.unhide'), unhide_admin_tag_path(tag.id, **@filter_params), method: :post - - else - = table_link_to 'eye-slash', t('admin.tags.hide'), hide_admin_tag_path(tag.id, **@filter_params), method: :post + + %small + = t('admin.tags.in_directory', count: tag.accounts_count) + • + = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts]) + + - if tag.trending? + = fa_icon 'fire fw' + = t('admin.tags.trending_right_now') + + .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml index 4ba395860..5e4ee21f5 100644 --- a/app/views/admin/tags/index.html.haml +++ b/app/views/admin/tags/index.html.haml @@ -3,17 +3,19 @@ .filters .filter-subset - %strong= t('admin.reports.status') + %strong= t('admin.tags.context') %ul - %li= filter_link_to t('admin.tags.visible'), hidden: nil - %li= filter_link_to t('admin.tags.hidden'), hidden: '1' + %li= filter_link_to t('generic.all'), context: nil + %li= filter_link_to t('admin.tags.directory'), context: 'directory' -.table-wrapper - %table.table - %thead - %tr - %th= t('admin.tags.name') - %th= t('admin.tags.accounts') - %th - %tbody - = render @tags + .filter-subset + %strong= t('admin.tags.review') + %ul + %li= filter_link_to t('generic.all'), review: nil + %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed' + %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review' + +%hr.spacer/ + += render @tags += paginate @tags diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml new file mode 100644 index 000000000..27c8dc92b --- /dev/null +++ b/app/views/admin/tags/show.html.haml @@ -0,0 +1,16 @@ +- content_for :page_title do + = "##{@tag.name}" + += simple_form_for @tag, url: admin_tag_path(@tag.id) do |f| + = render 'shared/error_messages', object: @tag + + .fields-group + = f.input :name, wrapper: :with_block_label + + .fields-group + = f.input :usable, as: :boolean, wrapper: :with_label + = f.input :trendable, as: :boolean, wrapper: :with_label + = f.input :listable, as: :boolean, wrapper: :with_label + + .actions + = f.button :button, t('generic.save_changes'), type: :submit diff --git a/app/views/admin_mailer/new_trending_tag.text.erb b/app/views/admin_mailer/new_trending_tag.text.erb new file mode 100644 index 000000000..f3087df37 --- /dev/null +++ b/app/views/admin_mailer/new_trending_tag.text.erb @@ -0,0 +1,5 @@ +<%= raw t('application_mailer.salutation', name: display_name(@me)) %> + +<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %> + +<%= raw t('application_mailer.view')%> <%= admin_tags_url(review: 'pending_review') %> diff --git a/app/views/settings/preferences/notifications/show.html.haml b/app/views/settings/preferences/notifications/show.html.haml index acc646fc3..f666ae4ff 100644 --- a/app/views/settings/preferences/notifications/show.html.haml +++ b/app/views/settings/preferences/notifications/show.html.haml @@ -15,6 +15,7 @@ - if current_user.staff? = ff.input :report, as: :boolean, wrapper: :with_label = ff.input :pending_account, as: :boolean, wrapper: :with_label + = ff.input :trending_tag, as: :boolean, wrapper: :with_label .fields-group = f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff| diff --git a/config/locales/en.yml b/config/locales/en.yml index 6c1a34300..9b62aac3a 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -483,13 +483,14 @@ en: title: Account statuses with_media: With media tags: - accounts: Accounts - hidden: Hidden - hide: Hide from directory - name: Hashtag + context: Context + directory: In directory + in_directory: "%{count} in directory" + review: Review status + reviewed: Reviewed title: Hashtags - unhide: Show in directory - visible: Visible + trending_right_now: Trending right now + unique_uses_today: "%{count} posting today" title: Administration warning_presets: add_new: Add new @@ -505,6 +506,9 @@ en: body: "%{reporter} has reported %{target}" body_remote: Someone from %{domain} has reported %{target} subject: New report for %{instance} (#%{id}) + new_trending_tag: + body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.' + subject: New hashtag up for review on %{instance} (#%{name}) appearance: advanced_web_interface: Advanced web interface advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.' @@ -939,6 +943,8 @@ en: pinned: Pinned toot reblogged: boosted sensitive_content: Sensitive content + tags: + does_not_match_previous_name: does not match the previous name terms: body_html: |