Compare commits

...

9 Commits

Author SHA1 Message Date
Claire ee66f5790f
Fix unbounded recursion in account discovery (v3.5 backport) (#22026)
* Fix trying to fetch posts from other users when fetching featured posts

* Rate-limit discovery of new subdomains

* Put a limit on recursively discovering new accounts
2022-12-15 19:21:17 +01:00
Claire 696f7b3608 Bump version to 3.5.5 2022-11-14 22:26:24 +01:00
Claire b22e1476ca Fix nodes order being sometimes mangled when rewriting emoji (#20677)
* Fix front-end emoji tests

* Fix nodes order being sometimes mangled when rewriting emoji
2022-11-14 22:20:29 +01:00
Claire 105ab82425 Bump version to 3.5.4 2022-11-14 20:09:16 +01:00
Claire 2dd8f977e8 Fix emoji substitution not applying only to text nodes in backend code
Signed-off-by: Claire <claire.github-309c@sitedethib.com>
2022-11-14 11:20:41 +01:00
Claire 2db06e1d08 Fix emoji substitution not applying only to text nodes in Web UI
Signed-off-by: Claire <claire.github-309c@sitedethib.com>
2022-11-14 11:20:41 +01:00
Eugen Rochko 063579373e Fix rate limiting for paths with formats 2022-11-14 11:20:41 +01:00
Pierre Bourdon 1659788de4 blurhash_transcoder: prevent out-of-bound reads with <8bpp images (#20388)
The Blurhash library used by Mastodon requires an input encoded as 24
bits raw RGB data. The conversion to raw RGB using Imagemagick did not
previously specify the desired bit depth. In some situations, this leads
Imagemagick to output in a pixel format using less bpp than expected.
This then manifested as segfaults of the Sidekiq process due to
out-of-bounds read, or potentially a (highly noisy) memory infoleak.

Fixes #19235.
2022-11-14 11:20:41 +01:00
Claire 47eaf85f02 Fix crash when a remote Flag activity mentions a private post (#18760)
* Add tests

* Fix crash when a remote Flag activity mentions a private post
2022-11-14 11:20:41 +01:00
25 changed files with 422 additions and 145 deletions

View File

@ -3,6 +3,23 @@ Changelog
All notable changes to this project will be documented in this file.
## [3.4.5] - 2022-11-14
## Fixed
- Fix nodes order being sometimes mangled when rewriting emoji ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20677))
## [3.5.4] - 2022-11-14
### Fixed
- Fix error when a remote report includes a private post the server has no access to ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18760))
### Security
- Fix emoji substitution not applying only to text nodes in backend code ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20641))
- Fix emoji substitution not applying only to text nodes in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20640))
- Fix rate limiting for paths with formats ([Gargron](https://github.com/mastodon/mastodon/pull/20675))
- Fix out-of-bound reads in blurhash transcoder ([delroth](https://github.com/mastodon/mastodon/pull/20388))
## [3.5.3] - 2022-05-26
### Added

View File

@ -66,6 +66,7 @@ gem 'oj', '~> 3.13'
gem 'ox', '~> 2.14'
gem 'parslet'
gem 'posix-spawn'
gem 'public_suffix', '~> 4.0.7'
gem 'pundit', '~> 2.2'
gem 'premailer-rails'
gem 'rack-attack', '~> 6.6'

View File

@ -803,6 +803,7 @@ DEPENDENCIES
private_address_check (~> 0.5)
pry-byebug (~> 3.9)
pry-rails (~> 0.3)
public_suffix (~> 4.0.7)
puma (~> 5.6)
pundit (~> 2.2)
rack (~> 2.2.3)

View File

@ -11,8 +11,8 @@ describe('emoji', () => {
});
it('works with unclosed tags', () => {
expect(emojify('hello>')).toEqual('hello>');
expect(emojify('<hello')).toEqual('<hello');
expect(emojify('hello>')).toEqual('hello&gt;');
expect(emojify('<hello')).toEqual('');
});
it('works with unclosed shortcodes', () => {
@ -22,23 +22,23 @@ describe('emoji', () => {
it('does unicode', () => {
expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).toEqual(
'<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg" />');
'<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg">');
expect(emojify('👨‍👩‍👧‍👧')).toEqual(
'<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg" />');
expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg" />');
'<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg">');
expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg">');
expect(emojify('\u2757')).toEqual(
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
});
it('does multiple unicode', () => {
expect(emojify('\u2757 #\uFE0F\u20E3')).toEqual(
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" />');
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
expect(emojify('\u2757#\uFE0F\u20E3')).toEqual(
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" />');
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
expect(emojify('\u2757 #\uFE0F\u20E3 \u2757')).toEqual(
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" /> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
'<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
expect(emojify('foo \u2757 #\uFE0F\u20E3 bar')).toEqual(
'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" /> bar');
'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> bar');
});
it('ignores unicode inside of tags', () => {
@ -46,16 +46,16 @@ describe('emoji', () => {
});
it('does multiple emoji properly (issue 5188)', () => {
expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />');
expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />');
expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
});
it('does an emoji that has no shortcode', () => {
expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg" />');
expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg">');
});
it('does an emoji whose filename is irregular', () => {
expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg" />');
expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg">');
});
it('avoid emojifying on invisible text', () => {
@ -67,26 +67,26 @@ describe('emoji', () => {
it('avoid emojifying on invisible text with nested tags', () => {
expect(emojify('<span class="invisible">😄<span class="foo">bar</span>😴</span>😇'))
.toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
.toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
expect(emojify('<span class="invisible">😄<span class="invisible">😕</span>😴</span>😇'))
.toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
expect(emojify('<span class="invisible">😄<br/>😴</span>😇'))
.toEqual('<span class="invisible">😄<br/>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
.toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
expect(emojify('<span class="invisible">😄<br>😴</span>😇'))
.toEqual('<span class="invisible">😄<br>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
});
it('skips the textual presentation VS15 character', () => {
expect(emojify('✴︎')) // This is U+2734 EIGHT POINTED BLACK STAR then U+FE0E VARIATION SELECTOR-15
.toEqual('<img draggable="false" class="emojione" alt="✴" title=":eight_pointed_black_star:" src="/emoji/2734_border.svg" />');
.toEqual('<img draggable="false" class="emojione" alt="✴" title=":eight_pointed_black_star:" src="/emoji/2734_border.svg">');
});
it('does an simple emoji properly', () => {
expect(emojify('♀♂'))
.toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg" /><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg" />');
.toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg"><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg">');
});
it('does an emoji containing ZWJ properly', () => {
expect(emojify('💂‍♀️💂‍♂️'))
.toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg" /><img draggable="false" class="emojione" alt="💂\u200D♂" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg" />');
.toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg"><img draggable="false" class="emojione" alt="💂\u200D♂" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg">');
});
});
});

View File

@ -19,15 +19,26 @@ const emojiFilename = (filename) => {
return borderedEmoji.includes(filename) ? (filename + '_border') : filename;
};
const emojify = (str, customEmojis = {}) => {
const tagCharsWithoutEmojis = '<&';
const tagCharsWithEmojis = Object.keys(customEmojis).length ? '<&:' : '<&';
let rtn = '', tagChars = tagCharsWithEmojis, invisible = 0;
const domParser = new DOMParser();
const emojifyTextNode = (node, customEmojis) => {
let str = node.textContent;
const fragment = new DocumentFragment();
for (;;) {
let match, i = 0, tag;
while (i < str.length && (tag = tagChars.indexOf(str[i])) === -1 && (invisible || !(match = trie.search(str.slice(i))))) {
i += str.codePointAt(i) < 65536 ? 1 : 2;
let match, i = 0;
if (customEmojis === null) {
while (i < str.length && !(match = trie.search(str.slice(i)))) {
i += str.codePointAt(i) < 65536 ? 1 : 2;
}
} else {
while (i < str.length && str[i] !== ':' && !(match = trie.search(str.slice(i)))) {
i += str.codePointAt(i) < 65536 ? 1 : 2;
}
}
let rend, replacement = '';
if (i === str.length) {
break;
@ -35,8 +46,6 @@ const emojify = (str, customEmojis = {}) => {
if (!(() => {
rend = str.indexOf(':', i + 1) + 1;
if (!rend) return false; // no pair of ':'
const lt = str.indexOf('<', i + 1);
if (!(lt === -1 || lt >= rend)) return false; // tag appeared before closing ':'
const shortname = str.slice(i, rend);
// now got a replacee as ':shortname:'
// if you want additional emoji handler, add statements below which set replacement and return true.
@ -47,29 +56,6 @@ const emojify = (str, customEmojis = {}) => {
}
return false;
})()) rend = ++i;
} else if (tag >= 0) { // <, &
rend = str.indexOf('>;'[tag], i + 1) + 1;
if (!rend) {
break;
}
if (tag === 0) {
if (invisible) {
if (str[i + 1] === '/') { // closing tag
if (!--invisible) {
tagChars = tagCharsWithEmojis;
}
} else if (str[rend - 2] !== '/') { // opening tag
invisible++;
}
} else {
if (str.startsWith('<span class="invisible">', i)) {
// avoid emojifying on invisible text
invisible = 1;
tagChars = tagCharsWithoutEmojis;
}
}
}
i = rend;
} else { // matched to unicode emoji
const { filename, shortCode } = unicodeMapping[match];
const title = shortCode ? `:${shortCode}:` : '';
@ -80,10 +66,43 @@ const emojify = (str, customEmojis = {}) => {
rend += 1;
}
}
rtn += str.slice(0, i) + replacement;
fragment.append(document.createTextNode(str.slice(0, i)));
if (replacement) {
fragment.append(domParser.parseFromString(replacement, 'text/html').documentElement.getElementsByTagName('img')[0]);
}
node.textContent = str.slice(0, i);
str = str.slice(rend);
}
return rtn + str;
fragment.append(document.createTextNode(str));
node.parentElement.replaceChild(fragment, node);
};
const emojifyNode = (node, customEmojis) => {
for (const child of node.childNodes) {
switch(child.nodeType) {
case Node.TEXT_NODE:
emojifyTextNode(child, customEmojis);
break;
case Node.ELEMENT_NODE:
if (!child.classList.contains('invisible'))
emojifyNode(child, customEmojis);
break;
}
}
};
const emojify = (str, customEmojis = {}) => {
const wrapper = document.createElement('div');
wrapper.innerHTML = str;
if (!Object.keys(customEmojis).length)
customEmojis = null;
emojifyNode(wrapper, customEmojis);
return wrapper.innerHTML;
};
export default emojify;

View File

@ -222,7 +222,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
return if tag['href'].blank?
account = account_from_uri(tag['href'])
account = ActivityPub::FetchRemoteAccountService.new.call(tag['href']) if account.nil?
account = ActivityPub::FetchRemoteAccountService.new.call(tag['href'], request_id: @options[:request_id]) if account.nil?
return if account.nil?

View File

@ -18,7 +18,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
def update_account
return reject_payload! if @account.uri != object_uri
ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true)
ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true, request_id: @options[:request_id])
end
def update_status
@ -28,6 +28,6 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
return if @status.nil?
ActivityPub::ProcessStatusUpdateService.new.call(@status, @object)
ActivityPub::ProcessStatusUpdateService.new.call(@status, @object, request_id: @options[:request_id])
end
end

View File

@ -23,48 +23,40 @@ class EmojiFormatter
def to_s
return html if custom_emojis.empty? || html.blank?
i = -1
tag_open_index = nil
inside_shortname = false
shortname_start_index = -1
invisible_depth = 0
last_index = 0
result = ''.dup
tree = Nokogiri::HTML.fragment(html)
tree.xpath('./text()|.//text()[not(ancestor[@class="invisible"])]').to_a.each do |node|
i = -1
inside_shortname = false
shortname_start_index = -1
last_index = 0
text = node.content
result = Nokogiri::XML::NodeSet.new(tree.document)
while i + 1 < html.size
i += 1
while i + 1 < text.size
i += 1
if invisible_depth.zero? && inside_shortname && html[i] == ':'
inside_shortname = false
shortcode = html[shortname_start_index + 1..i - 1]
char_after = html[i + 1]
if inside_shortname && text[i] == ':'
inside_shortname = false
shortcode = text[shortname_start_index + 1..i - 1]
char_after = text[i + 1]
next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode])
next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode])
result << html[last_index..shortname_start_index - 1] if shortname_start_index.positive?
result << image_for_emoji(shortcode, emoji)
last_index = i + 1
elsif tag_open_index && html[i] == '>'
tag = html[tag_open_index..i]
tag_open_index = nil
result << Nokogiri::XML::Text.new(text[last_index..shortname_start_index - 1], tree.document) if shortname_start_index.positive?
result << Nokogiri::HTML.fragment(image_for_emoji(shortcode, emoji))
if invisible_depth.positive?
invisible_depth += count_tag_nesting(tag)
elsif tag == '<span class="invisible">'
invisible_depth = 1
last_index = i + 1
elsif text[i] == ':' && (i.zero? || !DISALLOWED_BOUNDING_REGEX.match?(text[i - 1]))
inside_shortname = true
shortname_start_index = i
end
elsif html[i] == '<'
tag_open_index = i
inside_shortname = false
elsif !tag_open_index && html[i] == ':' && (i.zero? || !DISALLOWED_BOUNDING_REGEX.match?(html[i - 1]))
inside_shortname = true
shortname_start_index = i
end
result << Nokogiri::XML::Text.new(text[last_index..-1], tree.document)
node.replace(result)
end
result << html[last_index..-1]
result.html_safe # rubocop:disable Rails/OutputSafety
tree.to_html.html_safe # rubocop:disable Rails/OutputSafety
end
private

View File

@ -3,11 +3,24 @@
module DomainMaterializable
extend ActiveSupport::Concern
include Redisable
included do
after_create_commit :refresh_instances_view
end
def refresh_instances_view
Instance.refresh unless domain.nil? || Instance.where(domain: domain).exists?
return if domain.nil? || Instance.exists?(domain: domain)
Instance.refresh
count_unique_subdomains!
end
def count_unique_subdomains!
second_and_top_level_domain = PublicSuffix.domain(domain, ignore_private: true)
with_redis do |redis|
redis.pfadd("unique_subdomains_for:#{second_and_top_level_domain}", domain)
redis.expire("unique_subdomains_for:#{second_and_top_level_domain}", 1.minute.seconds)
end
end
end

View File

@ -3,10 +3,11 @@
class ActivityPub::FetchFeaturedCollectionService < BaseService
include JsonLdHelper
def call(account)
def call(account, **options)
return if account.featured_collection_url.blank? || account.suspended? || account.local?
@account = account
@options = options
@json = fetch_resource(@account.featured_collection_url, true, local_follower)
return unless supported_context?(@json)
@ -38,9 +39,9 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
def process_items(items)
status_ids = items.filter_map do |item|
uri = value_or_id(item)
next if ActivityPub::TagManager.instance.local_uri?(uri)
next if ActivityPub::TagManager.instance.local_uri?(uri) || invalid_origin?(uri)
status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower)
status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower, expected_actor_uri: @account.uri, request_id: @options[:request_id])
next unless status&.account_id == @account.id
status.id

View File

@ -8,7 +8,7 @@ class ActivityPub::FetchRemoteAccountService < BaseService
SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
# Does a WebFinger roundtrip on each call, unless `only_key` is true
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false)
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, request_id: nil)
return if domain_not_allowed?(uri)
return ActivityPub::TagManager.instance.uri_to_resource(uri, Account) if ActivityPub::TagManager.instance.local_uri?(uri)
@ -28,7 +28,7 @@ class ActivityPub::FetchRemoteAccountService < BaseService
return unless only_key || verified_webfinger?
ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key)
ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key, request_id: request_id)
rescue Oj::ParseError
nil
end

View File

@ -4,7 +4,8 @@ class ActivityPub::FetchRemoteStatusService < BaseService
include JsonLdHelper
# Should be called when uri has already been checked for locality
def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil)
def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
@request_id = request_id
@json = begin
if prefetched_body.nil?
fetch_resource(uri, id, on_behalf_of)
@ -30,6 +31,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
end
return if activity_json.nil? || object_uri.nil? || !trustworthy_attribution?(@json['id'], actor_uri)
return if expected_actor_uri.present? && actor_uri != expected_actor_uri
return ActivityPub::TagManager.instance.uri_to_resource(object_uri, Status) if ActivityPub::TagManager.instance.local_uri?(object_uri)
actor = account_from_uri(actor_uri)
@ -40,7 +42,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
# activity as an update rather than create
activity_json['type'] = 'Update' if equals_or_includes_any?(activity_json['type'], %w(Create)) && Status.where(uri: object_uri, account_id: actor.id).exists?
ActivityPub::Activity.factory(activity_json, actor).perform
ActivityPub::Activity.factory(activity_json, actor, request_id: request_id).perform
end
private
@ -52,7 +54,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
def account_from_uri(uri)
actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account)
actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true) if actor.nil? || actor.possibly_stale?
actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true, request_id: @request_id) if actor.nil? || actor.possibly_stale?
actor
end

View File

@ -6,6 +6,9 @@ class ActivityPub::ProcessAccountService < BaseService
include Redisable
include Lockable
SUBDOMAINS_RATELIMIT = 10
DISCOVERIES_PER_REQUEST = 400
# Should be called with confirmed valid JSON
# and WebFinger-resolved username and domain
def call(username, domain, json, options = {})
@ -15,9 +18,12 @@ class ActivityPub::ProcessAccountService < BaseService
@json = json
@uri = @json['id']
@username = username
@domain = domain
@domain = TagManager.instance.normalize_domain(domain)
@collections = {}
# The key does not need to be unguessable, it just needs to be somewhat unique
@options[:request_id] ||= "#{Time.now.utc.to_i}-#{username}@#{domain}"
with_lock("process_account:#{@uri}") do
@account = Account.remote.find_by(uri: @uri) if @options[:only_key]
@account ||= Account.find_remote(@username, @domain)
@ -25,7 +31,18 @@ class ActivityPub::ProcessAccountService < BaseService
@old_protocol = @account&.protocol
@suspension_changed = false
create_account if @account.nil?
if @account.nil?
with_redis do |redis|
return nil if redis.pfcount("unique_subdomains_for:#{PublicSuffix.domain(@domain, ignore_private: true)}") >= SUBDOMAINS_RATELIMIT
discoveries = redis.incr("discovery_per_request:#{@options[:request_id]}")
redis.expire("discovery_per_request:#{@options[:request_id]}", 5.minutes.seconds)
return nil if discoveries > DISCOVERIES_PER_REQUEST
end
create_account
end
update_account
process_tags
@ -149,7 +166,7 @@ class ActivityPub::ProcessAccountService < BaseService
end
def check_featured_collection!
ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id)
ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'request_id' => @options[:request_id] })
end
def check_links!
@ -249,7 +266,7 @@ class ActivityPub::ProcessAccountService < BaseService
def moved_account
account = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account)
account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true)
account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true, request_id: @options[:request_id])
account
end

View File

@ -5,7 +5,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
include Redisable
include Lockable
def call(status, json)
def call(status, json, request_id: nil)
raise ArgumentError, 'Status has unsaved changes' if status.changed?
@json = json
@ -15,6 +15,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
@account = status.account
@media_attachments_changed = false
@poll_changed = false
@request_id = request_id
# Only native types can be updated at the moment
return @status if !expected_type? || already_updated_more_recently?
@ -185,7 +186,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
next if href.blank?
account = ActivityPub::TagManager.instance.uri_to_resource(href, Account)
account ||= ActivityPub::FetchRemoteAccountService.new.call(href)
account ||= ActivityPub::FetchRemoteAccountService.new.call(href, request_id: @request_id)
next if account.nil?

View File

@ -57,7 +57,16 @@ class ReportService < BaseService
end
def reported_status_ids
AccountStatusesFilter.new(@target_account, @source_account).results.with_discarded.find(Array(@status_ids)).pluck(:id)
return AccountStatusesFilter.new(@target_account, @source_account).results.with_discarded.find(Array(@status_ids)).pluck(:id) if @source_account.local?
# If the account making reports is remote, it is likely anonymized so we have to relax the requirements for attaching statuses.
domain = @source_account.domain.to_s.downcase
has_followers = @target_account.followers.where(Account.arel_table[:domain].lower.eq(domain)).exists?
visibility = has_followers ? %i(public unlisted private) : %i(public unlisted)
scope = @target_account.statuses.with_discarded
scope.merge!(scope.where(visibility: visibility).or(scope.where('EXISTS (SELECT 1 FROM mentions m JOIN accounts a ON m.account_id = a.id WHERE lower(a.domain) = ?)', domain)))
# Allow missing posts to not drop reports that include e.g. a deleted post
scope.where(id: Array(@status_ids)).pluck(:id)
end
def payload

View File

@ -5,8 +5,10 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker
sidekiq_options queue: 'pull', lock: :until_executed
def perform(account_id)
ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id))
def perform(account_id, options = {})
options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys)
ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id), **options)
rescue ActiveRecord::RecordNotFound
true
end

View File

@ -8,7 +8,7 @@ image:
# built from the most recent commit
#
# tag: latest
tag: v3.5.2
tag: v3.5.5
# use `Always` when using `latest` tag
pullPolicy: IfNotPresent

View File

@ -17,6 +17,18 @@ class Rack::Attack
@remote_ip ||= (@env["action_dispatch.remote_ip"] || ip).to_s
end
def throttleable_remote_ip
@throttleable_remote_ip ||= begin
ip = IPAddr.new(remote_ip)
if ip.ipv6?
ip.mask(64)
else
ip
end
end.to_s
end
def authenticated_user_id
authenticated_token&.resource_owner_id
end
@ -29,6 +41,10 @@ class Rack::Attack
path.start_with?('/api')
end
def path_matches?(other_path)
/\A#{Regexp.escape(other_path)}(\..*)?\z/ =~ path
end
def web_request?
!api_request?
end
@ -51,19 +67,19 @@ class Rack::Attack
end
throttle('throttle_unauthenticated_api', limit: 300, period: 5.minutes) do |req|
req.remote_ip if req.api_request? && req.unauthenticated?
req.throttleable_remote_ip if req.api_request? && req.unauthenticated?
end
throttle('throttle_api_media', limit: 30, period: 30.minutes) do |req|
req.authenticated_user_id if req.post? && req.path.match?('^/api/v\d+/media')
req.authenticated_user_id if req.post? && req.path.match?(/\A\/api\/v\d+\/media\z/i)
end
throttle('throttle_media_proxy', limit: 30, period: 10.minutes) do |req|
req.remote_ip if req.path.start_with?('/media_proxy')
req.throttleable_remote_ip if req.path.start_with?('/media_proxy')
end
throttle('throttle_api_sign_up', limit: 5, period: 30.minutes) do |req|
req.remote_ip if req.post? && req.path == '/api/v1/accounts'
req.throttleable_remote_ip if req.post? && req.path == '/api/v1/accounts'
end
throttle('throttle_authenticated_paging', limit: 300, period: 15.minutes) do |req|
@ -71,39 +87,34 @@ class Rack::Attack
end
throttle('throttle_unauthenticated_paging', limit: 300, period: 15.minutes) do |req|
req.remote_ip if req.paging_request? && req.unauthenticated?
req.throttleable_remote_ip if req.paging_request? && req.unauthenticated?
end
API_DELETE_REBLOG_REGEX = /\A\/api\/v1\/statuses\/[\d]+\/unreblog/.freeze
API_DELETE_STATUS_REGEX = /\A\/api\/v1\/statuses\/[\d]+/.freeze
API_DELETE_REBLOG_REGEX = /\A\/api\/v1\/statuses\/[\d]+\/unreblog\z/.freeze
API_DELETE_STATUS_REGEX = /\A\/api\/v1\/statuses\/[\d]+\z/.freeze
throttle('throttle_api_delete', limit: 30, period: 30.minutes) do |req|
req.authenticated_user_id if (req.post? && req.path.match?(API_DELETE_REBLOG_REGEX)) || (req.delete? && req.path.match?(API_DELETE_STATUS_REGEX))
end
throttle('throttle_sign_up_attempts/ip', limit: 25, period: 5.minutes) do |req|
if req.post? && req.path == '/auth'
addr = req.remote_ip
addr = IPAddr.new(addr) if addr.is_a?(String)
addr = addr.mask(64) if addr.ipv6?
addr.to_s
end
req.throttleable_remote_ip if req.post? && req.path_matches?('/auth')
end
throttle('throttle_password_resets/ip', limit: 25, period: 5.minutes) do |req|
req.remote_ip if req.post? && req.path == '/auth/password'
req.throttleable_remote_ip if req.post? && req.path_matches?('/auth/password')
end
throttle('throttle_password_resets/email', limit: 5, period: 30.minutes) do |req|
req.params.dig('user', 'email').presence if req.post? && req.path == '/auth/password'
req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/password')
end
throttle('throttle_email_confirmations/ip', limit: 25, period: 5.minutes) do |req|
req.remote_ip if req.post? && %w(/auth/confirmation /api/v1/emails/confirmations).include?(req.path)
req.throttleable_remote_ip if req.post? && (req.path_matches?('/auth/confirmation') || req.path == '/api/v1/emails/confirmations')
end
throttle('throttle_email_confirmations/email', limit: 5, period: 30.minutes) do |req|
if req.post? && req.path == '/auth/password'
if req.post? && req.path_matches?('/auth/password')
req.params.dig('user', 'email').presence
elsif req.post? && req.path == '/api/v1/emails/confirmations'
req.authenticated_user_id
@ -111,11 +122,11 @@ class Rack::Attack
end
throttle('throttle_login_attempts/ip', limit: 25, period: 5.minutes) do |req|
req.remote_ip if req.post? && req.path == '/auth/sign_in'
req.throttleable_remote_ip if req.post? && req.path_matches?('/auth/sign_in')
end
throttle('throttle_login_attempts/email', limit: 25, period: 1.hour) do |req|
req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path == '/auth/sign_in'
req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in')
end
self.throttled_responder = lambda do |request|

View File

@ -47,7 +47,7 @@ Rails.application.routes.draw do
end
end
devise_for :users, path: 'auth', controllers: {
devise_for :users, path: 'auth', format: false, controllers: {
omniauth_callbacks: 'auth/omniauth_callbacks',
sessions: 'auth/sessions',
registrations: 'auth/registrations',
@ -182,7 +182,7 @@ Rails.application.routes.draw do
resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update]
get '/public', to: 'public_timelines#show', as: :public_timeline
get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy
get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy, format: false
resource :authorize_interaction, only: [:show, :create]
resource :share, only: [:show, :create]
@ -353,7 +353,7 @@ Rails.application.routes.draw do
get '/admin', to: redirect('/admin/dashboard', status: 302)
namespace :api do
namespace :api, format: false do
# OEmbed
get '/oembed', to: 'oembed#show', as: :oembed

View File

@ -44,7 +44,7 @@ services:
web:
build: .
image: tootsuite/mastodon
image: tootsuite/mastodon:v3.5.5
restart: always
env_file: .env.production
command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@ -65,7 +65,7 @@ services:
streaming:
build: .
image: tootsuite/mastodon
image: tootsuite/mastodon:v3.5.5
restart: always
env_file: .env.production
command: node ./streaming
@ -83,7 +83,7 @@ services:
sidekiq:
build: .
image: tootsuite/mastodon
image: tootsuite/mastodon:v3.5.5
restart: always
env_file: .env.production
command: bundle exec sidekiq

View File

@ -13,7 +13,7 @@ module Mastodon
end
def patch
3
5
end
def flags

View File

@ -5,7 +5,7 @@ module Paperclip
def make
return @file unless options[:style] == :small || options[:blurhash]
pixels = convert(':source RGB:-', source: File.expand_path(@file.path)).unpack('C*')
pixels = convert(':source -depth 8 RGB:-', source: File.expand_path(@file.path)).unpack('C*')
geometry = options.fetch(:file_geometry_parser).from_file(@file)
attachment.instance.blurhash = Blurhash.encode(geometry.width, geometry.height, pixels, **(options[:blurhash] || {}))

View File

@ -1,7 +1,7 @@
require 'rails_helper'
RSpec.describe ActivityPub::Activity::Flag do
let(:sender) { Fabricate(:account, domain: 'example.com', uri: 'http://example.com/account') }
let(:sender) { Fabricate(:account, username: 'example.com', domain: 'example.com', uri: 'http://example.com/actor') }
let(:flagged) { Fabricate(:account) }
let(:status) { Fabricate(:status, account: flagged, uri: 'foobar') }
let(:flag_id) { nil }
@ -23,16 +23,88 @@ RSpec.describe ActivityPub::Activity::Flag do
describe '#perform' do
subject { described_class.new(json, sender) }
before do
subject.perform
context 'when the reported status is public' do
before do
subject.perform
end
it 'creates a report' do
report = Report.find_by(account: sender, target_account: flagged)
expect(report).to_not be_nil
expect(report.comment).to eq 'Boo!!'
expect(report.status_ids).to eq [status.id]
end
end
it 'creates a report' do
report = Report.find_by(account: sender, target_account: flagged)
context 'when the reported status is private and should not be visible to the remote server' do
let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
expect(report).to_not be_nil
expect(report.comment).to eq 'Boo!!'
expect(report.status_ids).to eq [status.id]
before do
subject.perform
end
it 'creates a report with no attached status' do
report = Report.find_by(account: sender, target_account: flagged)
expect(report).to_not be_nil
expect(report.comment).to eq 'Boo!!'
expect(report.status_ids).to eq []
end
end
context 'when the reported status is private and the author has a follower on the remote instance' do
let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
let(:follower) { Fabricate(:account, domain: 'example.com', uri: 'http://example.com/users/account') }
before do
follower.follow!(flagged)
subject.perform
end
it 'creates a report with the attached status' do
report = Report.find_by(account: sender, target_account: flagged)
expect(report).to_not be_nil
expect(report.comment).to eq 'Boo!!'
expect(report.status_ids).to eq [status.id]
end
end
context 'when the reported status is private and the author mentions someone else on the remote instance' do
let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
let(:mentioned) { Fabricate(:account, domain: 'example.com', uri: 'http://example.com/users/account') }
before do
status.mentions.create(account: mentioned)
subject.perform
end
it 'creates a report with the attached status' do
report = Report.find_by(account: sender, target_account: flagged)
expect(report).to_not be_nil
expect(report.comment).to eq 'Boo!!'
expect(report.status_ids).to eq [status.id]
end
end
context 'when the reported status is private and the author mentions someone else on the local instance' do
let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }
let(:mentioned) { Fabricate(:account) }
before do
status.mentions.create(account: mentioned)
subject.perform
end
it 'creates a report with no attached status' do
report = Report.find_by(account: sender, target_account: flagged)
expect(report).to_not be_nil
expect(report.comment).to eq 'Boo!!'
expect(report.status_ids).to eq []
end
end
end

View File

@ -109,4 +109,98 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
end
end
end
context 'discovering many subdomains in a short timeframe' do
before do
stub_const 'ActivityPub::ProcessAccountService::SUBDOMAINS_RATELIMIT', 5
end
let(:subject) do
8.times do |i|
domain = "test#{i}.testdomain.com"
json = {
id: "https://#{domain}/users/1",
type: 'Actor',
inbox: "https://#{domain}/inbox",
}.with_indifferent_access
described_class.new.call('alice', domain, json)
end
end
it 'creates at least some accounts' do
expect { subject }.to change { Account.remote.count }.by_at_least(2)
end
it 'creates no more account than the limit allows' do
expect { subject }.to change { Account.remote.count }.by_at_most(5)
end
end
context 'accounts referencing other accounts' do
before do
stub_const 'ActivityPub::ProcessAccountService::DISCOVERIES_PER_REQUEST', 5
end
let(:payload) do
{
'@context': ['https://www.w3.org/ns/activitystreams'],
id: 'https://foo.test/users/1',
type: 'Person',
inbox: 'https://foo.test/inbox',
featured: 'https://foo.test/users/1/featured',
preferredUsername: 'user1',
}.with_indifferent_access
end
before do
8.times do |i|
actor_json = {
'@context': ['https://www.w3.org/ns/activitystreams'],
id: "https://foo.test/users/#{i}",
type: 'Person',
inbox: 'https://foo.test/inbox',
featured: "https://foo.test/users/#{i}/featured",
preferredUsername: "user#{i}",
}.with_indifferent_access
status_json = {
'@context': ['https://www.w3.org/ns/activitystreams'],
id: "https://foo.test/users/#{i}/status",
attributedTo: "https://foo.test/users/#{i}",
type: 'Note',
content: "@user#{i + 1} test",
tag: [
{
type: 'Mention',
href: "https://foo.test/users/#{i + 1}",
name: "@user#{i + 1 }",
}
],
to: [ 'as:Public', "https://foo.test/users/#{i + 1}" ]
}.with_indifferent_access
featured_json = {
'@context': ['https://www.w3.org/ns/activitystreams'],
id: "https://foo.test/users/#{i}/featured",
type: 'OrderedCollection',
totelItems: 1,
orderedItems: [status_json],
}.with_indifferent_access
webfinger = {
subject: "acct:user#{i}@foo.test",
links: [{ rel: 'self', href: "https://foo.test/users/#{i}" }],
}.with_indifferent_access
stub_request(:get, "https://foo.test/users/#{i}").to_return(status: 200, body: actor_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
stub_request(:get, "https://foo.test/users/#{i}/featured").to_return(status: 200, body: featured_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
stub_request(:get, "https://foo.test/users/#{i}/status").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
stub_request(:get, "https://foo.test/.well-known/webfinger?resource=acct:user#{i}@foo.test").to_return(body: webfinger.to_json, headers: { 'Content-Type': 'application/jrd+json' })
end
end
it 'creates at least some accounts' do
expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_least(2)
end
it 'creates no more account than the limit allows' do
expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_most(5)
end
end
end

View File

@ -28,6 +28,31 @@ RSpec.describe ReportService, type: :service do
end
end
context 'when the reported status is a DM' do
let(:target_account) { Fabricate(:account) }
let(:status) { Fabricate(:status, account: target_account, visibility: :direct) }
subject do
-> { described_class.new.call(source_account, target_account, status_ids: [status.id]) }
end
context 'when it is addressed to the reporter' do
before do
status.mentions.create(account: source_account)
end
it 'creates a report' do
is_expected.to change { target_account.targeted_reports.count }.from(0).to(1)
end
end
context 'when it is not addressed to the reporter' do
it 'errors out' do
is_expected.to raise_error
end
end
end
context 'when other reports already exist for the same target' do
let!(:target_account) { Fabricate(:account) }
let!(:other_report) { Fabricate(:report, target_account: target_account) }