diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4cdd881..129519519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,23 @@ Changelog All notable changes to this project will be documented in this file. +## [3.4.5] - 2022-11-14 +## Fixed + +- Fix nodes order being sometimes mangled when rewriting emoji ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20677)) + +## [3.5.4] - 2022-11-14 +### Fixed + +- Fix error when a remote report includes a private post the server has no access to ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18760)) + +### Security + +- Fix emoji substitution not applying only to text nodes in backend code ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20641)) +- Fix emoji substitution not applying only to text nodes in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20640)) +- Fix rate limiting for paths with formats ([Gargron](https://github.com/mastodon/mastodon/pull/20675)) +- Fix out-of-bound reads in blurhash transcoder ([delroth](https://github.com/mastodon/mastodon/pull/20388)) + ## [3.5.3] - 2022-05-26 ### Added diff --git a/Gemfile b/Gemfile index 2e77fb42a..f76776c8c 100644 --- a/Gemfile +++ b/Gemfile @@ -66,6 +66,7 @@ gem 'oj', '~> 3.13' gem 'ox', '~> 2.14' gem 'parslet' gem 'posix-spawn' +gem 'public_suffix', '~> 4.0.7' gem 'pundit', '~> 2.2' gem 'premailer-rails' gem 'rack-attack', '~> 6.6' diff --git a/Gemfile.lock b/Gemfile.lock index e12fdc237..2742de860 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -803,6 +803,7 @@ DEPENDENCIES private_address_check (~> 0.5) pry-byebug (~> 3.9) pry-rails (~> 0.3) + public_suffix (~> 4.0.7) puma (~> 5.6) pundit (~> 2.2) rack (~> 2.2.3) diff --git a/app/javascript/mastodon/features/emoji/__tests__/emoji-test.js b/app/javascript/mastodon/features/emoji/__tests__/emoji-test.js index 07b3d8c53..2f19aab7e 100644 --- a/app/javascript/mastodon/features/emoji/__tests__/emoji-test.js +++ b/app/javascript/mastodon/features/emoji/__tests__/emoji-test.js @@ -11,8 +11,8 @@ describe('emoji', () => { }); it('works with unclosed tags', () => { - expect(emojify('hello>')).toEqual('hello>'); - expect(emojify('')).toEqual('hello>'); + expect(emojify(' { @@ -22,23 +22,23 @@ describe('emoji', () => { it('does unicode', () => { expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).toEqual( - '๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘ฆโ€๐Ÿ‘ฆ'); + '๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘ฆโ€๐Ÿ‘ฆ'); expect(emojify('๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ง')).toEqual( - '๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ง'); - expect(emojify('๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘ฆ')).toEqual('๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘ฆ'); + '๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ง'); + expect(emojify('๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘ฆ')).toEqual('๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘ฆ'); expect(emojify('\u2757')).toEqual( - 'โ—'); + 'โ—'); }); it('does multiple unicode', () => { expect(emojify('\u2757 #\uFE0F\u20E3')).toEqual( - 'โ— #๏ธโƒฃ'); + 'โ— #๏ธโƒฃ'); expect(emojify('\u2757#\uFE0F\u20E3')).toEqual( - 'โ—#๏ธโƒฃ'); + 'โ—#๏ธโƒฃ'); expect(emojify('\u2757 #\uFE0F\u20E3 \u2757')).toEqual( - 'โ— #๏ธโƒฃ โ—'); + 'โ— #๏ธโƒฃ โ—'); expect(emojify('foo \u2757 #\uFE0F\u20E3 bar')).toEqual( - 'foo โ— #๏ธโƒฃ bar'); + 'foo โ— #๏ธโƒฃ bar'); }); it('ignores unicode inside of tags', () => { @@ -46,16 +46,16 @@ describe('emoji', () => { }); it('does multiple emoji properly (issue 5188)', () => { - expect(emojify('๐Ÿ‘Œ๐ŸŒˆ๐Ÿ’•')).toEqual('๐Ÿ‘Œ๐ŸŒˆ๐Ÿ’•'); - expect(emojify('๐Ÿ‘Œ ๐ŸŒˆ ๐Ÿ’•')).toEqual('๐Ÿ‘Œ ๐ŸŒˆ ๐Ÿ’•'); + expect(emojify('๐Ÿ‘Œ๐ŸŒˆ๐Ÿ’•')).toEqual('๐Ÿ‘Œ๐ŸŒˆ๐Ÿ’•'); + expect(emojify('๐Ÿ‘Œ ๐ŸŒˆ ๐Ÿ’•')).toEqual('๐Ÿ‘Œ ๐ŸŒˆ ๐Ÿ’•'); }); it('does an emoji that has no shortcode', () => { - expect(emojify('๐Ÿ‘โ€๐Ÿ—จ')).toEqual('๐Ÿ‘โ€๐Ÿ—จ'); + expect(emojify('๐Ÿ‘โ€๐Ÿ—จ')).toEqual('๐Ÿ‘โ€๐Ÿ—จ'); }); it('does an emoji whose filename is irregular', () => { - expect(emojify('โ†™๏ธ')).toEqual('โ†™๏ธ'); + expect(emojify('โ†™๏ธ')).toEqual('โ†™๏ธ'); }); it('avoid emojifying on invisible text', () => { @@ -67,26 +67,26 @@ describe('emoji', () => { it('avoid emojifying on invisible text with nested tags', () => { expect(emojify('๐Ÿ˜‡')) - .toEqual('๐Ÿ˜‡'); + .toEqual('๐Ÿ˜‡'); expect(emojify('๐Ÿ˜‡')) - .toEqual('๐Ÿ˜‡'); - expect(emojify('๐Ÿ˜‡')) - .toEqual('๐Ÿ˜‡'); + .toEqual('๐Ÿ˜‡'); + expect(emojify('๐Ÿ˜‡')) + .toEqual('๐Ÿ˜‡'); }); it('skips the textual presentation VS15 character', () => { expect(emojify('โœด๏ธŽ')) // This is U+2734 EIGHT POINTED BLACK STAR then U+FE0E VARIATION SELECTOR-15 - .toEqual('โœด'); + .toEqual('โœด'); }); it('does an simple emoji properly', () => { expect(emojify('โ™€โ™‚')) - .toEqual('โ™€โ™‚'); + .toEqual('โ™€โ™‚'); }); it('does an emoji containing ZWJ properly', () => { expect(emojify('๐Ÿ’‚โ€โ™€๏ธ๐Ÿ’‚โ€โ™‚๏ธ')) - .toEqual('๐Ÿ’‚\u200Dโ™€๏ธ๐Ÿ’‚\u200Dโ™‚๏ธ'); + .toEqual('๐Ÿ’‚\u200Dโ™€๏ธ๐Ÿ’‚\u200Dโ™‚๏ธ'); }); }); }); diff --git a/app/javascript/mastodon/features/emoji/emoji.js b/app/javascript/mastodon/features/emoji/emoji.js index fb1a3804c..52a8458fb 100644 --- a/app/javascript/mastodon/features/emoji/emoji.js +++ b/app/javascript/mastodon/features/emoji/emoji.js @@ -19,15 +19,26 @@ const emojiFilename = (filename) => { return borderedEmoji.includes(filename) ? (filename + '_border') : filename; }; -const emojify = (str, customEmojis = {}) => { - const tagCharsWithoutEmojis = '<&'; - const tagCharsWithEmojis = Object.keys(customEmojis).length ? '<&:' : '<&'; - let rtn = '', tagChars = tagCharsWithEmojis, invisible = 0; +const domParser = new DOMParser(); + +const emojifyTextNode = (node, customEmojis) => { + let str = node.textContent; + + const fragment = new DocumentFragment(); + for (;;) { - let match, i = 0, tag; - while (i < str.length && (tag = tagChars.indexOf(str[i])) === -1 && (invisible || !(match = trie.search(str.slice(i))))) { - i += str.codePointAt(i) < 65536 ? 1 : 2; + let match, i = 0; + + if (customEmojis === null) { + while (i < str.length && !(match = trie.search(str.slice(i)))) { + i += str.codePointAt(i) < 65536 ? 1 : 2; + } + } else { + while (i < str.length && str[i] !== ':' && !(match = trie.search(str.slice(i)))) { + i += str.codePointAt(i) < 65536 ? 1 : 2; + } } + let rend, replacement = ''; if (i === str.length) { break; @@ -35,8 +46,6 @@ const emojify = (str, customEmojis = {}) => { if (!(() => { rend = str.indexOf(':', i + 1) + 1; if (!rend) return false; // no pair of ':' - const lt = str.indexOf('<', i + 1); - if (!(lt === -1 || lt >= rend)) return false; // tag appeared before closing ':' const shortname = str.slice(i, rend); // now got a replacee as ':shortname:' // if you want additional emoji handler, add statements below which set replacement and return true. @@ -47,29 +56,6 @@ const emojify = (str, customEmojis = {}) => { } return false; })()) rend = ++i; - } else if (tag >= 0) { // <, & - rend = str.indexOf('>;'[tag], i + 1) + 1; - if (!rend) { - break; - } - if (tag === 0) { - if (invisible) { - if (str[i + 1] === '/') { // closing tag - if (!--invisible) { - tagChars = tagCharsWithEmojis; - } - } else if (str[rend - 2] !== '/') { // opening tag - invisible++; - } - } else { - if (str.startsWith('