Change robots.txt to exclude some URLs (#10037)

- Exclude static assets
- Exclude uploaded files
- Exclude alternate versions of the profile page
- Exclude media proxy URLs

public/robots.txt

@@ -1,5 +1,13 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
+User-Agent: *
-#
+Disallow: /users/*/followers
-# To ban all spiders from the entire site uncomment the next two lines:
+Disallow: /users/*/following
-# User-agent: *
+Disallow: /@*/media
-# Disallow: /
+Disallow: /@*/with_replies
+Disallow: /@*/tagged/*
+Disallow: /media_proxy/*
+Disallow: /emoji/*
+Disallow: /packs/*
+Disallow: /sounds/*
+Disallow: /system/*
+Disallow: /avatars/*
+Disallow: /headers/*