Правильный robots.txt? [вопросы/обсуждение] - страница 3 - Поисковая оптимизация, продвижение сайтов на Joomla (SEO)

User-agent: Yandex
Disallow: /administrator/
Disallow: /cache/
Disallow: /components/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /libraries/
Disallow: /media/
Disallow: /modules/
Disallow: /plugins/
Disallow: /templates/
Disallow: /tmp/
Disallow: /xmlrpc/
Disallow: /index.php?
Disallow: /index.php?*
Disallow: /index2.php
Disallow: /index2.php?option=com_content&task=emailform
Disallow: /*?sl*
Disallow: /name.php?action=print
Disallow: /trackback
Disallow: /*rss.html
Disallow: /*atom.html
Disallow: /component
Disallow: /feedback
Disallow: /library
Disallow: /88-tents
Disallow: /104-biblioteka
Disallow: /social-network
Disallow: /moj-blog
Disallow: /blogi-podvodnyx-oxotnikov
Crawl-delay: 2
Host: aquanaft.ru
Sitemap: http://www.aquanaft.ru/index.php?option=com_xmap&sitemap=1&view=xml

User-agent: *
Disallow: /administrator/
Disallow: /cache/
Disallow: /components/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /libraries/
Disallow: /media/
Disallow: /modules/
Disallow: /plugins/
Disallow: /templates/
Disallow: /tmp/
Disallow: /xmlrpc/
Disallow: /index.php?
Disallow: /index.php?*
Disallow: /index2.php
Disallow: /index2.php?option=com_content&task=emailform
Disallow: /*?sl*
Disallow: *.pdf$
Disallow: /name.php?action=print
Disallow: /trackback
Disallow: /*rss.html
Disallow: /*atom.html
Disallow: /component
Disallow: /feedback
Disallow: /library
Disallow: /88-tents
Disallow: /104-biblioteka
Disallow: /social-network
Disallow: /moj-blog
Disallow: /blogi-podvodnyx-oxotnikov
Sitemap: http://www.aquanaft.ru/index.php?option=com_xmap&sitemap=1&view=xml

User-agent: *
Allow: /*.html$ #последняя версия Joomla! добавляет к адресам именно этот суффикс
Allow: /*.jpeg$
Allow: /*.jpg$
Allow: /*.png$
Allow: /$ # без этой строчки главная страница сайта не будет проиндексирована Яндексом
Disallow: /пункт меню.html #запрещаем адреса, включающие в себя названия пунктов меню
Disallow: /

User-agent: *
Disallow: /administrator/
Disallow: /cache/
Disallow: /cli/
Disallow: /components/
Disallow: /images/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /libraries/
Disallow: /logs/
Disallow: /media/
Disallow: /modules/
Disallow: /plugins/
Disallow: /templates/
Disallow: /tmp/

Allow: /index.php?option=com_xmap&view=xml&id=1
Sitemap: http://smotronik.16mb.com/index.php?option=com_xmap&view=xml&id=1
Host: smotronik.16mb.com

#User-agent: Yandex
Crawl-delay: 1

## Can be commented out if causes errors, see notes above.
Options +FollowSymLinks

## Mod_rewrite in use.

RewriteEngine On

## Begin - Rewrite rules to block out some common exploits.
# If you experience problems on your site block out the operations listed below
# This attempts to block the most common type of exploit `attempts` to Joomla!
#
# Block out any script trying to base64_encode data within the URL.
RewriteCond %{QUERY_STRING} base64_encode[^(]*\([^)]*\) [OR]
# Block out any script that includes a <script> tag in URL.
RewriteCond %{QUERY_STRING} (<|%3C)([^s]*s)+cript.*(>|%3E) [NC,OR]
# Block out any script trying to set a PHP GLOBALS variable via URL.
RewriteCond %{QUERY_STRING} GLOBALS(=|\[|\%[0-9A-Z]{0,2}) [OR]
# Block out any script trying to modify a _REQUEST variable via URL.
RewriteCond %{QUERY_STRING} _REQUEST(=|\[|\%[0-9A-Z]{0,2})
# Return 403 Forbidden header and show the content of the root homepage
RewriteRule .* index.php [F]
#
## End - Rewrite rules to block out some common exploits.

## Begin - Custom redirects
#
# If you need to redirect some pages, or set a canonical non-www to
# www redirect (or vice versa), place that code here. Ensure those
# redirects use the correct RewriteRule syntax and the [R=301,L] flags.
#
## End - Custom redirects

##
# Uncomment following line if your webserver's URL
# is not directly related to physical file paths.
# Update Your Joomla! Directory (just / for root).
##

# RewriteBase /

## Begin - Joomla! core SEF Section.
#
RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]
#
# If the requested path and file is not /index.php and the request
# has not already been internally rewritten to the index.php script
RewriteCond %{REQUEST_URI} !^/index\.php
# and the request is for something within the component folder,
# or for the site root, or for an extensionless URL, or the
# requested URL ends with one of the listed extensions
RewriteCond %{REQUEST_URI} /component/|(/[^.]*|\.(php|html?|feed|pdf|vcf|raw))$ [NC]
# and the requested path and file doesn't directly match a physical file
RewriteCond %{REQUEST_FILENAME} !-f
# and the requested path and file doesn't directly match a physical folder
RewriteCond %{REQUEST_FILENAME} !-d
# internally rewrite the request to the index.php script
RewriteRule .* index.php [L]
#
## End - Joomla! core SEF Section.

Disallow: /*?	#	#	#все ссылки которые содержат этот знак не индексируются !
Disallow: /*%	#	#	#все ссылки которые содержат этот знак не индексируются !
Disallow: /*& 	#	#	#все ссылки которые содержат этот знак не индексируются !

http://www.bibliothai.ru/turistam/otchety-turistov/otchety-turistov.html?pid=73&sid=95:%D0%9D%D0%B0%D1%88%D0%B5-%D0%BF%D1%83%D1%82%D0%B5%D1%88%D0%B5%D1%81%D1%82%D0%B2%D0%B8%D0%B5-%D0%BD%D0%B0-%D0%9F%D1%85%D1%83%D0%BA%D0%B5%D1%82

Allow: /turistam/otchety-turistov/*