User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /wp-admin/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cgi-bin/
Disallow: /envira/

Allow: /wp-admin/admin-ajax.php

User-agent: bingbot
Disallow: /wp-admin/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /feed/
Crawl-delay: 10

Sitemap:https://www.theshelbyreport.com/sitemap_index.xml