User-agent: Scrapy

Allow: /
User-agent: *

# Disallow crawling of the entire site
Disallow: /

# But allow access to this file as it is often used front-end by themes and plugins
Allow: /wp-admin/admin-ajax.php

# Also allow access to other resources needed for mobile friendly test
Allow: /wp-content/
Allow: /wp-includes/