User-agent: *

Disallow: /online/*
Disallow: /air/*
Disallow: /web/*
Disallow: /agreement

#block spiders
User-agent: BLEXBot
User-agent: YandexBot
User-agent: AhrefsBot
User-agent: XoviBot
User-agent: panscient.com
User-agent: crawler4j
User-agent: MegaIndex.ru/2.0
User-agent: MegaIndex.ru/
User-agent: MJ12
User-agent: MJ12bot
User-agent: Steeler
User-agent: Findxbot
User-agent: SeznamBot
User-agent: Seznam
User-agent: Vagabondo
User-agent: SMTBot
User-agent: memoryBot
User-agent: smrjbot
User-agent: spbot
User-agent: dotbot
Disallow:/