private static readonly List<string> KnownCrawlers = new List<string>
{
    "bot","crawler","baiduspider","80legs","ia_archiver","ahrefsBot","twitterbot",
    "yoozbot","yandexBot","bitlybot","other", "sogou web spider", "python requests",
    "voyager","curl","wget","yahoo! slurp","mediapartners-google", "mj12bot",
    "seznamBot", "Sogou web spider", "360Spider", "sogouwebspider"
};

public static bool IsBotOrCrawler(string agent)
{
    agent = agent.ToLower();
    return KnownCrawlers.Any(crawler => agent.Contains(crawler) || agent.Equals(crawler));
}