Monday, April 11, 2011

How to implment a dynamic robots.txt in ASP.NET MVC

http://www.iwantmymvc.com/2011-03-29-crawler-access-multi-target-deploy

Controller:
using System.Configuration;
using System.Web.Mvc;

namespace Website.Controllers
{
    public class CrawlerController : Controller
    {
        public ActionResult RobotsTextFile()
        {
            string content;
            if(ConfigurationManager.AppSettings["RobotTextAccess"].ToLower() == "private")
            {
                content = "User-agent: *\r\nDisallow: /";
            }
            else
            {
                content = "User-agent: *\r\nAllow: /";
            }
            Response.ContentType = "text/plain";
            Response.Write(content);
            return null;
        }
    }
}
Routing
public static void RegisterRoutes(RouteCollection routes)
{
    routes.IgnoreRoute("{resource}.axd/{*pathInfo}");

    routes.MapRoute(
        "Robots.txt",
        "Robots.txt",
        new { controller = "Crawler", action = "RobotsTextFile" }
    );

    routes.MapRoute(
        "Default", // Route name
        "{controller}/{action}/{id}", // URL with parameters
        new { controller = "Home", action = "Index", id = UrlParameter.Optional } // Parameter defaults
    );

}

3 comments:

  1. Great! Really Helpful thank you for posting this!

    ReplyDelete
  2. Very interesting, but how your Appsetting section looks like in web config? I am still a beginner....

    Thanks!

    ReplyDelete