Adding the ability to set a robots.txt for each site (ticket #87)

app/controllers/admin/robots_controller.rb

@@ -0,0 +1,18 @@
+module Admin
+  class RobotsController < ActionController::Base
+
+    include Locomotive::Routing::SiteDispatcher
+
+    include Locomotive::Render
+
+    before_filter :require_site
+
+    respond_to :txt
+
+    def show
+      template = Liquid::Template.parse(current_site.robots_txt)
+      render :text => template.render('request_host' => self.request.host.downcase)
+    end
+
+  end
+end

app/models/site.rb

@@ -10,6 +10,7 @@ class Site
 
   ## fields ##
   field :name
+  field :robots_txt
 
   ## associations ##
   references_many :pages, :validate => false

app/views/admin/current_sites/_form.html.haml

@@ -66,3 +66,11 @@
         - else
           .role
             %em.locked= t("admin.memberships.roles.#{membership.role}")
+
+- if can?(:manage, current_site)
+
+  = f.foldable_inputs :name => :robots_txt do
+    = f.custom_input :robots_txt, :css => 'code full', :with_label => false do
+      = f.label :robots_txt
+      %code{ :class => 'html' }
+        = f.text_area :robots_txt, :class => 'small'

config/assets.yml

@@ -45,6 +45,7 @@ javascripts:
     - public/javascripts/admin/content_types.js
   site:
     - public/javascripts/admin/site.js
+    - public/javascripts/admin/plugins/codemirror/codemirror.min.js
   import:
     - public/javascripts/admin/plugins/json2.js
     - public/javascripts/admin/plugins/smartupdater.js

config/locales/formtastic.en.yml

@@ -5,6 +5,7 @@ en:
       advanced_options: Advanced options
       meta: SEO Metadata
       seo: SEO settings
+      robots_txt: Robots.txt file
       code: Code
       raw_template: Template
       credentials: Credentials
@@ -67,6 +68,7 @@ en:
         meta_keywords: "Meta keywords used within the head tag of the page. They are separated by a comma. Required for SEO."
         meta_description: "Meta description used within the head tag of the page. Required for SEO."
         domain_name: "ex: locomotiveapp.org"
+        robots_txt: "Content of the <span class='code'>/robots.txt</span> file. Check the following <a href='http://www.w3.org/TR/html4/appendix/notes.html#h-B.4.1.1'>url</a> for more information."
       theme_asset:
         slug: "You do not need to add the extension file (.css or .js)"
         edit:

config/locales/formtastic.fr.yml

@@ -5,6 +5,7 @@ fr:
       advanced_options: Options avancées
       meta: SEO Metadata
       seo: Paramètres SEO
+      robots_txt: Fichier Robots.txt
       code: Code
       raw_template: Gabarit
       credentials: Informations de connexion

config/routes.rb

@@ -60,6 +60,9 @@ Rails.application.routes.draw do
   # sitemap
   match '/sitemap.xml' => 'admin/sitemaps#show', :format => 'xml'
 
+  # robots.txt
+  match '/robots.txt' => 'admin/robots#show', :format => 'txt'
+
   # magic urls
   match '/' => 'admin/rendering#show'
   match '*path/edit' => 'admin/rendering#show', :defaults => { :editing => true }

doc/changelogs/20110704.txt

@@ -25,4 +25,5 @@ Here is a pretty exhaustive list of what the BIG MERGE includes:
   - slight modifications on forms: separator between fields, text shadow, error message redesigned, ...etc
   - quick link to edit directly a model from the submenu
 - new sites picker (if the multi_sites mode is enabled and correctly set up)
+- the robots.txt can be edited directly from the back-office (site settings).
 - a lot of internal refactoring as well as a ton of bug corrections

features/engine/robots.feature

@@ -0,0 +1,22 @@
+Feature: Robots
+  As a designer
+  I want to be able to define a robots.txt file for my site
+
+Background:
+  Given I have the site: "test site" set up
+
+Scenario: Simple robots text
+  Given a robot_txt set to "robots text value"
+  When I view the rendered page at "/robots.txt"
+  Then the rendered output should look like:
+    """
+    robots text value
+    """
+
+Scenario: Robots.txt should provide access to request_host
+  Given a robot_txt set to "host: {{request_host}}"
+  When I view the rendered page at "/robots.txt"
+  Then the rendered output should look like:
+    """
+    host: test.example.com
+    """

features/step_definitions/site_steps.rb

@@ -22,3 +22,9 @@ Then /^I should be a administrator of the "([^"]*)" site$/ do |name|
   m = site.memberships.detect { |m| m.account_id == @admin._id && m.admin? }
   m.should_not be_nil
 end
+
+# sets the robot_txt for a site
+
+Given /^a robot_txt set to "([^"]*)"$/ do |value|
+  @site.update_attributes(:robots_txt => value)
+end

public/robots.txt

@@ -1,5 +0,0 @@
-# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-Agent: *
-# Disallow: /

public/stylesheets/admin/formtastic_changes.css

@@ -126,6 +126,7 @@ form.formtastic fieldset ol li.date input {
 
 form.formtastic fieldset ol li p.inline-hints {  margin-left: 20%; }
 form.formtastic fieldset ol li p.inline-hints a { color: #1f82bc; }
+form.formtastic fieldset ol li p.inline-hints .code { background-color: #c8cad0; color: #5E5F64; text-shadow: rgba(255, 255, 255, 0.8) 0px 1px; }
 
 form.formtastic fieldset ol li code { display: block; border: 1px solid #d1d6e6; margin: 10px 20px 0 20px; }
 form.formtastic fieldset ol li code.nude textarea {
@@ -161,7 +162,7 @@ form.formtastic fieldset ol li.code div.inline-errors {
   margin-left: 20px;
 }
 
-form.formtastic fieldset ol li.small-code code { margin: 0px; display: inline-block; width: 75%;  }
+form.formtastic fieldset ol li.small-code code { margin: 0px; display: inline-block; width: 75%; }
 
 form.formtastic fieldset ol .more { text-align: right; width: auto; margin: 10px 20px 0 0; line-height: 0.6em; }
 form.formtastic fieldset ol .more a { text-decoration: none; color: #787A89; font-size: 0.7em; }
@@ -172,6 +173,10 @@ form.formtastic fieldset ol .more a.picture {
   background: transparent url(/images/admin/icons/asset_add.png) no-repeat left 1px;
 }
 
+form.formtastic fieldset li.full .inline-errors, form.formtastic fieldset li.full .inline-hints {
+  margin-left: 20px;
+}
+
 /* ___ pages ___ */
 
 form.formtastic fieldset ol li.path em {
@@ -525,10 +530,6 @@ form.formtastic fieldset.email li.full input {
   margin-left: 20px;
 }
 
-form.formtastic fieldset.email li.full .inline-errors {
-  margin-left: 20px;
-}
-
 form.formtastic fieldset.memberships ol li .role {
   position: absolute;
   top: 2px;