From 6a724a7f0d087ff1bf638584d5ca50b75f581fac Mon Sep 17 00:00:00 2001
From: MothOnMars <mothonmars@gmail.com>
Date: Mon, 15 Jan 2018 14:37:35 -0800
Subject: [PATCH] strip comments from rules

---
 lib/robotex.rb       | 2 +-
 spec/robotex_spec.rb | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/robotex.rb b/lib/robotex.rb
index bf186ab..77a90b6 100644
--- a/lib/robotex.rb
+++ b/lib/robotex.rb
@@ -31,7 +31,7 @@ def initialize(uri, user_agent)
         arr = line.split(":")
         key = arr.shift
         value = arr.join(":").strip
-        value.strip!
+        value.gsub!(/\s*#.+$/,'')
         case key.downcase
           when "user-agent"
             agent = to_regex(value)
diff --git a/spec/robotex_spec.rb b/spec/robotex_spec.rb
index d8b4388..4d90f6f 100644
--- a/spec/robotex_spec.rb
+++ b/spec/robotex_spec.rb
@@ -13,6 +13,7 @@
 
 User-Agent: *
 Disallow: /login
+Disallow: /archive/ #old content
 Allow: /
 
 Disallow: /locked
@@ -65,6 +66,13 @@
         robotex.allowed?(SPEC_DOMAIN + 'locked').should be_false
       end
     end
+
+    context 'when a rule includes a comment' do
+      it 'returns false' do
+        robotex = Robotex.new
+        robotex.allowed?(SPEC_DOMAIN + 'archive/old').should be_false
+      end
+    end
   end
 
   describe '#delay' do