Better error handling + failure messages

segiddins · segiddins · commit 91376393cdff · 2024-10-25T16:41:57.000-04:00
Signed-off-by: Samuel Giddins &lt;segiddins@segiddins.me&gt;
diff --git a/lib/quickdraw/configuration.rb b/lib/quickdraw/configuration.rb
@@ -8,6 +8,7 @@ def initialize
 		@registry = Quickdraw::Registry.new
 		@failure_symbol = "\e[31m⨯\e[0m"
 		@success_symbol = "\e[32m∘\e[0m"
+		@error_symbol = "\e[31me\e[0m"
 		@processes = DEFAULT_PROCESSES
 		@threads = DEFAULT_THREADS
 		@success_emoji = %w[💃 🕺 🎉 🎊 💪 👏 🙌 ✨ 🥳 🎈 🌈 🎯 🏆]
@@ -20,6 +21,7 @@ def initialize
 	attr_accessor :success_emoji
 	attr_accessor :success_symbol
 	attr_accessor :threads
+	attr_accessor :error_symbol
 
 	def matcher(matcher, *types)
 		@registry.register(matcher, *types)
diff --git a/lib/quickdraw/context.rb b/lib/quickdraw/context.rb
@@ -34,6 +34,8 @@ def run_test(name, skip, runner, &)
 			instance = new(name, skip, runner, matchers)
 			instance.instance_exec(&)
 			instance.resolve
+		rescue Exception => error
+			runner.error!(name, error)
 		end
 	end
 
@@ -74,37 +76,37 @@ def resolve
 
 	def assert(value)
 		if value
-			success!
+			success!(depth: 1)
 		elsif block_given?
-			failure! { yield(value) }
+			failure!(depth: 1) { yield(value) }
 		else
-			failure! { "expected #{value.inspect} to be truthy" }
+			failure!(depth: 1) { "expected #{value.inspect} to be truthy" }
 		end
 	end
 
 	def refute(value)
 		if !value
-			success!
+			success!(depth: 1)
 		elsif block_given?
-			failure! { yield(value) }
+			failure!(depth: 1) { yield(value) }
 		else
-			failure! { "expected #{value.inspect} to be falsy" }
+			failure!(depth: 1) { "expected #{value.inspect} to be falsy" }
 		end
 	end
 
-	def success!
+	def success!(depth:)
 		if @skip
-			@runner.failure! { "The skipped test `#{@name}` started passing." }
+			@runner.failure!(@name, depth:) { "The skipped test `#{@name}` started passing." }
 		else
-			@runner.success!(@name)
+			@runner.success!(@name, depth:)
 		end
 	end
 
-	def failure!(&)
+	def failure!(depth:, &)
 		if @skip
-			@runner.success!(@name)
+			@runner.success!(@name, depth:)
 		else
-			@runner.failure!(&)
+			@runner.failure!(@name, depth:, &)
 		end
 	end
 end
diff --git a/lib/quickdraw/expectation.rb b/lib/quickdraw/expectation.rb
@@ -37,29 +37,29 @@ def initialize(context, subject = Quickdraw::Null, &block)
 		@made_expectations = false
 	end
 
-	def success!
-		@context.success!
+	def success!(depth:)
+		@context.success!(depth: depth)
 		@made_expectations = true
 	end
 
-	def failure!(&)
-		@context.failure!(&)
+	def failure!(depth:, &)
+		@context.failure!(depth:, &)
 		@made_expectations = true
 	end
 
 	def resolve
 		unless @made_expectations
-			failure! { "You didn't make any expectations." }
+			failure!(depth: 2) { "You didn't make any expectations." }
 		end
 	end
 
 	private
 
-	def assert(value, &)
-		value ? success! : failure!(&)
+	def assert(value, depth: 1, &)
+		value ? success!(depth:) : failure!(depth:, &)
 	end
 
-	def refute(value, &)
-		value ? failure!(&) : success!
+	def refute(value, depth: 1, &)
+		value ? failure!(depth:, &) : success!(depth:)
 	end
 end
diff --git a/lib/quickdraw/matchers/to_have_attributes.rb b/lib/quickdraw/matchers/to_have_attributes.rb
@@ -8,6 +8,6 @@ def to_have_attributes(**attributes)
 			end
 		end
 	rescue NoMethodError => e
-		failure! { "expected `#{@subject.inspect}` to respond to `#{e.name}`" }
+		failure!(depth: 1) { "expected `#{@subject.inspect}` to respond to `#{e.name}`" }
 	end
 end
diff --git a/lib/quickdraw/matchers/to_raise.rb b/lib/quickdraw/matchers/to_raise.rb
@@ -7,20 +7,20 @@ def to_raise(error = ::Exception)
 		begin
 			expectation_block.call
 		rescue error => e
-			success!
+			success!(depth: 1)
 			yield(e) if block_given?
 			return
 		rescue ::Exception => e
-			return failure! { "expected `#{error.inspect}` to be raised but `#{e.class.inspect}` was raised" }
+			return failure!(depth: 1) { "expected `#{error.inspect}` to be raised but `#{e.class.inspect}` was raised" }
 		end
 
-		failure! { "expected #{error} to be raised but wasn't" }
+		failure!(depth: 1) { "expected #{error} to be raised but wasn't" }
 	end
 
 	def not_to_raise
 		@block.call
-		success!
+		success!(depth: 1)
 	rescue ::Exception => e
-		failure! { "expected the block not to raise, but it raised `#{e.class}`" }
+		failure!(depth: 1) { "expected the block not to raise, but it raised `#{e.class}`" }
 	end
 end
diff --git a/lib/quickdraw/matchers/to_receive.rb b/lib/quickdraw/matchers/to_receive.rb
@@ -9,7 +9,7 @@ def to_receive(method_name, &expectation_block)
 		context = @context
 
 		interceptor.define_method(method_name) do |*args, **kwargs, &block|
-			expectation.success!
+			expectation.success!(depth: 2)
 			super_block = -> (*a, &b) { ((a.length > 0) || b) ? super(*a, &b) : super(*args, **kwargs, &block) }
 			original_super = context.instance_variable_get(:@super)
 			begin
diff --git a/lib/quickdraw/run.rb b/lib/quickdraw/run.rb
@@ -31,16 +31,27 @@ def call
 
 		results.each do |r|
 			failures = r["failures"]
+			errors = r["errors"]
 
 			i = 0
 			number_of_failures = failures.size
 			total_failures += number_of_failures
 			while i < number_of_failures
 				failure = failures[i]
-				path, lineno, message = failure
-				puts "#{path}:#{lineno} #{message}"
+				test_name, path, lineno, message = failure
+				puts "#{path}:#{lineno} in #{test_name.inspect}: #{message}"
 				i += 1
 			end
+
+			number_of_errors = errors.size
+			total_failures += number_of_errors
+			while i < number_of_errors
+				error = errors[i]
+				test_name, cls, message, backtrace = error
+				puts "Unexpected #{cls} in #{test_name}:\n#{message}\n#{backtrace.join("\n\t")}"
+				i += 1
+			end
+
 		end
 
 		puts "Ran with seed: #{@seed}"
diff --git a/lib/quickdraw/runner.rb b/lib/quickdraw/runner.rb
@@ -9,6 +9,7 @@ def initialize(queue:, threads:)
 
 		@failures = Concurrent::Array.new
 		@successes = Concurrent::Array.new
+		@errors = Concurrent::Array.new
 	end
 
 	def call
@@ -24,26 +25,35 @@ def call
 			"pid" => Process.pid,
 			"failures" => @failures.to_a,
 			"successes" => @successes.size,
+			"errors" => @errors.to_a,
 		}
 	end
 
-	def success!(name)
+	def success!(name, depth: 0)
 		@successes << name
 
 		Kernel.print(
 			Quickdraw::Config.success_symbol,
 		)
 	end
 
-	def failure!
-		location = caller_locations(3, 1).first
-		@failures << [location.path, location.lineno, yield]
+	def failure!(name, depth: 0)
+		location = caller_locations(2 + depth, 1).first
+		@failures << [name, location.path, location.lineno, yield]
 
 		Kernel.print(
 			Quickdraw::Config.failure_symbol,
 		)
 	end
 
+	def error!(name, error)
+		@errors << [name, error.class.name, error.detailed_message, error.backtrace]
+
+		Kernel.print(
+			Quickdraw::Config.error_symbol,
+		)
+	end
+
 	private
 
 	def drain_queue
diff --git a/test/assert.test.rb b/test/assert.test.rb
@@ -11,7 +11,7 @@
 test "assert with falsy value" do
 	expect {
 		test { assert false }
-	}.to_fail message: "expected false to be truthy"
+	}.to_fail message: "expected false to be truthy", location: [__FILE__, __LINE__ - 1]
 end
 
 test "assert with truthy value" do
@@ -23,5 +23,11 @@
 test "assert with custom failure message" do
 	expect {
 		test { assert(false) { "Message" } }
-	}.to_fail message: "Message"
+	}.to_fail message: "Message", location: [__FILE__, __LINE__ - 1]
+end
+
+test "assert with custom message raising an error" do
+	expect {
+		test { assert(false) { raise ArgumentError } }
+	}.to_error
 end
diff --git a/test/matchers/pattern_match.test.rb b/test/matchers/pattern_match.test.rb
@@ -9,5 +9,5 @@
 test "when not equal" do
 	expect {
 		test { expect("a") =~ /b/ }
-	}.to_fail message: %(expected `"a"` to =~ `"/b/"`)
+	}.to_fail message: %(expected `"a"` to =~ `/b/`)
 end
diff --git a/test/matchers/to_have_attributes.test.rb b/test/matchers/to_have_attributes.test.rb
@@ -19,7 +19,7 @@
 				name: "Jill",
 			)
 		}
-	}.to_fail message: %(expected `#<data name="Joel">` to have the attribute `:name` equal to `"Jill"`)
+	}.to_fail message: %(expected `#{User.new(name: "Joel").inspect}` to have the attribute `:name` equal to `"Jill"`)
 end
 
 test "failure with missing reader method" do
@@ -29,5 +29,5 @@
 				email: "joel@drapper.me",
 			)
 		}
-	}.to_fail message: %(expected `#<data name="Joel">` to respond to `email`)
+	}.to_fail message: %(expected `#{User.new(name: "Joel").inspect}` to respond to `email`)
 end
diff --git a/test/support/matchers/pass_fail.rb b/test/support/matchers/pass_fail.rb
@@ -18,16 +18,22 @@ class Result
 			def initialize
 				@successes = []
 				@failures = []
+				@errors = []
 			end
 
-			attr_reader :successes, :failures
+			attr_reader :successes, :failures, :errors
 
-			def success!(name)
+			def success!(name, depth: 0)
 				@successes << name
 			end
 
-			def failure!
-				@failures << yield
+			def failure!(name, depth: 0)
+				location = caller_locations(2 + depth, 1).first
+				@failures << [name, location.path, location.lineno, yield]
+			end
+
+			def error!(name, error)
+				@errors << [name, error.class.name, error.detailed_message, error.backtrace]
 			end
 		end
 
@@ -45,16 +51,48 @@ def to_pass
 				context.run_test(name, skip, result, &test)
 			end
 
-			assert result.failures.empty? do
+			assert result.failures.empty?, depth: 3 do
 				"expected the test to pass, but it failed"
 			end
 
-			assert result.successes.length > 0 do
+			assert result.errors.empty?, depth: 3 do
+				error = result.errors.first
+				<<~MSG
+					expected the test to fail but an error occurred:
+					#{error[0]}: #{error[1]}
+					#{error[2].join("\n\t")}
+				MSG
+			end
+
+			assert result.successes.length > 0, depth: 3 do
 				"expected the test to pass but no assertions were made"
 			end
 		end
 
-		def to_fail(message: nil)
+		def to_error
+			run = Run.new
+			result = Result.new
+			definition = @block
+
+			Class.new(Quickdraw::Context) do
+				define_singleton_method(:run) { run }
+				class_exec(&definition)
+			end
+
+			run.tests.each do |(name, skip, test, context)|
+				context.run_test(name, skip, result, &test)
+			end
+
+			assert result.failures.empty?, depth: 3 do
+				"expected the test to error, but it failed"
+			end
+
+			refute result.errors.empty?, depth: 3 do
+				"expected the test to error"
+			end
+		end
+
+		def to_fail(message: nil, location: nil)
 			run = Run.new
 			result = Result.new
 			definition = @block
@@ -68,9 +106,22 @@ def to_fail(message: nil)
 				context.run_test(name, skip, result, &test)
 			end
 
-			assert result.failures.length > 0 do
+			assert result.errors.empty?, depth: 3 do
+				name, cls, msg, bt = result.errors.first
+				<<~MSG
+					expected the test to fail but a #{cls} occurred in #{name.inspect}:
+					#{msg}
+					#{bt.join("\n\t")}
+				MSG
+			end
+			assert result.failures.length > 0, depth: 3 do
 				"expected the test to fail"
 			end
+
+			name, file, line, msg = result.failures.first
+			assert(file == location[0], depth: 3) { "expected file #{file.inspect} to be #{location[0].inspect}" } if location
+			assert(line == location[1], depth: 3) { "expected line #{line.inspect} to be #{location[1].inspect}" } if location
+			assert(message === msg, depth: 3) { "expected message #{msg.inspect} to be #{message.inspect}" } if message
 		end
 	end
 end
