diff --git a/lib/sshkit/runners/parallel.rb b/lib/sshkit/runners/parallel.rb
index fd5d018a..b1543029 100644
--- a/lib/sshkit/runners/parallel.rb
+++ b/lib/sshkit/runners/parallel.rb
@@ -16,7 +16,24 @@ def execute
             end
           end
         end
-        threads.each(&:join)
+
+        wait_for_threads(threads)
+      end
+
+      private
+
+      def wait_for_threads(threads)
+        exception = nil
+
+        threads.map do |t|
+          begin
+            t.join
+          rescue ExecuteError => e
+            exception ||= e
+          end
+        end
+
+        raise exception if exception
       end
     end
 
diff --git a/test/unit/runners/test_parallel.rb b/test/unit/runners/test_parallel.rb
index 4a90900b..c6a066bd 100644
--- a/test/unit/runners/test_parallel.rb
+++ b/test/unit/runners/test_parallel.rb
@@ -13,6 +13,31 @@ def test_wraps_ruby_standard_error_in_execute_error
         assert_match(/deployer@example/, error.message)
         assert_match(/oh no!/, error.message)
       end
+
+      def test_waits_for_all_threads_to_finish_on_error
+        hosts = [Host.new("deployer@example"), Host.new("deployer@example2"), Host.new("deployer@example3")]
+        completed_one, completed_three = false, false
+        runner = Parallel.new(hosts) do |host|
+          case host.hostname
+          when "example"
+            sleep 0.1
+            completed_one = true
+          when "example2"
+            raise "Boom!"
+          when "example3"
+            sleep 0.3
+            completed_three = true
+          end
+        end
+
+        error = assert_raises(SSHKit::Runner::ExecuteError) do
+          runner.execute
+        end
+        assert_match(/deployer@example2/, error.message)
+        assert_match(/Boom!/, error.message)
+        assert completed_one
+        assert completed_three
+      end
     end
   end
 end