Improve exception handling

pre_commit_license_headers/check_license_headers.py

@@ -36,9 +36,7 @@ def check_license_headers(filepath: Path, header_pattern: str, debug: bool) -> b
     content_lines = []
 
     with filepath.open() as f:
-        tokens_generator = list(tokenize.generate_tokens(f.readline))
-
-        for token in tokens_generator:
+        for token in tokenize.generate_tokens(f.readline):
             if token.type not in HEADER_TOKENS:
                 # we've reached the end of the header
                 break

tests/check_license_headers_test.py

@@ -3,6 +3,8 @@
 # be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause
 
 from pathlib import Path
+from tokenize import TokenError
+from unittest import mock
 
 import pytest
 
@@ -29,7 +31,8 @@ def get_abspath_str(filename: str) -> str:
         ("valid_2.py", 0),
         ("invalid_owner.txt", 1),
         ("missing_header.py", 1),
-        ("tokenize_fail.yaml", 2),
+        ("content_tokenization_skipped.yaml", 0),
+        ("content_tokenization_skipped.sh", 0),
     ),
 )
 def test_check_license_headers(filename, expected_retval):
@@ -135,3 +138,15 @@ def test_ignored_owner(capsys):
     assert e.value.code == 1
     stdout, _ = capsys.readouterr()
     assert "'--owner' will be ignored" in stdout
+
+
+def test_tokenize_exception():
+    """check_license_headers raises TokenError marks file as skipped."""
+    with pytest.raises(SystemExit) as e:
+        with mock.patch(
+            "pre_commit_license_headers.check_license_headers.check_license_headers"
+        ) as m:
+            m.side_effect = [TokenError]
+            main(base_args + [get_abspath_str("valid_1.py")])
+    assert e.type == SystemExit
+    assert e.value.code == 2

tests/resources/content_tokenization_skipped.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# Copyright (c) 2014-2016, 2018, 2020-2021 AFakeCompany Ltd
+#
+# Use of this source code is governed by a BSD-3-clause license that can
+# be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause
+
+show_help() {
+    cat <<EOF
+Usage: ${0##*/} [-h]
+
+Downloads bank transaction updates in OFX format.
+EOF
+}
+
+while getopts h opt ; do
+    case $opt in
+        h)
+            echo "Help"
+            exit 0
+            ;;
+        *)  # Generates IndentationError if parsed with Python tokenizer
+            echo "Help" >&2
+            exit 1
+            ;;
+    esac
+done