[#64] implement copy paste protection

CHANGES.md

@@ -36,11 +36,13 @@ Unreleased
   + Now we call references to anchors in current file (e.g. `[a](#b)`) as
   `file-local` references instead of calling them `current file` (which was ambiguous).
 * [#233](https://github.com/serokell/xrefcheck/pull/233)
-  + Now xrefxcheck does not follow redirect links by default. It fails for permanent
+  + Now xrefcheck does not follow redirect links by default. It fails for permanent
     redirect responses (i.e. 301 and 308) and passes for temporary ones (i.e. 302, 303, 307).
 * [#231](https://github.com/serokell/xrefcheck/pull/231)
   + Anchor analysis takes now into account the appropriate case-sensitivity depending on
   the configured Markdown flavour.
+* [240](https://github.com/serokell/xrefcheck/pull/240)
+  + Now xrefcheck is able to detect possible copy-pastes relying on links and their names.
 
 0.2.2
 ==========

README.md

@@ -45,6 +45,7 @@ Comparing to alternative solutions, this tool tries to achieve the following poi
 * Supports external links (`http`, `https`, `ftp` and `ftps`).
 * Detects broken and ambiguous anchors in local links.
 * Integration with GitHub Actions.
+* Detects possible bad copy-pastes of links.
 
 ## Dependencies [↑](#xrefcheck)
 
@@ -148,6 +149,21 @@ There are several ways to fix this:
     * By default, `xrefcheck` will ignore links to localhost.
     * This behavior can be disabled by removing the corresponding entry from the `ignoreExternalRefsTo` list in the config file.
 
+1. How do I disable copy-paste check for specific links?
+    * Add a `<!-- xrefcheck: no duplication check in link -->` annotation before the link:
+      ```md
+      <!-- xrefcheck: no duplication check in link -->
+      Links with bad copypaste:
+      [good link](https://good.link.uri/).
+      [copypasted link](https://good.link.uri/).
+      ```
+      ```md
+      A [good link](https://good.link.uri/)
+      followed by an <!-- xrefcheck: no duplication check in link --> [copypasted intentionally](https://good.link.uri/).
+      ```
+    * You can use a `<!-- xrefcheck: no duplication check in paragraph -->` annotation to disable copy-paste check in a paragraph.
+    * You can use a `<!-- xrefcheck: no duplication check in file -->` annotation to disable copy-paste check within an entire file.
+
 ## Further work [↑](#xrefcheck)
 
 - [ ] Support link detection in different languages, not only Markdown.

src/Xrefcheck/Command.hs

@@ -28,7 +28,7 @@ import Xrefcheck.Scan
 import Xrefcheck.Scanners.Markdown (markdownSupport)
 import Xrefcheck.System (askWithinCI)
 import Xrefcheck.Util
-import Xrefcheck.Verify (reportVerifyErrs, verifyErrors, verifyRepo)
+import Xrefcheck.Verify (reportCopyPasteErrors, reportVerifyErrs, verifyErrors, verifyRepo)
 
 readConfig :: FilePath -> IO Config
 readConfig path = fmap (normaliseConfigFilePaths . overrideConfig) do
@@ -81,11 +81,12 @@ defaultAction Options{..} = do
 
     whenJust (nonEmpty $ sortBy (compare `on` seFile) scanErrs) $ reportScanErrs
 
-    verifyRes <- allowRewrite showProgressBar $ \rw -> do
+    (verifyRes, copyPasteErrors) <- allowRewrite showProgressBar $ \rw -> do
       let fullConfig = config
             { cNetworking = addNetworkingOptions (cNetworking config) oNetworkingOptions }
       verifyRepo rw fullConfig oMode oRoot repoInfo
 
+    whenJust (nonEmpty copyPasteErrors) reportCopyPasteErrors
     case verifyErrors verifyRes of
       Nothing | null scanErrs -> fmtLn "All repository links are valid."
       Nothing -> exitFailure

src/Xrefcheck/Config.hs

@@ -73,6 +73,8 @@ data ScannersConfig' f = ScannersConfig
   , scAnchorSimilarityThreshold :: Field f Double
     -- ^ On 'anchor not found' error, how much similar anchors should be displayed as
     -- hint. Number should be between 0 and 1, larger value means stricter filter.
+  , scCopyPasteCheckEnabled :: Field f Bool
+    -- ^ Whether copy-paste check is enabled globally.
   } deriving stock (Generic)
 
 makeLensesWith postfixFields ''Config'
@@ -94,6 +96,9 @@ overrideConfig config
                   , scAnchorSimilarityThreshold =
                     fromMaybe (scAnchorSimilarityThreshold defScanners)
                     $ scAnchorSimilarityThreshold (cScanners config)
+                  , scCopyPasteCheckEnabled =
+                    fromMaybe (scCopyPasteCheckEnabled defScanners)
+                    $ scCopyPasteCheckEnabled (cScanners config)
                   }
     }
   where

src/Xrefcheck/Config/Default.hs

@@ -67,6 +67,9 @@ scanners:
     #
     # This affects which anchors are generated for headers.
     flavor: #s{flavor}
+
+  # Whether copy-paste check is enabled globally.
+  copyPasteCheckEnabled: True
 |]
   where
     ignoreLocalRefsFrom :: NonEmpty Text

src/Xrefcheck/Core.hs

@@ -12,6 +12,7 @@ module Xrefcheck.Core where
 import Universum
 
 import Control.Lens (makeLenses)
+import Control.Lens.Combinators (makeLensesWith)
 import Data.Aeson (FromJSON (..), withText)
 import Data.Char (isAlphaNum)
 import Data.Char qualified as C
@@ -70,14 +71,17 @@ instance Given ColorMode => Buildable Position where
 
 -- | Full info about a reference.
 data Reference = Reference
-  { rName   :: Text
+  { rName           :: Text
     -- ^ Text displayed as reference.
-  , rLink   :: Text
+  , rLink           :: Text
     -- ^ File or site reference points to.
-  , rAnchor :: Maybe Text
+  , rAnchor         :: Maybe Text
     -- ^ Section or custom anchor tag.
-  , rPos    :: Position
+  , rPos            :: Position
+  , rCheckCopyPaste :: Bool
+    -- ^ Whether to check bad copy/paste for this link
   } deriving stock (Show, Generic)
+makeLensesWith postfixFields ''Reference
 
 -- | Context of anchor.
 data AnchorType
@@ -102,9 +106,9 @@ data FileInfoDiff = FileInfoDiff
   }
 makeLenses ''FileInfoDiff
 
-diffToFileInfo :: FileInfoDiff -> FileInfo
-diffToFileInfo (FileInfoDiff refs anchors) =
-    FileInfo (DList.toList refs) (DList.toList anchors)
+diffToFileInfo :: Bool -> FileInfoDiff -> FileInfo
+diffToFileInfo ignoreCpcInFile (FileInfoDiff refs anchors) =
+    FileInfo (DList.toList refs) (DList.toList anchors) ignoreCpcInFile
 
 instance Semigroup FileInfoDiff where
   FileInfoDiff a b <> FileInfoDiff c d = FileInfoDiff (a <> c) (b <> d)
@@ -114,13 +118,14 @@ instance Monoid FileInfoDiff where
 
 -- | All information regarding a single file we care about.
 data FileInfo = FileInfo
-  { _fiReferences :: [Reference]
-  , _fiAnchors    :: [Anchor]
+  { _fiReferences     :: [Reference]
+  , _fiAnchors        :: [Anchor]
+  , _fiCopyPasteCheck :: Bool
   } deriving stock (Show, Generic)
 makeLenses ''FileInfo
 
 instance Default FileInfo where
-  def = diffToFileInfo mempty
+  def = diffToFileInfo True mempty
 
 data ScanPolicy
   = OnlyTracked

src/Xrefcheck/Scan.hs

@@ -117,18 +117,27 @@ data ScanErrorDescription
   = LinkErr
   | FileErr
   | ParagraphErr Text
+  | LinkErrCpc
+  | FileErrCpc
+  | ParagraphErrCpc Text
   | UnrecognisedErr Text
   deriving stock (Show, Eq)
 
 instance Buildable ScanErrorDescription where
   build = \case
     LinkErr -> [int||Expected a LINK after "ignore link" annotation|]
+    LinkErrCpc -> [int||Expected a LINK after "no duplication check in link" annotation|]
     FileErr -> [int||Annotation "ignore all" must be at the top of \
                      markdown or right after comments at the top|]
+    FileErrCpc -> [int||Annotation "no duplication check in file" must be at the top of \
+                     markdown or right after comments at the top|]
     ParagraphErr txt -> [int||Expected a PARAGRAPH after \
                               "ignore paragraph" annotation, but found #{txt}|]
-    UnrecognisedErr txt -> [int||Unrecognised option "#{txt}" perhaps you meant \
-                                 <"ignore link"|"ignore paragraph"|"ignore all">|]
+    ParagraphErrCpc txt -> [int||Expected a PARAGRAPH after \
+                              "no duplication check in paragraph" annotation, but found #{txt}|]
+    UnrecognisedErr txt -> [int||Unrecognised option "#{txt}", perhaps you meant
+                                 <"ignore link"|"ignore paragraph"|"ignore all">
+                                 or "no duplication check in <link|paragraph|file>"?|]
 
 specificFormatsSupport :: [([Extension], ScanAction)] -> FormatsSupport
 specificFormatsSupport formats = \ext -> M.lookup ext formatsMap