Create dynamically markdown rules

Author: davesnx

src/dune_rules/odoc.ml

@@ -486,7 +486,6 @@ let setup_generate sctx ~search_db odoc_file out =
       , [ Command.Args.A "-o"
         ; Command.Args.Path (Path.build (Paths.markdown_root ctx))
         ; Command.Args.Dep (Path.build odoc_file.odocl_file)
-        ; Command.Args.Hidden_targets [ Output_format.target out odoc_file ]
         ] )
     | Html | Json ->
       let search_args =
@@ -520,9 +519,6 @@ let setup_generate_html_and_json sctx ~search_db odoc_file =
   setup_generate sctx ~search_db:(Some search_db) odoc_file Json
 ;;
 
-let setup_generate_markdown sctx odoc_file =
-  setup_generate sctx ~search_db:None odoc_file Markdown
-;;
 
 let setup_css_rule sctx =
   let ctx = Super_context.context sctx in
@@ -891,10 +887,15 @@ let out_files ctx (output : Output_format.t) odocs =
 ;;
 
 let add_format_alias_deps ctx format target odocs =
-  let paths = out_files ctx format odocs in
-  Rules.Produce.Alias.add_deps
-    (Dep.format_alias format ctx target)
-    (Action_builder.paths paths)
+  match (format : Output_format.t) with
+  | Markdown ->
+    (* skip intermediate aliases since package directories are directory targets *)
+    Memo.return ()
+  | Html | Json ->
+    let paths = out_files ctx format odocs in
+    Rules.Produce.Alias.add_deps
+      (Dep.format_alias format ctx target)
+      (Action_builder.paths paths)
 ;;
 
 let setup_lib_html_rules_def =
@@ -994,9 +995,7 @@ let setup_pkg_html_rules sctx ~pkg : unit Memo.t =
 ;;
 
 let setup_lib_markdown_rules sctx lib =
-  let target = Lib lib in
-  let* odocs = odoc_artefacts sctx target in
-  let* () = Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) in
+  (* we don't call setup_generate_markdown per module because all modules in a package write to the same directory. Instead, the package-level rule will handle all generation. *)
   Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib)
 ;;
 
@@ -1009,8 +1008,45 @@ let setup_pkg_markdown_rules_def =
       Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib))
     in
     let all_odocs = pkg_odocs @ lib_odocs in
+    (* Generate all markdown for this package in ONE rule with directory target.
+       Since odoc generates multiple .md files per .odocl and we can't predict the filenames, we declare the entire package directory
+       as a directory target. *)
+    let* () =
+      if List.is_empty all_odocs
+      then Memo.return ()
+      else
+        let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in
+        let markdown_root = Paths.markdown_root ctx in
+        let rule =
+          let bash_cmd_args =
+            let open Action_builder.O in
+            let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in
+            let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in
+            let bash_cmd =
+              List.map all_odocs ~f:(fun odoc ->
+                let odocl_rel = Path.reach (Path.build odoc.odocl_file) ~from:(Path.build markdown_root) in
+                Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel)
+              |> String.concat ~sep:" && "
+            in
+            let* () =
+              List.map all_odocs ~f:(fun odoc -> Action_builder.path (Path.build odoc.odocl_file))
+              |> Action_builder.all
+              >>| ignore
+            in
+            Action_builder.return (Command.Args.S [ A "-c"; A bash_cmd ])
+          in
+          let deps = Action_builder.env_var "ODOC_SYNTAX" in
+          let open Action_builder.With_targets.O in
+          Action_builder.with_no_targets deps
+          >>> Command.run
+            ~dir:(Path.build markdown_root)
+            (Ok (Path.of_string "/bin/bash"))
+            [ Dyn bash_cmd_args ]
+          |> Action_builder.With_targets.add_directories ~directory_targets:[ pkg_markdown_dir ]
+        in
+        add_rule sctx rule
+    in
     let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in
-    let* () = Memo.parallel_iter pkg_odocs ~f:(setup_generate_markdown sctx) in
     add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs
   in
   setup_pkg_rules_def "setup-package-markdown-rules" f
@@ -1031,11 +1067,21 @@ let setup_package_aliases_format sctx (pkg : Package.t) (output : Output_format.
   let* libs =
     Context.name ctx |> libs_of_pkg ~pkg:name >>| List.map ~f:(fun lib -> Lib lib)
   in
-  Pkg name :: libs
-  |> List.map ~f:(Dep.format_alias output ctx)
-  |> Dune_engine.Dep.Set.of_list_map ~f:(fun f -> Dune_engine.Dep.alias f)
-  |> Action_builder.deps
-  |> Rules.Produce.Alias.add_deps alias
+  let deps =
+    match (output : Output_format.t) with
+    | Markdown ->
+      let pkg_markdown_dir = Paths.markdown ctx (Pkg name) in
+      let index_path = Paths.markdown_index ctx in
+      let open Action_builder.O in
+      let* () = Action_builder.path (Path.build pkg_markdown_dir) in
+      Action_builder.path (Path.build index_path)
+    | Html | Json ->
+      Pkg name :: libs
+      |> List.map ~f:(Dep.format_alias output ctx)
+      |> Dune_engine.Dep.Set.of_list_map ~f:(fun f -> Dune_engine.Dep.alias f)
+      |> Action_builder.deps
+  in
+  Rules.Produce.Alias.add_deps alias deps
 ;;
 
 let setup_package_aliases sctx (pkg : Package.t) =
@@ -1171,36 +1217,31 @@ let gen_rules sctx ~dir rest =
        >>> setup_css_rule sctx
        >>> setup_toplevel_index_rule sctx Html
        >>> setup_toplevel_index_rule sctx Json)
-  | [ "_markdown" ] -> has_rules (setup_toplevel_index_rule sctx Markdown)
-  | [ "_markdown"; lib_unique_name_or_pkg ] ->
+  | [ "_markdown" ] ->
+    (* Generate all markdown and declare package directories as directory targets *)
+    let* packages = Dune_load.packages () in
+    let ctx = Super_context.context sctx in
+    (* Collect all package directories that will be generated as directory targets *)
+    let pkg_dirs =
+      Package.Name.Map.to_list packages
+      |> List.map ~f:(fun (_, (pkg : Package.t)) ->
+        let pkg_name = Package.name pkg in
+        Paths.markdown ctx (Pkg pkg_name))
+    in
+    let directory_targets =
+      List.fold_left pkg_dirs ~init:Path.Build.Map.empty ~f:(fun acc dir ->
+        Path.Build.Map.set acc dir Loc.none)
+    in
     has_rules
-      (
-       let ctx = Super_context.context sctx in
-       let* lib, lib_db = Scope_key.of_string (Context.name ctx) lib_unique_name_or_pkg in
-       let* lib =
-         let+ lib = Lib.DB.find lib_db lib in
-         Option.bind ~f:Lib.Local.of_lib lib
-       in
-       let+ () =
-         match lib with
-         | None -> Memo.return ()
-         | Some lib ->
-           (match Lib_info.package (Lib.Local.info lib) with
-            | None ->
-              (* lib with no package above it *)
-              setup_lib_markdown_rules sctx lib
-            | Some pkg -> setup_pkg_markdown_rules sctx ~pkg)
-       and+ () =
-         let* packages = Dune_load.packages () in
-         match
-           Package.Name.Map.find packages (Package.Name.of_string lib_unique_name_or_pkg)
-         with
-         | None -> Memo.return ()
-         | Some pkg ->
-           let name = Package.name pkg in
-           setup_pkg_markdown_rules sctx ~pkg:name
-       in
-       ())
+      ~directory_targets
+      (let* () = setup_toplevel_index_rule sctx Markdown in
+       Package.Name.Map.to_seq packages
+       |> Memo.parallel_iter_seq ~f:(fun (_, (pkg : Package.t)) ->
+         let pkg_name = Package.name pkg in
+         setup_pkg_markdown_rules sctx ~pkg:pkg_name))
+  | [ "_markdown"; _lib_unique_name_or_pkg ] ->
+    (* Package directories are directory targets, no rules here *)
+    Memo.return Gen_rules.no_rules
   | [ "_mlds"; pkg ] ->
     with_package pkg ~f:(fun pkg ->
       let pkg = Package.name pkg in