feat: update articles or snippets

Author: wx-chevalier

dd-search-engine/sg-index-doc/src/cli/build-meili-index.ts

@@ -21,28 +21,34 @@ program
       "commit-all.sh",
       "Private",
       "Playground",
+      "Awesome-Lists",
+      "Awesome-CS-Books-and-Digests",
     ];
 
     firstDirs
       .filter((d) => !ignores.includes(d))
       .forEach(async (dir) => {
-        const dirPath = path.join(baseLocalDir, dir);
-
-        const repos = fs
-          .readdirSync(dirPath)
-          .filter((d) => !ignores.includes(d))
-          .map(
-            (repoDirName) =>
-              ({
-                name: repoDirName,
-                title: repoDirName,
-                localPath: path.join(dirPath, repoDirName),
-                description: repoDirName,
-                sUrl: `https://ng-tech.icu/${repoDirName}`,
-              } as ReposityConfig)
-          );
-
-        await buildDocIndex(repos);
+        try {
+          const dirPath = path.join(baseLocalDir, dir);
+
+          const repos = fs
+            .readdirSync(dirPath)
+            .filter((d) => !ignores.includes(d))
+            .map(
+              (repoDirName) =>
+                ({
+                  name: repoDirName,
+                  title: repoDirName,
+                  localPath: path.join(dirPath, repoDirName),
+                  description: repoDirName,
+                  sUrl: `https://ng-tech.icu/${repoDirName}`,
+                } as ReposityConfig)
+            );
+
+          await buildDocIndex(repos);
+        } catch (e) {
+          console.error(e);
+        }
       });
   });
 

dd-search-engine/sg-index-doc/src/service/index/buildDocIndexWithMeili.ts

@@ -27,28 +27,77 @@ export async function buildDocIndex(repos: ReposityConfig[]) {
     attributesForFaceting: ["categories", "repo"],
   });
 
-  repos.forEach((repo) => {
-    // 获取仓库的配置信息
-
-    const files = walkSync(repo.localPath).filter(
-      (path) =>
-        (path.endsWith(".md") || path.endsWith(".pdf")) &&
-        path !== "README.md" &&
-        path.indexOf("Weekly") === -1 &&
-        path.indexOf("examples") === -1
-    );
-
-    files.forEach(async (file) => {
-      // 封装出 href
-      const href = `${repo.sUrl}/blob/master/${file}`;
-      const absoluteFile = `${repo.localPath}/${file}`;
-      let fileName: string = file.split("/").reverse()[0];
-
-      // 判断是否为 PDF
-      if (file.endsWith("pdf")) {
-        index.addDocuments([
-          {
-            id: md5(href),
+  for (const repo of repos) {
+    try {
+      console.log(`>>>${repo.name}>>>start`);
+
+      // 获取仓库的配置信息
+      const files = walkSync(repo.localPath).filter(
+        (path) =>
+          (path.endsWith(".md") || path.endsWith(".pdf")) &&
+          path !== "README.md" &&
+          path.indexOf("Weekly") === -1 &&
+          path.indexOf("examples") === -1
+      );
+
+      console.log(`>>>${repo.name}>>>${files.length}`);
+
+      for (const file of files) {
+        try {
+          // 封装出 href
+          const href = `${repo.sUrl}/blob/master/${file}`;
+          const absoluteFile = `${repo.localPath}/${file}`;
+          let fileName: string = file.split("/").reverse()[0];
+
+          // 判断是否为 PDF
+          if (file.endsWith("pdf")) {
+            index.addDocuments([
+              {
+                id: md5(href),
+                fileName,
+                repo: repo.name,
+                categories: file
+                  .split("/")
+                  .filter(
+                    (c) =>
+                      Number.isNaN(parseInt(c, 10)) && c.indexOf(".md") === -1
+                  ),
+                href,
+                desc: fileName,
+                content: fileName,
+              },
+            ]);
+
+            continue;
+          }
+
+          // 读取文件内容
+          const content = await readFileAsync(absoluteFile, {
+            encoding: "utf-8",
+          });
+          const desc = (await readMarkdownHeadersFromFile(absoluteFile)).join(
+            ", "
+          );
+
+          if (!content) {
+            continue;
+          }
+
+          // 这里对文件内容进行预处理
+          const filteredContent = (content as string)
+            .replace(/\n/g, "")
+            .replace(/\[(.*)\]\(http.*\)/g, (_, __) => __)
+            .replace(/```\w*.*```/g, "");
+
+          const contents = [filteredContent];
+
+          if (!contents) {
+            continue;
+          }
+
+          // 分割过长内容
+          const objs = contents.map((content, index) => ({
+            id: index ? md5(`${href}${index}`) : md5(href),
             fileName,
             repo: repo.name,
             categories: file
@@ -57,56 +106,19 @@ export async function buildDocIndex(repos: ReposityConfig[]) {
                 (c) => Number.isNaN(parseInt(c, 10)) && c.indexOf(".md") === -1
               ),
             href,
-            desc: fileName,
-            content: fileName,
-          },
-        ]);
-
-        return;
-      }
-
-      // 读取文件内容
-      const content = await readFileAsync(absoluteFile, { encoding: "utf-8" });
-      const desc = (await readMarkdownHeadersFromFile(absoluteFile)).join(", ");
-
-      if (!content) {
-        return;
+            desc,
+            content,
+          }));
+
+          index.addDocuments(objs);
+        } catch (e) {
+          console.error(e);
+        }
       }
 
-      // 这里对文件内容进行预处理
-      const filteredContent = (content as string)
-        .replace(/\n/g, "")
-        .replace(/\[(.*)\]\(http.*\)/g, (_, __) => __)
-        .replace(/```\w*.*```/g, "");
-
-      const contents = [filteredContent];
-
-      if (!contents) {
-        return;
-      }
-
-      // 分割过长内容
-      const objs = contents.map((content, index) => ({
-        id: index ? md5(`${href}${index}`) : md5(href),
-        fileName,
-        repo: repo.name,
-        categories: file
-          .split("/")
-          .filter(
-            (c) => Number.isNaN(parseInt(c, 10)) && c.indexOf(".md") === -1
-          ),
-        href,
-        desc,
-        content,
-      }));
-
-      try {
-        index.addDocuments(objs);
-      } catch (e) {
-        console.error(e);
-      }
-    });
-
-    console.log(`${repo.name} indexed finally.`);
-  });
+      console.log(`${repo.name} indexed finally.`);
+    } catch (e) {
+      console.error(e);
+    }
+  }
 }