add geyser docs download

Author: Delwing

build.gradle

@@ -23,7 +23,7 @@ ext {
 }
 
 dependencies {
-	implementation 'org.springframework.cloud:spring-cloud-starter-openfeign'
+	implementation 'org.springframework.boot:spring-boot-starter-json:2.5.4'
 	implementation 'com.dropbox.core:dropbox-core-sdk:4.0.0'
 	compileOnly 'org.projectlombok:lombok'
 	annotationProcessor 'org.projectlombok:lombok'

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/PagesUrls.java renamed to src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/AppConfig.java

@@ -7,9 +7,9 @@
 import java.util.List;
 
 @Component
-@ConfigurationProperties(prefix = "")
+@ConfigurationProperties(prefix = "mudlet-docs")
 @Data
-public class PagesUrls {
+public class AppConfig {
 
     private List<String> pages;
 

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/MudletDocsApplication.java

@@ -2,11 +2,9 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.web.client.RestTemplate;
 
-@EnableFeignClients(basePackages = "pl.nullpointersoftware")
 @SpringBootApplication
 public class MudletDocsApplication {
 

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/PagesRunner.java

@@ -8,38 +8,38 @@
 import pl.nullpointersoftware.mudlet.mudletdocs.model.LuaDescriptor;
 import pl.nullpointersoftware.mudlet.mudletdocs.service.LuaWriter;
 import pl.nullpointersoftware.mudlet.mudletdocs.service.PageAnalyzer;
+import pl.nullpointersoftware.mudlet.mudletdocs.service.api.GithubRestClient;
 import pl.nullpointersoftware.mudlet.mudletdocs.service.api.WikiRestClient;
 import pl.nullpointersoftware.mudlet.mudletdocs.service.dropbox.DropBoxUploader;
 import pl.nullpointersoftware.mudlet.mudletdocs.service.dropbox.DropBoxUploaderFactory;
+import pl.nullpointersoftware.mudlet.mudletdocs.service.github.GithubDownloader;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.List;
 import java.util.Set;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @Component
 @Data
 @Slf4j
 public class PagesRunner implements ApplicationRunner {
 
-    private final PagesUrls pagesUrls;
+    private final AppConfig appConfig;
     private final PageAnalyzer pageAnalyzer;
     private final LuaWriter luaWriter;
     private final WikiRestClient wikiRestClient;
+    private final GithubDownloader githubDownloader;
     private final DropBoxUploaderFactory dropBoxUploaderFactory;
 
     @Override
     public void run(ApplicationArguments args) throws InterruptedException, URISyntaxException, IOException {
         DropBoxUploader dropBoxUploader = dropBoxUploaderFactory.getClient();
-        ExecutorService executor = Executors.newFixedThreadPool(pagesUrls.getPages().size());
-        pagesUrls.getPages().forEach( pageName -> {
+        ExecutorService executor = Executors.newFixedThreadPool(appConfig.getPages().size());
+        appConfig.getPages().forEach(pageName -> {
             executor.submit(() -> {
                 log.info("Processing {}", pageName);
                 Set<LuaDescriptor> descriptors = pageAnalyzer.analyze(wikiRestClient.getPage(pageName).getParse().getWikitext());
@@ -50,6 +50,8 @@ public void run(ApplicationArguments args) throws InterruptedException, URISynta
         executor.shutdown();
         executor.awaitTermination(1, TimeUnit.MINUTES);
 
+        githubDownloader.downloadAll(dropBoxUploader);
+
         dropBoxUploader.addFile(luaWriter.generateFile("lfs.lua", LfsDoc.DESCRIPTORS));
         dropBoxUploader.addFile(luaWriter.generateFile("rex.lua", RexDoc.DESCRIPTORS));
 

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/model/GithubBlobResponse.java

@@ -0,0 +1,10 @@
+package pl.nullpointersoftware.mudlet.mudletdocs.model;
+
+import lombok.Data;
+
+@Data
+public class GithubBlobResponse {
+
+    private String content;
+
+}

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/model/GithubTreeResponse.java

@@ -0,0 +1,21 @@
+package pl.nullpointersoftware.mudlet.mudletdocs.model;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class GithubTreeResponse {
+
+    private String sha;
+    private String url;
+    private List<TreeItem> tree;
+
+    @Data
+    public static class TreeItem {
+        private String path;
+        private String url;
+        private String sha;
+    }
+
+}

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/service/api/GithubRestClient.java

@@ -0,0 +1,41 @@
+package pl.nullpointersoftware.mudlet.mudletdocs.service.api;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Component;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.RestTemplate;
+import pl.nullpointersoftware.mudlet.mudletdocs.model.GithubBlobResponse;
+import pl.nullpointersoftware.mudlet.mudletdocs.model.GithubTreeResponse;
+
+@Component
+@RequiredArgsConstructor
+public class GithubRestClient {
+
+    public static final String TREE_URL = "https://api.github.com/repos/Mudlet/Mudlet/git/trees/development?recursive=1";
+    public static final String BLOB_URL = "https://api.github.com/repos/Mudlet/Mudlet/git/blobs/";
+    private final RestTemplate restTemplate;
+
+    public GithubTreeResponse getTree() {
+        MultiValueMap<String, String> headers = new LinkedMultiValueMap<>();
+        headers.add("User-Agent", "Dargoth parser");
+
+        HttpEntity<String> entity = new HttpEntity<>("body", headers);
+
+        return restTemplate.exchange(TREE_URL, HttpMethod.GET, entity, GithubTreeResponse.class).getBody();
+    }
+
+    public GithubBlobResponse getBlob(String sha) {
+        MultiValueMap<String, String> headers = new LinkedMultiValueMap<>();
+        headers.add("User-Agent", "Dargoth parser");
+
+        HttpEntity<String> entity = new HttpEntity<>("body", headers);
+
+        return restTemplate.exchange(BLOB_URL + sha, HttpMethod.GET, entity, GithubBlobResponse.class).getBody();
+    }
+
+
+
+}

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/service/api/WikiApiClient.java

@@ -25,4 +25,6 @@ public WikiResponse getPage(String page) {
         return restTemplate.exchange(String.format(URL, page), HttpMethod.GET, entity, WikiResponse.class).getBody();
     }
 
+
+
 }

src/main/java/pl/nullpointersoftware/mudlet/mudletdocs/service/api/WikiRestClient.java

@@ -5,7 +5,10 @@
 import java.nio.file.Path;
 
 public interface DropBoxUploader {
-    void addFile(Path path);
+    default void addFile(Path path) {
+        addFile(path, "");
+    }
+    void addFile(Path path, String dir);
 
     @SneakyThrows
     void finish();