#1181: UrlUpdater fetches every defined version for Docker

url-updater/src/main/java/com/devonfw/tools/ide/url/tool/docker/DockerDesktopUrlUpdater.java

@@ -1,5 +1,7 @@
 package com.devonfw.tools.ide.url.tool.docker;
 
+import static java.lang.String.format;
+
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -19,6 +21,32 @@ public class DockerDesktopUrlUpdater extends WebsiteUrlUpdater {
 
   private static final Pattern VERSION_PATTERN = Pattern.compile("(4\\.\\d{1,4}+\\.\\d+)");
 
+  private final static String AMD_ARCH_TYPE = "amd64";
+  private final static String ARM_ARCH_TYPE = "arm64";
+  private final static String CHECKSUM_FILE = "checksums.txt";
+
+  private final static String WIN_VERSION = "win";
+  private final static String WIN_FILE = "Docker%20Desktop%20Installer.exe";
+  private final static String WIN_BASE_URL = "/win/main/";
+  private final static String WIN_AMD_URL = WIN_BASE_URL + AMD_ARCH_TYPE + "/";
+  private final static String WIN_ARM_URL = WIN_BASE_URL + ARM_ARCH_TYPE + "/";
+
+  private final static String MAC_VERSION = "mac";
+  private final static String MAC_FILE = "Docker.dmg";
+  private final static String MAC_BASE_URL = "/mac/main/";
+  private final static String MAC_AMD_URL = MAC_BASE_URL + AMD_ARCH_TYPE + "/";
+  private final static String MAC_ARM_URL = MAC_BASE_URL + ARM_ARCH_TYPE + "/";
+
+  private final static String REGEX_FOR_DOWNLOAD_URLS = "/%s/main/%s/%s/";
+  private final static String REGEX_FOR_DOCKER_VERSION =
+      "href=#%s"                                  // Find the href with the readable version - %s provided by urlVersion
+          + ".{0,300}"                            // We have to look in close range for the next part (docker lists a summary at top. If this range is too big
+          // we'll find the latest listed version with download links that doesn't match the version we are looking for
+          + "href=(\")?%s"                         // Start of download link
+          + ".*?"                                 // We don't care if its windows or mac - match as least as possible characters
+          + "(\\d{5,6})";                         // Associated docker-version to readable version we are looking for
+
+
   @Override
   protected String getTool() {
 
@@ -32,26 +60,100 @@ protected void addVersion(UrlVersion urlVersion) {
     String version = urlVersion.getName().replaceAll("\\.", "");
     // get Code for version
     String body = doGetResponseBodyAsString(getVersionUrl());
-    String regex = "href=#" + version
-        // .......1.........................................................2.................
-        + ".{8,12}(\r\n|\r|\n).{0,350}href=https://desktop\\.docker\\.com.*?(\\d{5,6}).*\\.exe";
-    Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
-    Matcher matcher = pattern.matcher(body);
+    String regexForDockerVersion = format(REGEX_FOR_DOCKER_VERSION, version, getDownloadBaseUrl());
+    Pattern patternForDockerVersion = Pattern.compile(regexForDockerVersion, Pattern.DOTALL);
+    Matcher matcherForDockerVersion = patternForDockerVersion.matcher(body);
     String code;
-    if (matcher.find()) {
-      code = matcher.group(2);
-      boolean success = doAddVersion(urlVersion,
-          getDownloadBaseUrl() + "/win/main/amd64/" + code + "/Docker%20Desktop%20Installer.exe", WINDOWS);
+    if (matcherForDockerVersion.find()) {
+      code = matcherForDockerVersion.group(2);
+      boolean success;
+      success = addVersionsForWindows(urlVersion, code, body);
       if (!success) {
         return;
       }
       if (WINDOWS_ONLY_VERSIONS.stream().noneMatch(i -> vid.compareVersion(i).isEqual())) {
-        doAddVersion(urlVersion, getDownloadBaseUrl() + "/mac/main/amd64/" + code + "/Docker.dmg", MAC, X64);
-        doAddVersion(urlVersion, getDownloadBaseUrl() + "/mac/main/arm64/" + code + "/Docker.dmg", MAC, ARM64);
+        addVersionsForMac(urlVersion, code, body);
       }
     }
   }
 
+  /**
+   * Adds the windows versions for docker if they exist
+   *
+   * @param urlVersion the readable version e.g. 4332 (4.33.2)
+   * @param dockerVersion the associated docker version to readable version e.g. 179689
+   * @param body the html body to search in
+   * @return {@code true} if version was added successfully, {@code false} if not.
+   */
+  private boolean addVersionsForWindows(UrlVersion urlVersion, String dockerVersion, String body) {
+    boolean versionExists = checkIfVersionExists(dockerVersion, body, WIN_VERSION, AMD_ARCH_TYPE);
+    boolean success = false;
+    if (versionExists) {
+      String cs = getChecksum(getDownloadBaseUrl() + WIN_AMD_URL + dockerVersion + "/" + CHECKSUM_FILE);
+      success = doAddVersion(urlVersion, getDownloadBaseUrl() + WIN_AMD_URL + dockerVersion + "/" + WIN_FILE, WINDOWS, X64, cs);
+    }
+    versionExists = checkIfVersionExists(dockerVersion, body, WIN_VERSION, ARM_ARCH_TYPE);
+    if (versionExists) {
+      String cs = getChecksum(getDownloadBaseUrl() + WIN_ARM_URL + dockerVersion + "/" + CHECKSUM_FILE);
+      success = doAddVersion(urlVersion, getDownloadBaseUrl() + WIN_ARM_URL + dockerVersion + "/" + WIN_FILE, WINDOWS, ARM64, cs);
+    }
+    return success;
+  }
+
+  /**
+   * Adds the mac versions for docker if they exist
+   *
+   * @param urlVersion the readable version e.g. 4332 (4.33.2)
+   * @param dockerVersion the associated docker version to readable version e.g. 179689
+   * @param body the html body to search in
+   * @return {@code true} if version was added successfully, {@code false} if not.
+   */
+  private boolean addVersionsForMac(UrlVersion urlVersion, String dockerVersion, String body) {
+    boolean versionExists = checkIfVersionExists(dockerVersion, body, MAC_VERSION, AMD_ARCH_TYPE);
+    boolean success = false;
+    if (versionExists) {
+      String cs = getChecksum(getDownloadBaseUrl() + MAC_AMD_URL + dockerVersion + "/" + CHECKSUM_FILE);
+      success = doAddVersion(urlVersion, getDownloadBaseUrl() + MAC_AMD_URL + dockerVersion + "/" + MAC_FILE, MAC, X64, cs);
+    }
+    versionExists = checkIfVersionExists(dockerVersion, body, MAC_VERSION, ARM_ARCH_TYPE);
+    if (versionExists) {
+      String cs = getChecksum(getDownloadBaseUrl() + MAC_ARM_URL + dockerVersion + "/" + CHECKSUM_FILE);
+      success = doAddVersion(urlVersion, getDownloadBaseUrl() + MAC_ARM_URL + dockerVersion + "/" + MAC_FILE, MAC, ARM64, cs);
+    }
+    return success;
+  }
+
+  /**
+   * As docker is very inconsistent by releasing versions we have to check every single one if the download link exists to prevent failing downloads
+   * (403-errors) E.g. for only amd64 windows download link provided- <a href="https://docs.docker.com/desktop/release-notes/#4241">4.24.1</a> E.g. for no
+   * download links provided - <a href="https://docs.docker.com/desktop/release-notes/#4242">4.24.2</a> E.g. for only mac download links provided - <a
+   * href="https://docs.docker.com/desktop/release-notes/#4361">4.36.1</a>
+   *
+   * @param dockerVersion the associated docker version to readable version e.g. 179689 (4.33.2)
+   * @param body the html body to search in
+   * @param osVersion the os versions - win or mac
+   * @param archType the archType - amd64 or arm64
+   * @return true if the version exists - false if not
+   */
+  private boolean checkIfVersionExists(String dockerVersion, String body, String osVersion, String archType) {
+    String regexForDownloadUrlS = format(getDownloadBaseUrl() + REGEX_FOR_DOWNLOAD_URLS, osVersion, archType, dockerVersion);
+    Pattern patternForDownloadUrls = Pattern.compile(regexForDownloadUrlS, Pattern.DOTALL);
+    Matcher matcherForDownloadUrls = patternForDownloadUrls.matcher(body);
+    return matcherForDownloadUrls.find();
+  }
+
+  /**
+   * Retrieves the checksum for the passed url
+   *
+   * @param url url for specific version to download
+   * @return the checksum in string format
+   */
+  private String getChecksum(String url) {
+    String checksumAsString = doGetResponseBodyAsString(url);
+    // Example checksum response: e832d4c2c99300436096b2e990220068e69ede845137a9dd63eff0a51e8a14e9 *Docker Desktop Installer.exe
+    return checksumAsString.split(" ")[0];
+  }
+
   @Override
   protected String getDownloadBaseUrl() {
 

url-updater/src/test/java/com/devonfw/tools/ide/url/tool/docker/DockerDesktopUrlUpdaterMock.java

@@ -0,0 +1,35 @@
+package com.devonfw.tools.ide.url.tool.docker;
+
+import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
+
+/**
+ * Mock of {@link DockerDesktopUrlUpdater} to allow integration testing with wiremock.
+ */
+public class DockerDesktopUrlUpdaterMock extends DockerDesktopUrlUpdater {
+
+  private final String baseUrl;
+
+  /**
+   * The constructor.
+   *
+   * @param wireMockRuntimeInfo wireMock server on a random port
+   */
+  public DockerDesktopUrlUpdaterMock(WireMockRuntimeInfo wireMockRuntimeInfo) {
+    super();
+    this.baseUrl = wireMockRuntimeInfo.getHttpBaseUrl();
+  }
+
+  @Override
+  public String getDownloadBaseUrl() {
+
+    return baseUrl;
+  }
+
+  @Override
+  protected String getVersionBaseUrl() {
+
+    return this.baseUrl;
+  }
+
+}
+

url-updater/src/test/java/com/devonfw/tools/ide/url/tool/docker/DockerDesktopUrlUpdaterTest.java

@@ -0,0 +1,55 @@
+package com.devonfw.tools.ide.url.tool.docker;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.any;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import com.devonfw.tools.ide.url.model.folder.UrlRepository;
+import com.devonfw.tools.ide.url.updater.AbstractUrlUpdaterTest;
+import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
+import com.github.tomakehurst.wiremock.junit5.WireMockTest;
+
+/**
+ * Test class for integrations of the {@link DockerDesktopUrlUpdater}.
+ */
+@WireMockTest
+public class DockerDesktopUrlUpdaterTest extends AbstractUrlUpdaterTest {
+
+  /**
+   * Integration test for {@link DockerDesktopUrlUpdater}: verifies that update creates expected files for DockerDesktop versions.
+   */
+  @Test
+  void testDockerDesktopUpdaterCreatesDownloadUrlsAndChecksums(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) throws IOException {
+    // arrange
+    stubFor(get(urlMatching("/desktop/release-notes/")).willReturn(aResponse().withStatus(200)
+        .withBody(readAndResolve(PATH_INTEGRATION_TEST.resolve("DockerDesktopUrlUpdater").resolve("index.html"), wmRuntimeInfo))));
+    stubFor(
+        any(urlMatching("/(win|mac)/main/(amd64|arm64)/[0-9.]+/D.*"))
+            .willReturn(aResponse().withStatus(200).withBody(DOWNLOAD_CONTENT)));
+    stubFor(
+        any(urlMatching("/.*/checksums.txt"))
+            .willReturn(
+                aResponse().withStatus(200).withBody(SHA_256)));
+
+    UrlRepository urlRepository = UrlRepository.load(tempDir);
+    DockerDesktopUrlUpdater updater = new DockerDesktopUrlUpdaterMock(wmRuntimeInfo);
+    // act
+    updater.update(urlRepository);
+
+    // assert
+    Path dockerDesktopEditionPath = tempDir.resolve("docker").resolve("docker");
+    for (String version : new String[] { "4.30.0", "4.34.0" }) {
+      Path pgadminVersionPath = dockerDesktopEditionPath.resolve(version);
+      assertUrlVersion(pgadminVersionPath, List.of("windows_x64", "windows_arm64", "mac_x64", "mac_arm64"));
+    }
+  }
+}