Content Downloading rework.

.github/workflows/javasteam-build-push.yml

@@ -51,3 +51,7 @@ jobs:
         with:
           name: javasteam-tf
           path: javasteam-tf/build/libs
+      - uses: actions/upload-artifact@v4
+        with:
+          name: javasteam-depotdownloader
+          path: javasteam-depotdownloader/build/libs

.gitignore

@@ -75,7 +75,9 @@ loginkey.txt
 sentry.bin
 server_list.bin
 /steamapps/
+/depots/
 /userfiles/
+refreshtoken.txt
 
 # Kotlin 2.0
 /.kotlin/sessions/

build.gradle.kts

@@ -20,7 +20,7 @@ plugins {
 
 allprojects {
     group = "in.dragonbra"
-    version = "1.7.1-SNAPSHOT"
+    version = "1.8.0-SNAPSHOT"
 }
 
 repositories {
@@ -133,10 +133,10 @@ tasks.withType<FormatTask> {
 
 dependencies {
     implementation(libs.bundles.ktor)
+    implementation(libs.bundles.okHttp)
     implementation(libs.commons.lang3)
     implementation(libs.kotlin.coroutines)
     implementation(libs.kotlin.stdib)
-    implementation(libs.okHttp)
     implementation(libs.protobuf.java)
     compileOnly(libs.xz)
     compileOnly(libs.zstd)

gradle/libs.versions.toml

@@ -21,6 +21,10 @@ publishPlugin = "2.0.0" # https://mvnrepository.com/artifact/io.github.gradle-ne
 xz = "1.10" # https://mvnrepository.com/artifact/org.tukaani/xz
 zstd = "1.5.7-4" # https://search.maven.org/artifact/com.github.luben/zstd-jni
 
+# Depot Downloader
+kotlin-serialization-json = "1.9.0" # https://mvnrepository.com/artifact/org.jetbrains.kotlinx/kotlinx-serialization-json-jvm
+okio = "3.16.0" # https://mvnrepository.com/artifact/com.squareup.okio/okio
+
 # Testing Lib versions
 commons-io = "2.20.0" # https://mvnrepository.com/artifact/commons-io/commons-io
 commonsCodec = "1.19.0" # https://mvnrepository.com/artifact/commons-codec/commons-codec
@@ -41,11 +45,16 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" }
 ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
 ktor-client-websocket = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" }
 okHttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okHttp" }
+okHttp-coroutines = { module = "com.squareup.okhttp3:okhttp-coroutines", version.ref = "okHttp" }
 protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" }
 protobuf-protoc = { module = "com.google.protobuf:protoc", version.ref = "protobuf" }
 xz = { module = "org.tukaani:xz", version.ref = "xz" }
 zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" }
 
+# Depot Downloader
+kotlin-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlin-serialization-json"}
+okio = { module = "com.squareup.okio:okio", version.ref = "okio"}
+
 # Tests
 test-commons-codec = { module = "commons-codec:commons-codec", version.ref = "commonsCodec" }
 test-commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" }
@@ -65,6 +74,7 @@ qrCode = { module = "pro.leaco.qrcode:console-qrcode", version.ref = "qrCode" }
 kotlin-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" }
 kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" }
 kotlin-kotlinter = { id = "org.jmailen.kotlinter", version.ref = "kotlinter" }
+kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" }
 maven-publish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "publishPlugin" }
 protobuf-gradle = { id = "com.google.protobuf", version.ref = "protobuf-gradle" }
 
@@ -86,3 +96,8 @@ ktor = [
     "ktor-client-cio",
     "ktor-client-websocket",
 ]
+
+okHttp = [
+    "okHttp",
+    "okHttp-coroutines",
+]

javasteam-depotdownloader/.gitignore

@@ -0,0 +1 @@
+build/

javasteam-depotdownloader/build.gradle.kts

@@ -0,0 +1,116 @@
+import org.gradle.kotlin.dsl.withType
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
+import org.jmailen.gradle.kotlinter.tasks.FormatTask
+import org.jmailen.gradle.kotlinter.tasks.LintTask
+
+plugins {
+    alias(libs.plugins.kotlin.dokka)
+    alias(libs.plugins.kotlin.jvm)
+    alias(libs.plugins.kotlin.kotlinter)
+    alias(libs.plugins.kotlin.serialization)
+    alias(libs.plugins.protobuf.gradle)
+    id("maven-publish")
+    id("signing")
+}
+
+repositories {
+    mavenCentral()
+}
+
+kotlin {
+    compilerOptions {
+        jvmTarget.set(JvmTarget.fromTarget(libs.versions.java.get()))
+    }
+}
+
+/* Protobufs */
+protobuf.protoc {
+    artifact = libs.protobuf.protoc.get().toString()
+}
+
+java {
+    sourceCompatibility = JavaVersion.toVersion(libs.versions.java.get())
+    targetCompatibility = JavaVersion.toVersion(libs.versions.java.get())
+    withSourcesJar()
+}
+
+/* Java-Kotlin Docs */
+dokka {
+    moduleName.set("JavaSteam")
+    dokkaSourceSets.main {
+        suppressGeneratedFiles.set(false) // Allow generated files to be documented.
+        perPackageOption {
+            // Deny most of the generated files.
+            matchingRegex.set("in.dragonbra.javasteam.(protobufs|enums|generated).*")
+            suppress.set(true)
+        }
+    }
+}
+
+// Make sure Maven Publishing gets javadoc
+val javadocJar by tasks.registering(Jar::class) {
+    dependsOn(tasks.dokkaGenerate)
+    archiveClassifier.set("javadoc")
+    from(layout.buildDirectory.dir("dokka/html"))
+}
+artifacts {
+    archives(javadocJar)
+}
+
+/* Kotlinter */
+tasks.withType<LintTask> {
+    this.source = this.source.minus(fileTree("build/generated")).asFileTree
+}
+tasks.withType<FormatTask> {
+    this.source = this.source.minus(fileTree("build/generated")).asFileTree
+}
+
+dependencies {
+    implementation(rootProject)
+
+    implementation(libs.bundles.ktor)
+    implementation(libs.commons.lang3)
+    implementation(libs.kotlin.coroutines)
+    implementation(libs.kotlin.serialization.json)
+    implementation(libs.kotlin.stdib)
+    implementation(libs.okio)
+    implementation(libs.protobuf.java)
+}
+
+/* Artifact publishing */
+publishing {
+    publications {
+        create<MavenPublication>("mavenJava") {
+            from(components["java"])
+            pom {
+                name = "JavaSteam-depotdownloader"
+                packaging = "jar"
+                description = "Depot Downloader for JavaSteam."
+                url = "https://github.com/Longi94/JavaSteam"
+                inceptionYear = "2025"
+                scm {
+                    connection = "scm:git:git://github.com/Longi94/JavaSteam.git"
+                    developerConnection = "scm:git:ssh://github.com:Longi94/JavaSteam.git"
+                    url = "https://github.com/Longi94/JavaSteam/tree/master"
+                }
+                licenses {
+                    license {
+                        name = "MIT License"
+                        url = "https://www.opensource.org/licenses/mit-license.php"
+                    }
+                }
+                developers {
+                    developer {
+                        id = "Longi"
+                        name = "Long Tran"
+                        email = "lngtrn94@gmail.com"
+                    }
+                }
+            }
+        }
+    }
+}
+
+signing {
+    sign(publishing.publications["mavenJava"])
+}

javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt

@@ -0,0 +1,138 @@
+package `in`.dragonbra.javasteam.depotdownloader
+
+import `in`.dragonbra.javasteam.steam.cdn.Client
+import `in`.dragonbra.javasteam.steam.cdn.Server
+import `in`.dragonbra.javasteam.util.log.LogManager
+import `in`.dragonbra.javasteam.util.log.Logger
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.concurrent.atomic.AtomicReference
+
+/**
+ * Manages a pool of CDN server connections for efficient content downloading.
+ * This class maintains a list of available CDN servers, automatically selects appropriate
+ * servers based on load and app compatibility, and handles connection rotation when
+ * servers fail or become unavailable.
+ *
+ * @param steamSession The Steam3 session for server communication
+ * @param appId The application ID to download - used to filter compatible CDN servers
+ * @param scope The coroutine scope for async operations
+ * @param debug If true, enables debug logging
+ *
+ * @author Oxters
+ * @author Lossy
+ * @since Nov 7, 2024
+ */
+class CDNClientPool(
+    private val steamSession: Steam3Session,
+    private val appId: Int,
+    private val scope: CoroutineScope,
+    debug: Boolean = false,
+) : AutoCloseable {
+
+    private var logger: Logger? = null
+
+    private val servers = AtomicReference<List<Server>>(emptyList())
+
+    private var nextServer: AtomicInteger = AtomicInteger(0)
+
+    private val mutex: Mutex = Mutex()
+
+    var cdnClient: Client? = null
+        private set
+
+    var proxyServer: Server? = null
+        private set
+
+    init {
+        cdnClient = Client(steamSession.steamClient)
+
+        if (debug) {
+            logger = LogManager.getLogger(CDNClientPool::class.java)
+        }
+    }
+
+    override fun close() {
+        logger?.debug("Closing...")
+
+        servers.set(emptyList())
+
+        cdnClient = null
+        proxyServer = null
+
+        logger = null
+    }
+
+    @Throws(Exception::class)
+    suspend fun updateServerList(maxNumServers: Int? = null) = mutex.withLock {
+        val serversForSteamPipe = steamSession.steamContent!!.getServersForSteamPipe(
+            cellId = steamSession.steamClient.cellID ?: 0,
+            maxNumServers = maxNumServers,
+            parentScope = scope
+        ).await()
+
+        proxyServer = serversForSteamPipe.firstOrNull { it.useAsProxy }
+
+        val weightedCdnServers = serversForSteamPipe
+            .filter { server ->
+                val isEligibleForApp = server.allowedAppIds.isEmpty() || server.allowedAppIds.contains(appId)
+                isEligibleForApp && (server.type == "SteamCache" || server.type == "CDN")
+            }
+            .sortedBy { it.weightedLoad }
+
+        // ContentServerPenalty removed for now.
+
+        servers.set(weightedCdnServers)
+
+        nextServer.set(0)
+
+        // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" }
+        logger?.debug("Found ${weightedCdnServers.size} Servers")
+
+        if (weightedCdnServers.isEmpty()) {
+            throw Exception("Failed to retrieve any download servers.")
+        }
+    }
+
+    fun getConnection(): Server {
+        val servers = servers.get()
+
+        val index = nextServer.getAndIncrement()
+        val server = servers[index % servers.size]
+
+        logger?.debug("Getting connection $server")
+
+        return server
+    }
+
+    fun returnConnection(server: Server?) {
+        if (server == null) {
+            logger?.error("null server returned to cdn pool.")
+            return
+        }
+
+        logger?.debug("Returning connection: $server")
+
+        // (SK) nothing to do, maybe remove from ContentServerPenalty?
+    }
+
+    fun returnBrokenConnection(server: Server?) {
+        if (server == null) {
+            logger?.error("null broken server returned to pool")
+            return
+        }
+
+        logger?.debug("Returning broken connection: $server")
+
+        val servers = servers.get()
+        val currentIndex = nextServer.get()
+
+        if (servers.isNotEmpty() && servers[currentIndex % servers.size] == server) {
+            nextServer.incrementAndGet()
+
+            // TODO: (SK) Add server to ContentServerPenalty
+        }
+    }
+}