From fdb9e33720155c9fd533b19cb0824478ba25f244 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 10:53:59 -0500 Subject: [PATCH 01/21] Initial push of a new Depot Downloader --- .gitignore | 2 + build.gradle.kts | 2 +- gradle/libs.versions.toml | 7 + javasteam-depotdownloader/.gitignore | 1 + javasteam-depotdownloader/build.gradle.kts | 80 + .../depotdownloader/CDNClientPool.kt | 140 ++ .../depotdownloader/ContentDownloader.kt | 1889 +++++++++++++++++ .../ContentDownloaderException.kt | 7 + .../depotdownloader/DepotConfigStore.kt | 53 + .../javasteam/depotdownloader/HttpClient.kt | 46 + .../depotdownloader/IDownloadListener.kt | 35 + .../depotdownloader/Steam3Session.kt | 340 +++ .../javasteam/depotdownloader/Util.kt | 207 ++ .../depotdownloader/data/ChunkMatch.kt | 9 + .../depotdownloader/data/DepotDownloadInfo.kt | 41 + .../depotdownloader/data/DepotFilesData.kt | 20 + .../depotdownloader/data/DownloadCounters.kt | 26 + .../depotdownloader/data/DownloadItems.kt | 62 + .../depotdownloader/data/DownloadProgress.kt | 72 + .../depotdownloader/data/FileStreamData.kt | 16 + javasteam-samples/build.gradle.kts | 3 +- .../_023_downloadapp/SampleDownloadApp.java | 323 ++- settings.gradle.kts | 7 +- .../dragonbra/javasteam/steam/cdn/Client.kt | 367 ++-- .../javasteam/steam/cdn/ClientLancache.kt | 128 +- .../javasteam/steam/cdn/ClientPool.kt | 142 -- .../javasteam/steam/cdn/DepotChunk.kt | 7 +- .../dragonbra/javasteam/steam/cdn/Server.kt | 72 +- .../steam/contentdownloader/ChunkMatch.kt | 8 - .../contentdownloader/ContentDownloader.kt | 724 ------- .../contentdownloader/DepotDownloadCounter.kt | 8 - .../contentdownloader/DepotDownloadInfo.kt | 11 - .../steam/contentdownloader/DepotFilesData.kt | 11 - .../contentdownloader/FileManifestProvider.kt | 210 -- .../steam/contentdownloader/FileStreamData.kt | 10 - .../GlobalDownloadCounter.kt | 6 - .../contentdownloader/IManifestProvider.kt | 36 - .../MemoryManifestProvider.kt | 28 - .../contentdownloader/ProgressCallback.kt | 8 - .../steamcontent/CDNAuthToken.kt} | 6 +- .../handlers/steamcontent/SteamContent.kt | 16 +- .../ISteamConfigurationBuilder.kt | 9 - .../configuration/SteamConfiguration.kt | 7 - .../SteamConfigurationBuilder.kt | 8 - .../configuration/SteamConfigurationState.kt | 2 - .../webapi/ContentServerDirectoryService.kt | 33 +- .../in/dragonbra/javasteam/types/PubFile.kt | 108 + .../in/dragonbra/javasteam/util/Utils.java | 36 - .../javasteam/steam/cdn/CDNClientTest.java | 10 +- 49 files changed, 3790 insertions(+), 1609 deletions(-) create mode 100644 javasteam-depotdownloader/.gitignore create mode 100644 javasteam-depotdownloader/build.gradle.kts create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt rename src/main/java/in/dragonbra/javasteam/steam/{cdn/AuthToken.kt => handlers/steamcontent/CDNAuthToken.kt} (74%) create mode 100644 src/main/java/in/dragonbra/javasteam/types/PubFile.kt diff --git a/.gitignore b/.gitignore index 2f54edb3..781bc548 100644 --- a/.gitignore +++ b/.gitignore @@ -75,7 +75,9 @@ loginkey.txt sentry.bin server_list.bin /steamapps/ +/depots/ /userfiles/ +refreshtoken.txt # Kotlin 2.0 /.kotlin/sessions/ diff --git a/build.gradle.kts b/build.gradle.kts index 753cfba6..aed549cc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -133,10 +133,10 @@ tasks.withType { dependencies { implementation(libs.bundles.ktor) + implementation(libs.bundles.okHttp) implementation(libs.commons.lang3) implementation(libs.kotlin.coroutines) implementation(libs.kotlin.stdib) - implementation(libs.okHttp) implementation(libs.protobuf.java) compileOnly(libs.xz) compileOnly(libs.zstd) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d42f0aef..4a35e6ca 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -41,6 +41,7 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-websocket = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" } okHttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okHttp" } +okHttp-coroutines = { module = "com.squareup.okhttp3:okhttp-coroutines", version.ref = "okHttp" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } protobuf-protoc = { module = "com.google.protobuf:protoc", version.ref = "protobuf" } xz = { module = "org.tukaani:xz", version.ref = "xz" } @@ -65,6 +66,7 @@ qrCode = { module = "pro.leaco.qrcode:console-qrcode", version.ref = "qrCode" } kotlin-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } kotlin-kotlinter = { id = "org.jmailen.kotlinter", version.ref = "kotlinter" } +kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } maven-publish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "publishPlugin" } protobuf-gradle = { id = "com.google.protobuf", version.ref = "protobuf-gradle" } @@ -86,3 +88,8 @@ ktor = [ "ktor-client-cio", "ktor-client-websocket", ] + +okHttp = [ + "okHttp", + "okHttp-coroutines", +] diff --git a/javasteam-depotdownloader/.gitignore b/javasteam-depotdownloader/.gitignore new file mode 100644 index 00000000..d1638636 --- /dev/null +++ b/javasteam-depotdownloader/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/javasteam-depotdownloader/build.gradle.kts b/javasteam-depotdownloader/build.gradle.kts new file mode 100644 index 00000000..5ad11c36 --- /dev/null +++ b/javasteam-depotdownloader/build.gradle.kts @@ -0,0 +1,80 @@ +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jmailen.gradle.kotlinter.tasks.FormatTask +import org.jmailen.gradle.kotlinter.tasks.LintTask + +plugins { + alias(libs.plugins.kotlin.dokka) + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.kotlin.kotlinter) + alias(libs.plugins.kotlin.serialization) + alias(libs.plugins.protobuf.gradle) + id("maven-publish") + id("signing") +} + +repositories { + mavenCentral() +} + +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget(libs.versions.java.get())) + } +} + +/* Protobufs */ +protobuf.protoc { + artifact = libs.protobuf.protoc.get().toString() +} + + +java { + sourceCompatibility = JavaVersion.toVersion(libs.versions.java.get()) + targetCompatibility = JavaVersion.toVersion(libs.versions.java.get()) + withSourcesJar() +} + +/* Java-Kotlin Docs */ +dokka { + moduleName.set("JavaSteam") + dokkaSourceSets.main { + suppressGeneratedFiles.set(false) // Allow generated files to be documented. + perPackageOption { + // Deny most of the generated files. + matchingRegex.set("in.dragonbra.javasteam.(protobufs|enums|generated).*") + suppress.set(true) + } + } +} + +// Make sure Maven Publishing gets javadoc +val javadocJar by tasks.registering(Jar::class) { + dependsOn(tasks.dokkaGenerate) + archiveClassifier.set("javadoc") + from(layout.buildDirectory.dir("dokka/html")) +} +artifacts { + archives(javadocJar) +} + +/* Kotlinter */ +tasks.withType { + this.source = this.source.minus(fileTree("build/generated")).asFileTree +} +tasks.withType { + this.source = this.source.minus(fileTree("build/generated")).asFileTree +} + +dependencies { + implementation(rootProject) // TODO verify if this causes something like a circular dependency. + + implementation("com.squareup.okio:okio:3.16.0") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + + implementation(libs.bundles.ktor) + implementation(libs.commons.lang3) + implementation(libs.kotlin.coroutines) + implementation(libs.kotlin.stdib) + implementation(libs.protobuf.java) +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt new file mode 100644 index 00000000..6ea50137 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -0,0 +1,140 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.steam.cdn.Client +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlin.jvm.Throws + +/** + * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. + * @param steamClient an instance of [SteamClient] + * @param appId the selected app id to ensure an endpoint supports the download. + * @param scope (optional) the [CoroutineScope] to use. + * @param debug enable or disable logging through [LogManager] + * + * @author Oxters + * @author Lossy + * @since Nov 7, 2024 + */ +class CDNClientPool( + private val steamClient: SteamClient, + private val appId: Int, + private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), + debug: Boolean = false, +) : AutoCloseable { + + companion object { + fun init( + steamClient: SteamClient, + appId: Int, + debug: Boolean, + ): CDNClientPool = CDNClientPool(steamClient = steamClient, appId = appId, debug = debug) + } + + private var logger: Logger? = null + + private val servers: ArrayList = arrayListOf() + + private var nextServer: Int = 0 + + private val mutex: Mutex = Mutex() + + var cdnClient: Client? = null + private set + + var proxyServer: Server? = null + private set + + init { + cdnClient = Client(steamClient) + + if (debug) { + logger = LogManager.getLogger(CDNClientPool::class.java) + } + } + + override fun close() { + scope.cancel() + + servers.clear() + + cdnClient = null + proxyServer = null + + logger = null + } + + @Throws(Exception::class) + suspend fun updateServerList(maxNumServers: Int? = null) = mutex.withLock { + if (servers.isNotEmpty()) { + servers.clear() + } + + val serversForSteamPipe = steamClient.getHandler()!!.getServersForSteamPipe( + cellId = steamClient.cellID ?: 0, + maxNumServers = maxNumServers, + parentScope = scope + ).await() + + proxyServer = serversForSteamPipe.firstOrNull { it.useAsProxy } + + val weightedCdnServers = serversForSteamPipe + .filter { server -> + val isEligibleForApp = server.allowedAppIds.isEmpty() || server.allowedAppIds.contains(appId) + isEligibleForApp && (server.type == "SteamCache" || server.type == "CDN") + } + .sortedBy { it.weightedLoad } + + // ContentServerPenalty removed for now. + + servers.addAll(weightedCdnServers) + + // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } + logger?.debug("Found ${servers.size} Servers: \n") + + if (servers.isEmpty()) { + throw Exception("Failed to retrieve any download servers.") + } + } + + suspend fun getConnection(): Server = mutex.withLock { + val server = servers[nextServer % servers.count()] + + logger?.debug("Getting connection $server") + + return server + } + + suspend fun returnConnection(server: Server?) = mutex.withLock { + if (server == null) { + return@withLock + } + + logger?.debug("Returning connection: $server") + + // nothing to do, maybe remove from ContentServerPenalty? + } + + suspend fun returnBrokenConnection(server: Server?) = mutex.withLock { + if (server == null) { + return@withLock + } + + logger?.debug("Returning broken connection: $server") + + if (servers[nextServer % servers.count()] == server) { + nextServer++ + + // TODO: Add server to ContentServerPenalty + } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt new file mode 100644 index 00000000..2571e2c0 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -0,0 +1,1889 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.AppItem +import `in`.dragonbra.javasteam.depotdownloader.data.ChunkMatch +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadCounter +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo +import `in`.dragonbra.javasteam.depotdownloader.data.DepotFilesData +import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadStatus +import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress +import `in`.dragonbra.javasteam.depotdownloader.data.FileStreamData +import `in`.dragonbra.javasteam.depotdownloader.data.GlobalDownloadCounter +import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress +import `in`.dragonbra.javasteam.depotdownloader.data.PubFileItem +import `in`.dragonbra.javasteam.depotdownloader.data.UgcItem +import `in`.dragonbra.javasteam.enums.EAccountType +import `in`.dragonbra.javasteam.enums.EAppInfoSection +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.steam.cdn.ClientLancache +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamapps.License +import `in`.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.callback.UGCDetailsCallback +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.types.FileData +import `in`.dragonbra.javasteam.types.KeyValue +import `in`.dragonbra.javasteam.types.PublishedFileID +import `in`.dragonbra.javasteam.types.UGCHandle +import `in`.dragonbra.javasteam.util.Adler32 +import `in`.dragonbra.javasteam.util.SteamKitWebRequestException +import `in`.dragonbra.javasteam.util.Strings +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsChannel +import io.ktor.http.HttpHeaders +import io.ktor.utils.io.core.readAvailable +import io.ktor.utils.io.core.remaining +import io.ktor.utils.io.readRemaining +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.future.future +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.withContext +import kotlinx.coroutines.yield +import okio.Buffer +import okio.FileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer +import org.apache.commons.lang3.SystemUtils +import java.io.Closeable +import java.io.IOException +import java.lang.IllegalStateException +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.* +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import kotlin.collections.mutableListOf +import kotlin.collections.set +import kotlin.text.toLongOrNull + +/** + * [ContentDownloader] is a JavaSteam module that is able to download Games, Workshop Items, and other content from Steam. + * @param steamClient an instance of [SteamClient] + * @param licenses a list of licenses the logged-in user has. This is provided by [LicenseListCallback] + * @param debug enable or disable logging through [LogManager] + * @param useLanCache try and detect a local Steam Cache server. + * @param maxDownloads the number of simultaneous downloads. + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +@Suppress("unused") +class ContentDownloader @JvmOverloads constructor( + private val steamClient: SteamClient, + private val licenses: List, // To be provided from [LicenseListCallback] + private val debug: Boolean = false, // Enable debugging features, such as logging + private val useLanCache: Boolean = false, // Try and detect a lan cache server. + private var maxDownloads: Int = 8, // Max concurrent downloads +) : Closeable { + + companion object { + const val INVALID_APP_ID: Int = Int.MAX_VALUE + const val INVALID_DEPOT_ID: Int = Int.MAX_VALUE + const val INVALID_MANIFEST_ID: Long = Long.MAX_VALUE + + const val CONFIG_DIR: String = ".DepotDownloader" + const val DEFAULT_BRANCH: String = "public" + const val DEFAULT_DOWNLOAD_DIR: String = "depots" + + val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" + } + + // What is a PriorityQueue? + + private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } + + private val items = CopyOnWriteArrayList(ArrayList()) + private val listeners = CopyOnWriteArrayList() + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private var logger: Logger? = null + private val isStarted: AtomicBoolean = AtomicBoolean(false) + private val processingChannel = Channel(Channel.UNLIMITED) + private val remainingItems = AtomicInteger(0) + private val lastFileProgressUpdate = ConcurrentHashMap() + private val progressUpdateInterval = 500L // ms + + private var steam3: Steam3Session? = null + + private var cdnClientPool: CDNClientPool? = null + + private var config: Config = Config() + + // region [REGION] Private data classes. + + private data class NetworkChunkItem( + val fileStreamData: FileStreamData, + val fileData: FileData, + val chunk: ChunkData, + val totalChunksForFile: Int, + ) + + private data class DirectoryResult(val success: Boolean, val installDir: Path?) + + private data class Config( + val installPath: Path? = null, + val betaPassword: String? = null, + val downloadAllPlatforms: Boolean = false, + val downloadAllArchs: Boolean = false, + val downloadAllLanguages: Boolean = false, + val androidEmulation: Boolean = false, + val downloadManifestOnly: Boolean = false, + val installToGameNameDirectory: Boolean = false, + + // Not used yet in code + val usingFileList: Boolean = false, + var filesToDownloadRegex: List = emptyList(), + var filesToDownload: HashSet = hashSetOf(), + var verifyAll: Boolean = false, + ) + + // endregion + + init { + if (debug) { + logger = LogManager.getLogger(ContentDownloader::class.java) + } + + logger?.debug("DepotDownloader launched with ${licenses.size} for account") + + steam3 = Steam3Session(steamClient, debug) + } + + // region [REGION] Downloading Operations + @Throws(IllegalStateException::class) + suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { + val details = requireNotNull( + steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) + ) { "Pub File Null" } + + if (!details.fileUrl.isNullOrBlank()) { + downloadWebFile(appId, details.filename, details.fileUrl) + } else if (details.hcontentFile > 0) { + downloadApp( + appId = appId, + depotManifestIds = listOf(appId to details.hcontentFile), + branch = DEFAULT_BRANCH, + os = null, + arch = null, + language = null, + lv = false, + isUgc = true, + ) + } else { + logger?.error("Unable to locate manifest ID for published file $publishedFileId") + } + } + + suspend fun downloadUGC( + appId: Int, + ugcId: Long, + ) { + var details: UGCDetailsCallback? = null + + val steamUser = requireNotNull(steam3!!.steamUser) + val steamId = requireNotNull(steamUser.steamID) + + if (steamId.accountType != EAccountType.AnonUser) { + val ugcHandle = UGCHandle(ugcId) + details = steam3!!.getUGCDetails(ugcHandle) + } else { + logger?.error("Unable to query UGC details for $ugcId from an anonymous account") + } + + if (!details?.url.isNullOrBlank()) { + downloadWebFile(appId = appId, fileName = details.fileName, url = details.url) + } else { + downloadApp( + appId = appId, + depotManifestIds = listOf(appId to ugcId), + branch = DEFAULT_BRANCH, + os = null, + arch = null, + language = null, + lv = false, + isUgc = true, + ) + } + } + + @Throws(IllegalStateException::class, IOException::class) + suspend fun downloadWebFile(appId: Int, fileName: String, url: String) { + val (success, installDir) = createDirectories(appId, 0, appId) + + if (!success) { + logger?.debug("Error: Unable to create install directories!") + return + } + + val stagingDir = installDir!! / "staging" + val fileStagingPath = stagingDir / fileName + val fileFinalPath = installDir / fileName + + filesystem.createDirectories(fileFinalPath.parent!!) + filesystem.createDirectories(fileStagingPath.parent!!) + + HttpClient.httpClient.use { client -> + logger?.debug("Starting download of $fileName...") + + val response = client.get(url) + val channel = response.bodyAsChannel() + + val totalBytes = response.headers[HttpHeaders.ContentLength]?.toLongOrNull() + + logger?.debug("File size: ${totalBytes?.let { Util.formatBytes(it) } ?: "Unknown"}") + + filesystem.sink(fileStagingPath).buffer().use { sink -> + val buffer = Buffer() + while (!channel.isClosedForRead) { + val packet = channel.readRemaining(DEFAULT_BUFFER_SIZE.toLong()) + if (!packet.exhausted()) { + // Read from Ktor packet into Okio buffer + val bytesRead = packet.remaining.toInt() + val tempArray = ByteArray(bytesRead) + packet.readAvailable(tempArray) + buffer.write(tempArray) + + // Write from buffer to sink + sink.writeAll(buffer) + } + } + } + + logger?.debug("Download completed.") + } + + if (filesystem.exists(fileFinalPath)) { + logger?.debug("Deleting $fileFinalPath") + filesystem.delete(fileFinalPath) + } + + try { + filesystem.atomicMove(fileStagingPath, fileFinalPath) + logger?.debug("File '$fileStagingPath' moved to final location: $fileFinalPath") + } catch (e: IOException) { + throw e + } + } + + // L4D2 (app) supports LV + @Throws(IllegalStateException::class) + suspend fun downloadApp( + appId: Int, + depotManifestIds: List>, + branch: String, + os: String?, + arch: String?, + language: String?, + lv: Boolean, + isUgc: Boolean, + ) { + var depotManifestIds = depotManifestIds.toMutableList() + + val steamUser = requireNotNull(steam3!!.steamUser) + cdnClientPool = CDNClientPool.init(steamClient, appId, debug) + + // Load our configuration data containing the depots currently installed + var configPath = config.installPath + if (configPath == null) { + configPath = DEFAULT_DOWNLOAD_DIR.toPath() + } + + filesystem.createDirectories(configPath) + DepotConfigStore.loadFromFile(configPath / CONFIG_DIR / "depot.config") + + steam3!!.requestAppInfo(appId) + + if (!accountHasAccess(appId, appId)) { + if (steamUser.steamID!!.accountType != EAccountType.AnonUser && steam3!!.requestFreeAppLicense(appId)) { + logger?.debug("Obtained FreeOnDemand license for app $appId") + + // Fetch app info again in case we didn't get it fully without a license. + steam3!!.requestAppInfo(appId, true) + } else { + val contentName = getAppName(appId) + throw ContentDownloaderException("App $appId ($contentName) is not available from this account.") + } + } + + val hasSpecificDepots = depotManifestIds.isNotEmpty() + val depotIdsFound = mutableListOf() + val depotIdsExpected = depotManifestIds.map { x -> x.first }.toMutableList() + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) + + if (isUgc) { + val workshopDepot = depots!!["workshopdepot"].asInteger() + if (workshopDepot != 0 && !depotIdsExpected.contains(workshopDepot)) { + depotIdsExpected.add(workshopDepot) + depotManifestIds = depotManifestIds.map { pair -> workshopDepot to pair.second }.toMutableList() + } + + depotIdsFound.addAll(depotIdsExpected) + } else { + logger?.debug("Using app branch: $branch") + + depots?.children?.forEach { depotSection -> + @Suppress("VariableInitializerIsRedundant") + var id = INVALID_DEPOT_ID + + if (depotSection.children.isEmpty()) { + return@forEach + } + + id = depotSection.name?.toIntOrNull() ?: return@forEach + + if (hasSpecificDepots && !depotIdsExpected.contains(id)) { + return@forEach + } + + if (!hasSpecificDepots) { + val depotConfig = depotSection["config"] + if (depotConfig != KeyValue.INVALID) { + if (!config.downloadAllPlatforms && + depotConfig["oslist"] != KeyValue.INVALID && + !depotConfig["oslist"].value.isNullOrBlank() + ) { + val osList = depotConfig["oslist"].value!!.split(",") + if (osList.indexOf(os ?: Util.getSteamOS(config.androidEmulation)) == -1) { + return@forEach + } + } + + if (!config.downloadAllArchs && + depotConfig["osarch"] != KeyValue.INVALID && + !depotConfig["osarch"].value.isNullOrBlank() + ) { + val depotArch = depotConfig["osarch"].value + if (depotArch != (arch ?: Util.getSteamArch())) { + return@forEach + } + } + + if (!config.downloadAllLanguages && + depotConfig["language"] != KeyValue.INVALID && + !depotConfig["language"].value.isNullOrBlank() + ) { + val depotLang = depotConfig["language"].value + if (depotLang != (language ?: "english")) { + return@forEach + } + } + + if (!lv && + depotConfig["lowviolence"] != KeyValue.INVALID && + depotConfig["lowviolence"].asBoolean() + ) { + return@forEach + } + } + } + + depotIdsFound.add(id) + + if (!hasSpecificDepots) { + depotManifestIds.add(id to INVALID_MANIFEST_ID) + } + } + + if (depotManifestIds.isEmpty() && !hasSpecificDepots) { + throw ContentDownloaderException("Couldn't find any depots to download for app $appId") + } + + if (depotIdsFound.size < depotIdsExpected.size) { + val remainingDepotIds = depotIdsExpected.subtract(depotIdsFound.toSet()) + throw ContentDownloaderException("Depot ${remainingDepotIds.joinToString(", ")} not listed for app $appId") + } + } + + val infos = mutableListOf() + + depotManifestIds.forEach { (depotId, manifestId) -> + val info = getDepotInfo(depotId, appId, manifestId, branch) + if (info != null) { + infos.add(info) + } + } + + downloadSteam3(infos) + } + + @Throws(IllegalStateException::class) + private suspend fun getDepotInfo( + depotId: Int, + appId: Int, + manifestId: Long, + branch: String, + ): DepotDownloadInfo? { + var manifestId = manifestId + var branch = branch + + if (appId != INVALID_APP_ID) { + steam3!!.requestAppInfo(appId) + } + + if (!accountHasAccess(appId, depotId)) { + logger?.error("Depot $depotId is not available from this account.") + return null + } + + if (manifestId == INVALID_MANIFEST_ID) { + manifestId = getSteam3DepotManifest(depotId, appId, branch) + + if (manifestId == INVALID_MANIFEST_ID && !branch.equals(DEFAULT_BRANCH, true)) { + logger?.error("Warning: Depot $depotId does not have branch named \"$branch\". Trying $DEFAULT_BRANCH branch.") + branch = DEFAULT_BRANCH + manifestId = getSteam3DepotManifest(depotId, appId, branch) + } + + if (manifestId == INVALID_MANIFEST_ID) { + logger?.error("Depot $depotId missing public subsection or manifest section.") + return null + } + } + + steam3!!.requestDepotKey(depotId, appId) + + val depotKey = steam3!!.depotKeys[depotId] + if (depotKey == null) { + logger?.error("No valid depot key for $depotId, unable to download.") + return null + } + + val uVersion = getSteam3AppBuildNumber(appId, branch) + + val (success, installDir) = createDirectories(depotId, uVersion, appId) + if (!success) { + logger?.error("Error: Unable to create install directories!") + return null + } + + // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id, unless the app is freetodownload + var containingAppId = appId + val proxyAppId = getSteam3DepotProxyAppId(depotId, appId) + if (proxyAppId != INVALID_APP_ID) { + val common = getSteam3AppSection(appId, EAppInfoSection.Common) + if (common == null || !common["FreeToDownload"].asBoolean()) { + containingAppId = proxyAppId + } + } + + return DepotDownloadInfo( + depotId = depotId, + appId = containingAppId, + manifestId = manifestId, + branch = branch, + installDir = installDir!!, + depotKey = depotKey + ) + } + + private suspend fun getSteam3DepotManifest( + depotId: Int, + appId: Int, + branch: String, + ): Long { + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) + var depotChild = depots?.get(depotId.toString()) ?: KeyValue.INVALID + + if (depotChild == KeyValue.INVALID) { + return INVALID_MANIFEST_ID + } + + // Shared depots can either provide manifests, or leave you relying on their parent app. + // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). + // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. + if (depotChild["manifests"] == KeyValue.INVALID && depotChild["depotfromapp"] != KeyValue.INVALID) { + val otherAppId = depotChild["depotfromapp"].asInteger() + if (otherAppId == appId) { + // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. + logger?.error("App $appId, Depot $depotId has depotfromapp of $otherAppId!") + return INVALID_MANIFEST_ID + } + + steam3!!.requestAppInfo(otherAppId) + + return getSteam3DepotManifest(depotId, otherAppId, branch) + } + + var manifests = depotChild["manifests"] + + if (manifests.children.isEmpty()) { + return INVALID_MANIFEST_ID + } + + var node = manifests[branch]["gid"] + + // Non passworded branch, found the manifest + if (node.value != null) { + return node.value!!.toLong() + } + + // If we requested public branch, and it had no manifest, nothing to do + if (branch.equals(DEFAULT_BRANCH, true)) { + return INVALID_MANIFEST_ID + } + + // Either the branch just doesn't exist, or it has a password + if (config.betaPassword.isNullOrBlank()) { + logger?.error("Branch $branch for depot $depotId was not found, either it does not exist or it has a password.") + return INVALID_MANIFEST_ID + } + + if (!steam3!!.appBetaPasswords.containsKey(branch)) { + // Submit the password to Steam now to get encryption keys + steam3!!.checkAppBetaPassword(appId, config.betaPassword!!) + + if (!steam3!!.appBetaPasswords.containsKey(branch)) { + logger?.error("Error: Password was invalid for branch $branch (or the branch does not exist)") + return INVALID_MANIFEST_ID + } + } + + // Got the password, request private depot section + // TODO: (SK) We're probably repeating this request for every depot? + val privateDepotSection = steam3!!.getPrivateBetaDepotSection(appId, branch) + + // Now repeat the same code to get the manifest gid from depot section + depotChild = privateDepotSection[depotId.toString()] + + if (depotChild == KeyValue.INVALID) { + return INVALID_MANIFEST_ID + } + + manifests = depotChild["manifests"] + + if (manifests.children.isEmpty()) { + return INVALID_MANIFEST_ID + } + + node = manifests[branch]["gid"] + + if (node.value == null) { + return INVALID_MANIFEST_ID + } + + return node.value!!.toLong() + } + + private fun getSteam3AppBuildNumber(appId: Int, branch: String): Int { + if (appId == INVALID_APP_ID) { + return 0 + } + + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) ?: KeyValue.INVALID + val branches = depots["branches"] + val node = branches[branch] + + if (node == KeyValue.INVALID) { + return 0 + } + + val buildId = node["buildid"] + + if (buildId == KeyValue.INVALID) { + return 0 + } + + return buildId.value!!.toInt() + } + + private fun getSteam3DepotProxyAppId(depotId: Int, appId: Int): Int { + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) ?: KeyValue.INVALID + val depotChild = depots[depotId.toString()] + + if (depotChild == KeyValue.INVALID) { + return INVALID_APP_ID + } + + if (depotChild["depotfromapp"] == KeyValue.INVALID) { + return INVALID_APP_ID + } + + return depotChild["depotfromapp"].asInteger() + } + + @Throws(IllegalStateException::class) + private fun createDirectories(depotId: Int, depotVersion: Int, appId: Int = 0): DirectoryResult { + var installDir: Path? + try { + if (config.installPath?.toString().isNullOrBlank()) { + // Android Check + if (SystemUtils.IS_OS_ANDROID) { + // This should propagate up to the caller. + throw IllegalStateException("Android must have an installation directory set.") + } + + filesystem.createDirectories(DEFAULT_DOWNLOAD_DIR.toPath()) + + if (config.installToGameNameDirectory) { + val gameName = getAppName(appId) + + if (gameName.isBlank()) { + throw IOException("Game name is blank, cannot create directory") + } + + installDir = DEFAULT_DOWNLOAD_DIR.toPath() / gameName + + filesystem.createDirectories(installDir) + } else { + val depotPath = DEFAULT_DOWNLOAD_DIR.toPath() / depotId.toString() + filesystem.createDirectories(depotPath) + + installDir = depotPath / depotVersion.toString() + filesystem.createDirectories(installDir) + } + + filesystem.createDirectories(installDir / CONFIG_DIR) + filesystem.createDirectories(installDir / STAGING_DIR) + } else { + filesystem.createDirectories(config.installPath!!) + + if (config.installToGameNameDirectory) { + val gameName = getAppName(appId) + + if (gameName.isBlank()) { + throw IOException("Game name is blank, cannot create directory") + } + + installDir = config.installPath!! / gameName + + filesystem.createDirectories(installDir) + } else { + installDir = config.installPath!! + } + + filesystem.createDirectories(installDir / CONFIG_DIR) + filesystem.createDirectories(installDir / STAGING_DIR) + } + } catch (e: IOException) { + logger?.error(e) + return DirectoryResult(false, null) + } + + return DirectoryResult(true, installDir) + } + + private fun getAppName(appId: Int): String { + val info = getSteam3AppSection(appId, EAppInfoSection.Common) ?: KeyValue.INVALID + return info["name"].asString() ?: "" + } + + private fun getSteam3AppSection(appId: Int, section: EAppInfoSection): KeyValue? { + if (steam3 == null) { + return null + } + + if (steam3!!.appInfo.isEmpty()) { + return null + } + + val app = steam3!!.appInfo[appId] ?: return null + + val appInfo = app.keyValues + val sectionKey = when (section) { + EAppInfoSection.Common -> "common" + EAppInfoSection.Extended -> "extended" + EAppInfoSection.Config -> "config" + EAppInfoSection.Depots -> "depots" + else -> throw ContentDownloaderException("${section.name} not implemented") + } + + val sectionKV = appInfo.children.firstOrNull { c -> c.name == sectionKey } + return sectionKV + } + + private suspend fun accountHasAccess(appId: Int, depotId: Int): Boolean { + val steamUser = requireNotNull(steam3!!.steamUser) + val steamID = requireNotNull(steamUser.steamID) + + if (licenses.isEmpty() && steamID.accountType != EAccountType.AnonUser) { + return false + } + + val licenseQuery = arrayListOf() + if (steamID.accountType == EAccountType.AnonUser) { + licenseQuery.add(17906) + } else { + licenseQuery.addAll(licenses.map { it.packageID }.distinct()) + } + + steam3!!.requestPackageInfo(licenseQuery) + + licenseQuery.forEach { license -> + steam3!!.packageInfo[license]?.let { pkg -> + val appIds = pkg.keyValues["appids"].children.map { it.asInteger() } + val depotIds = pkg.keyValues["depotids"].children.map { it.asInteger() } + if (depotId in appIds) { + return true + } + if (depotId in depotIds) { + return true + } + } + } + + // Check if this app is free to download without a license + val info = getSteam3AppSection(appId, EAppInfoSection.Common) + + return info != null && info["FreeToDownload"].asBoolean() + } + + private suspend fun downloadSteam3(depots: List): Unit = coroutineScope { + cdnClientPool?.updateServerList() + + val downloadCounter = GlobalDownloadCounter() + val depotsToDownload = ArrayList(depots.size) + val allFileNamesAllDepots = hashSetOf() + + var completedDepots = 0 + + // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup + depots.forEach { depot -> + val depotFileData = processDepotManifestAndFiles(depot, downloadCounter) + + if (depotFileData != null) { + depotsToDownload.add(depotFileData) + allFileNamesAllDepots.union(depotFileData.allFileNames) + } + + ensureActive() + } + + // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path + // This is in last-depot-wins order, from Steam or the list of depots supplied by the user + if (config.installPath != null && depotsToDownload.isNotEmpty()) { + val claimedFileNames = mutableSetOf() + for (i in depotsToDownload.indices.reversed()) { + // For each depot, remove all files from the list that have been claimed by a later depot + depotsToDownload[i].filteredFiles.removeAll { file -> file.fileName in claimedFileNames } + claimedFileNames.addAll(depotsToDownload[i].allFileNames) + } + } + + depotsToDownload.forEach { depotFileData -> + downloadSteam3DepotFiles(downloadCounter, depotFileData, allFileNamesAllDepots) + + completedDepots++ + + val snapshot = synchronized(downloadCounter) { + OverallProgress( + currentItem = completedDepots, + totalItems = depotsToDownload.size, + totalBytesDownloaded = downloadCounter.totalBytesUncompressed, + totalBytesExpected = downloadCounter.completeDownloadSize, + status = DownloadStatus.DOWNLOADING + ) + } + + notifyListeners { listener -> + listener.onOverallProgress(progress = snapshot) + } + } + + logger?.debug( + "Total downloaded: ${downloadCounter.totalBytesCompressed} bytes " + + "(${downloadCounter.totalBytesUncompressed} bytes uncompressed) from ${depots.size} depots" + ) + } + + private suspend fun processDepotManifestAndFiles( + depot: DepotDownloadInfo, + downloadCounter: GlobalDownloadCounter, + ): DepotFilesData? = withContext(Dispatchers.IO) { + val depotCounter = DepotDownloadCounter() + + logger?.debug("Processing depot ${depot.depotId}") + + var oldManifest: DepotManifest? = null + + @Suppress("VariableInitializerIsRedundant") + var newManifest: DepotManifest? = null + + val configDir = depot.installDir / CONFIG_DIR + + @Suppress("VariableInitializerIsRedundant") + var lastManifestId = INVALID_MANIFEST_ID + lastManifestId = DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] ?: INVALID_MANIFEST_ID + + // In case we have an early exit, this will force equiv of verifyall next run. + DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = INVALID_MANIFEST_ID + DepotConfigStore.save() + + if (lastManifestId != INVALID_MANIFEST_ID) { + // We only have to show this warning if the old manifest ID was different + val badHashWarning = lastManifestId != depot.manifestId + oldManifest = Util.loadManifestFromFile(configDir, depot.depotId, lastManifestId, badHashWarning) + } + + if (lastManifestId == depot.manifestId && oldManifest != null) { + newManifest = oldManifest + logger?.debug("Already have manifest ${depot.manifestId} for depot ${depot.depotId}.") + } else { + newManifest = Util.loadManifestFromFile(configDir, depot.depotId, depot.manifestId, true) + + if (newManifest != null) { + logger?.debug("Already have manifest ${depot.manifestId} for depot ${depot.depotId}.") + } else { + logger?.debug("Downloading depot ${depot.depotId} manifest") + notifyListeners { it.onStatusUpdate("Downloading manifest for depot ${depot.depotId}") } + + var manifestRequestCode: ULong = 0U + var manifestRequestCodeExpiration = Instant.MIN + + do { + ensureActive() + + var connection: Server? = null + + try { + connection = cdnClientPool!!.getConnection() + + var cdnToken: String? = null + + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") + } + } + + val now = Instant.now() + + // In order to download this manifest, we need the current manifest request code + // The manifest request code is only valid for a specific period in time + if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { + manifestRequestCode = steam3!!.getDepotManifestRequestCode( + depotId = depot.depotId, + appId = depot.appId, + manifestId = depot.manifestId, + branch = depot.branch, + ) + + // This code will hopefully be valid for one period following the issuing period + manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) + + // If we could not get the manifest code, this is a fatal error + if (manifestRequestCode == 0UL) { + cancel("manifestRequestCode is 0UL") + } + } + + logger?.debug("Downloading manifest ${depot.manifestId} from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") + + newManifest = cdnClientPool!!.cdnClient!!.downloadManifest( + depotId = depot.depotId, + manifestId = depot.manifestId, + manifestRequestCode = manifestRequestCode, + server = connection, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) + + cdnClientPool!!.returnConnection(connection) + } catch (e: CancellationException) { + // logger?.error("Connection timeout downloading depot manifest ${depot.depotId} ${depot.manifestId}. Retrying.") + logger?.error(e) + break + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet + if (e.statusCode == 403 && !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host)) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + + cdnClientPool!!.returnConnection(connection) + + continue + } + + cdnClientPool!!.returnBrokenConnection(connection) + + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${depot.depotId} for depot manifest ${depot.manifestId} ${e.statusCode}. Aborting.") + break + } + + // NotFound + if (e.statusCode == 404) { + logger?.error("Encountered 404 for depot manifest ${depot.depotId} ${depot.manifestId}. Aborting.") + break + } + + logger?.error("Encountered error downloading depot manifest ${depot.depotId} ${depot.manifestId}: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered error downloading manifest for depot ${depot.depotId} ${depot.manifestId}: ${e.message}") + } + } while (newManifest == null) + + if (newManifest == null) { + logger?.error("\nUnable to download manifest ${depot.manifestId} for depot ${depot.depotId}") + cancel() + } + + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() + + Util.saveManifestToFile(configDir, newManifest!!) + } + } + + logger?.debug("Manifest ${depot.manifestId} (${newManifest.creationTime})") + + if (config.downloadManifestOnly) { + Util.dumpManifestToTextFile(depot, newManifest) + return@withContext null + } + + val stagingDir = depot.installDir / STAGING_DIR + + val filesAfterExclusions = coroutineScope { + newManifest.files.filter { file -> + async { testIsFileIncluded(file.fileName) }.await() + } + } + val allFileNames = HashSet(filesAfterExclusions.size) + + // Pre-process + filesAfterExclusions.forEach { file -> + allFileNames.add(file.fileName) + + val fileFinalPath = depot.installDir / file.fileName + val fileStagingPath = stagingDir / file.fileName + + if (file.flags.contains(EDepotFileFlag.Directory)) { + filesystem.createDirectories(fileFinalPath) + filesystem.createDirectories(fileStagingPath) + } else { + // Some manifests don't explicitly include all necessary directories + filesystem.createDirectories(fileFinalPath.parent!!) + filesystem.createDirectories(fileStagingPath.parent!!) + + downloadCounter.completeDownloadSize += file.totalSize + depotCounter.completeDownloadSize += file.totalSize + } + } + + return@withContext DepotFilesData( + depotDownloadInfo = depot, + depotCounter = depotCounter, + stagingDir = stagingDir, + manifest = newManifest, + previousManifest = oldManifest, + filteredFiles = filesAfterExclusions.toMutableList(), + allFileNames = allFileNames, + ) + } + + @OptIn(DelicateCoroutinesApi::class) + private suspend fun downloadSteam3DepotFiles( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + allFileNamesAllDepots: HashSet, + ) = withContext(Dispatchers.IO) { + val depot = depotFilesData.depotDownloadInfo + val depotCounter = depotFilesData.depotCounter + + logger?.debug("Downloading depot ${depot.depotId}") + + val files = depotFilesData.filteredFiles.filter { !it.flags.contains(EDepotFileFlag.Directory) } + val networkChunkQueue = Channel(Channel.UNLIMITED) + + try { + val filesCompleted = AtomicInteger(0) + val lastReportedProgress = AtomicInteger(0) + coroutineScope { + // First parallel loop - process files and enqueue chunks + files.map { file -> + async { + yield() // Does this matter if its before? + downloadSteam3DepotFile( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = file, + networkChunkQueue = networkChunkQueue + ) + + val completed = filesCompleted.incrementAndGet() + if (completed % 10 == 0 || completed == files.size) { + val snapshot = synchronized(depotCounter) { + DepotProgress( + depotId = depot.depotId, + filesCompleted = completed, + totalFiles = files.size, + bytesDownloaded = depotCounter.sizeDownloaded, + totalBytes = depotCounter.completeDownloadSize, + status = DownloadStatus.PREPARING // Changed from DOWNLOADING + ) + } + + val lastReported = lastReportedProgress.get() + if (completed > lastReported && + lastReportedProgress.compareAndSet( + lastReported, + completed + ) + ) { + notifyListeners { listener -> + listener.onDepotProgress(depot.depotId, snapshot) + } + } + } + } + }.awaitAll() + + // Close the channel to signal no more items will be added + networkChunkQueue.close() + + // After all files allocated, send one update showing preparation complete + val progressReporter = launch { + while (true) { + delay(1000) + val snapshot = synchronized(depotCounter) { + DepotProgress( + depotId = depot.depotId, + filesCompleted = files.size, + totalFiles = files.size, + bytesDownloaded = depotCounter.sizeDownloaded, + totalBytes = depotCounter.completeDownloadSize, + status = DownloadStatus.DOWNLOADING + ) + } + notifyListeners { listener -> + listener.onDepotProgress(depot.depotId, snapshot) + } + } + } + + // Second parallel loop - process chunks from queue + try { + List(maxDownloads) { + async { + for (item in networkChunkQueue) { + downloadSteam3DepotFileChunk( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = item.fileData, + fileStreamData = item.fileStreamData, + chunk = item.chunk + ) + } + } + }.awaitAll() + } finally { + progressReporter.cancel() + } + } + } finally { + if (!networkChunkQueue.isClosedForSend) { + networkChunkQueue.close() + } + } + + // Check for deleted files if updating the depot. + if (depotFilesData.previousManifest != null) { + val previousFilteredFiles = depotFilesData.previousManifest.files + .filter { testIsFileIncluded(it.fileName) } + .map { it.fileName } + .toHashSet() + + // Check if we are writing to a single output directory. If not, each depot folder is managed independently + if (config.installPath == null) { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names + previousFilteredFiles.removeAll(depotFilesData.allFileNames) + } else { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded + previousFilteredFiles.removeAll(allFileNamesAllDepots) + } + + previousFilteredFiles.forEach { existingFileName -> + val fileFinalPath = depot.installDir / existingFileName + + if (!filesystem.exists(fileFinalPath)) { + return@forEach + } + + filesystem.delete(fileFinalPath) + logger?.debug("Deleted $fileFinalPath") + } + } + + DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = depot.manifestId + DepotConfigStore.save() + logger?.debug("Depot ${depot.depotId} - Downloaded ${depotCounter.depotBytesCompressed} bytes (${depotCounter.depotBytesUncompressed} bytes uncompressed)") + } + + private suspend fun downloadSteam3DepotFile( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + file: FileData, + networkChunkQueue: Channel, + ) = withContext(Dispatchers.IO) { + ensureActive() + + val depot = depotFilesData.depotDownloadInfo + val stagingDir = depotFilesData.stagingDir + val depotDownloadCounter = depotFilesData.depotCounter + val oldProtoManifest = depotFilesData.previousManifest + + var oldManifestFile: FileData? = null + if (oldProtoManifest != null) { + oldManifestFile = oldProtoManifest.files.singleOrNull { it.fileName == file.fileName } + } + + val fileFinalPath = depot.installDir / file.fileName + val fileStagingPath = stagingDir / file.fileName + + // This may still exist if the previous run exited before cleanup + if (filesystem.exists(fileStagingPath)) { + filesystem.delete(fileStagingPath) + } + + var neededChunks: MutableList? = null + val fileDidExist = filesystem.exists(fileFinalPath) + if (!fileDidExist) { + logger?.debug("Pre-allocating: $fileFinalPath") + notifyListeners { it.onStatusUpdate("Allocating file: ${file.fileName}") } + + // create new file. need all chunks + try { + filesystem.openReadWrite(fileFinalPath).use { handle -> + handle.resize(file.totalSize) + } + } catch (e: IOException) { + throw ContentDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") + } + + neededChunks = ArrayList(file.chunks) + } else { + // open existing + if (oldManifestFile != null) { + neededChunks = arrayListOf() + + val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) + if (config.verifyAll || !hashMatches) { + // we have a version of this file, but it doesn't fully match what we want + if (config.verifyAll) { + logger?.debug("Validating: $fileFinalPath") + } + + val matchingChunks = arrayListOf() + + file.chunks.forEach { chunk -> + val oldChunk = oldManifestFile.chunks.firstOrNull { c -> + c.chunkID.contentEquals(chunk.chunkID) + } + if (oldChunk != null) { + val chunkMatch = ChunkMatch(oldChunk, chunk) + matchingChunks.add(chunkMatch) + } else { + neededChunks.add(chunk) + } + } + + val orderedChunks = matchingChunks.sortedBy { x -> x.oldChunk.offset } + + val copyChunks = arrayListOf() + + filesystem.openReadOnly(fileFinalPath).use { handle -> + orderedChunks.forEach { match -> + // Read the chunk data into a byte array + val length = match.oldChunk.uncompressedLength + val buffer = ByteArray(length) + handle.read(match.oldChunk.offset, buffer, 0, length) + + // Calculate Adler32 checksum + val adler = Adler32.calculate(buffer) + + // Convert checksum to byte array for comparison + val checksumBytes = Buffer().apply { + writeIntLe(match.oldChunk.checksum) + }.readByteArray() + val calculatedChecksumBytes = Buffer().apply { + writeIntLe(adler) + }.readByteArray() + + if (!calculatedChecksumBytes.contentEquals(checksumBytes)) { + neededChunks.add(match.newChunk) + } else { + copyChunks.add(match) + } + } + } + + if (!hashMatches || neededChunks.isNotEmpty()) { + filesystem.atomicMove(fileFinalPath, fileStagingPath) + + try { + filesystem.openReadOnly(fileStagingPath).use { oldHandle -> + filesystem.openReadWrite(fileFinalPath).use { newHandle -> + try { + newHandle.resize(file.totalSize) + } catch (ex: IOException) { + throw ContentDownloaderException( + "Failed to resize file to expected size $fileFinalPath: ${ex.message}" + ) + } + + for (match in copyChunks) { + val tmp = ByteArray(match.oldChunk.uncompressedLength) + oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) + newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) + } + } + } + } catch (e: Exception) { + logger?.error(e) + } + + filesystem.delete(fileStagingPath) + } + } + } else { + // No old manifest or file not in old manifest. We must validate. + filesystem.openReadWrite(fileFinalPath).use { handle -> + val fileSize = filesystem.metadata(fileFinalPath).size ?: 0L + if (fileSize.toULong() != file.totalSize.toULong()) { + try { + handle.resize(file.totalSize) + } catch (ex: IOException) { + throw ContentDownloaderException( + "Failed to allocate file $fileFinalPath: ${ex.message}" + ) + } + } + + logger?.debug("Validating $fileFinalPath") + notifyListeners { it.onStatusUpdate("Validating: ${file.fileName}") } + + neededChunks = Util.validateSteam3FileChecksums( + handle = handle, + chunkData = file.chunks.sortedBy { it.offset } + ).toMutableList() + } + } + + if (neededChunks!!.isEmpty()) { + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += file.totalSize + + val percentage = + (depotDownloadCounter.sizeDownloaded / depotDownloadCounter.completeDownloadSize.toFloat()) * 100.0f + logger?.debug("%.2f%% %s".format(percentage, fileFinalPath)) + } + + synchronized(downloadCounter) { + downloadCounter.completeDownloadSize -= file.totalSize + } + + return@withContext + } + + val sizeOnDisk = file.totalSize - neededChunks!!.sumOf { it.uncompressedLength } + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += sizeOnDisk + } + + synchronized(downloadCounter) { + downloadCounter.completeDownloadSize -= sizeOnDisk + } + } + + val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) + if (fileIsExecutable && + (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) + ) { + fileFinalPath.toFile().setExecutable(true) + } else if (!fileIsExecutable && + oldManifestFile != null && + oldManifestFile.flags.contains(EDepotFileFlag.Executable) + ) { + fileFinalPath.toFile().setExecutable(false) + } + + val fileStreamData = FileStreamData( + fileHandle = null, + fileLock = Mutex(), + chunksToDownload = AtomicInteger(neededChunks!!.size) + ) + + neededChunks!!.forEach { chunk -> + networkChunkQueue.send( + NetworkChunkItem( + fileStreamData = fileStreamData, + fileData = file, + chunk = chunk, + totalChunksForFile = neededChunks!!.size + ) + ) + } + } + + private suspend fun downloadSteam3DepotFileChunk( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + file: FileData, + fileStreamData: FileStreamData, + chunk: ChunkData, + ): Unit = withContext(Dispatchers.IO) { + ensureActive() + + val depot = depotFilesData.depotDownloadInfo + val depotDownloadCounter = depotFilesData.depotCounter + + val chunkID = Strings.toHex(chunk.chunkID) + + var written = 0 + val chunkBuffer = ByteArray(chunk.uncompressedLength) + + try { + do { + ensureActive() + + var connection: Server? = null + + try { + connection = cdnClientPool!!.getConnection() + + var cdnToken: String? = null + + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") + } + } + + logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") + + written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( + depotId = depot.depotId, + chunk = chunk, + server = connection, + destination = chunkBuffer, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) + + cdnClientPool!!.returnConnection(connection) + + break + } catch (e: CancellationException) { + logger?.error(e) + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, + // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above + if (e.statusCode == 403 && + ( + !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host) || + steam3!!.cdnAuthTokens[depot.depotId to connection.host]?.isCompleted == false + ) + ) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + + cdnClientPool!!.returnConnection(connection) + + continue + } + + cdnClientPool!!.returnBrokenConnection(connection) + + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") + break + } + + logger?.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered unexpected error downloading chunk $chunkID", e) + } + } while (written == 0) + + if (written == 0) { + logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") + cancel() + } + + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() + + try { + fileStreamData.fileLock.lock() + + if (fileStreamData.fileHandle == null) { + val fileFinalPath = depot.installDir / file.fileName + fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) + } + + fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) + } finally { + fileStreamData.fileLock.unlock() + } + } finally { + } + + val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() + if (remainingChunks == 0) { + fileStreamData.fileHandle?.close() + } + + var sizeDownloaded = 0L + synchronized(depotDownloadCounter) { + sizeDownloaded = depotDownloadCounter.sizeDownloaded + written.toLong() + depotDownloadCounter.sizeDownloaded = sizeDownloaded + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + + val now = System.currentTimeMillis() + val fileKey = "${depot.depotId}:${file.fileName}" + val lastUpdate = lastFileProgressUpdate[fileKey] ?: 0L + + if (now - lastUpdate >= progressUpdateInterval || remainingChunks == 0) { + lastFileProgressUpdate[fileKey] = now + + val totalChunks = file.chunks.size + val completedChunks = totalChunks - remainingChunks + + // Approximate bytes based on completion ratio + val approximateBytesDownloaded = (file.totalSize * completedChunks) / totalChunks + + notifyListeners { listener -> + listener.onFileProgress( + depotId = depot.depotId, + fileName = file.fileName, + progress = FileProgress( + fileName = file.fileName, + bytesDownloaded = approximateBytesDownloaded, + totalBytes = file.totalSize, + chunksCompleted = completedChunks, + totalChunks = totalChunks, + status = if (remainingChunks == 0) DownloadStatus.COMPLETED else DownloadStatus.DOWNLOADING + ) + ) + } + } + + if (remainingChunks == 0) { + val fileFinalPath = depot.installDir / file.fileName + val percentage = (sizeDownloaded / depotDownloadCounter.completeDownloadSize.toFloat()) * 100.0f + logger?.debug("%.2f%% %s".format(percentage, fileFinalPath)) + } + } + + private fun testIsFileIncluded(filename: String): Boolean { + if (!config.usingFileList) { + return true + } + + val normalizedFilename = filename.replace('\\', '/') + + if (normalizedFilename in config.filesToDownload) { + return true + } + + for (regex in config.filesToDownloadRegex) { + if (regex.matches(normalizedFilename)) { + return true + } + } + + return false + } + + // endregion + + // region [REGION] Listener Operations + + fun addListener(listener: IDownloadListener) { + listeners.add(listener) + } + + fun removeListener(listener: IDownloadListener) { + listeners.remove(listener) + } + + private fun notifyListeners(action: (IDownloadListener) -> Unit) { + listeners.forEach { listener -> action(listener) } + } + + // endregion + + // region [REGION] Array Operations + + fun getItems(): List = items.toList() + + fun size(): Int = items.size + + fun isEmpty(): Boolean = items.isEmpty() + + fun get(index: Int): DownloadItem? = items.getOrNull(index) + + fun contains(item: DownloadItem): Boolean = items.contains(item) + + fun indexOf(item: DownloadItem): Int = items.indexOf(item) + + fun addAll(items: List) { + items.forEach(::add) + } + + fun add(item: DownloadItem) { + val index = items.size + items.add(item) + + if (isStarted.get()) { + remainingItems.incrementAndGet() + scope.launch { processingChannel.send(item) } + } + + notifyListeners { it.onItemAdded(item, index) } + } + + fun addFirst(item: DownloadItem) { + if (isStarted.get()) { + logger?.debug("Cannot add item when started.") + return + } + + items.add(0, item) + + notifyListeners { it.onItemAdded(item, 0) } + } + + fun addAt(index: Int, item: DownloadItem): Boolean { + if (isStarted.get()) { + logger?.debug("Cannot addAt item when started.") + return false + } + + return try { + items.add(index, item) + notifyListeners { it.onItemAdded(item, index) } + true + } catch (e: IndexOutOfBoundsException) { + false + } + } + + fun removeFirst(): DownloadItem? { + if (isStarted.get()) { + logger?.debug("Cannot removeFirst item when started.") + return null + } + + return if (items.isNotEmpty()) { + val item = items.removeAt(0) + notifyListeners { it.onItemRemoved(item, 0) } + item + } else { + null + } + } + + fun removeLast(): DownloadItem? { + if (isStarted.get()) { + logger?.debug("Cannot removeLast item when started.") + return null + } + + return if (items.isNotEmpty()) { + val lastIndex = items.size - 1 + val item = items.removeAt(lastIndex) + notifyListeners { it.onItemRemoved(item, lastIndex) } + item + } else { + null + } + } + + fun remove(item: DownloadItem): Boolean { + if (isStarted.get()) { + logger?.debug("Cannot remove item when started.") + return false + } + + val index = items.indexOf(item) + return if (index >= 0) { + items.removeAt(index) + notifyListeners { it.onItemRemoved(item, index) } + true + } else { + false + } + } + + fun removeAt(index: Int): DownloadItem? { + if (isStarted.get()) { + logger?.debug("Cannot removeAt item when started.") + return null + } + + return try { + val item = items.removeAt(index) + notifyListeners { it.onItemRemoved(item, index) } + item + } catch (e: IndexOutOfBoundsException) { + null + } + } + + fun moveItem(fromIndex: Int, toIndex: Int): Boolean { + if (isStarted.get()) { + logger?.debug("Cannot moveItem item when started.") + return false + } + + return try { + val item = items.removeAt(fromIndex) + items.add(toIndex, item) + true + } catch (e: IndexOutOfBoundsException) { + false + } + } + + fun clear() { + if (isStarted.get()) { + logger?.debug("Cannot clear item when started.") + return + } + + val oldItems = items.toList() + items.clear() + + notifyListeners { it.onQueueCleared(oldItems) } + } + + // endregion + + /** + * Some android emulators prefer using "Windows", so this will set downloading to prefer the Windows version. + */ + fun setAndroidEmulation(value: Boolean) { + if (isStarted.get()) { + logger?.error("Can't set android emulation value once started.") + return + } + + config = config.copy(androidEmulation = value) + + notifyListeners { it.onAndroidEmulation(config.androidEmulation) } + } + + @Throws(IllegalStateException::class) + fun start(): CompletableFuture = scope.future { + if (isStarted.getAndSet(true)) { + logger?.debug("Downloading already started.") + return@future false + } + + val initialItems = items.toList() + if (initialItems.isEmpty()) { + logger?.debug("No items to download") + return@future false + } + + // Send initial items + remainingItems.set(initialItems.size) + initialItems.forEach { processingChannel.send(it) } + + repeat(remainingItems.get()) { + // Process exactly this many + ensureActive() + + // Obtain the next item in queue. + val item = processingChannel.receive() + + try { + runBlocking { + if (useLanCache) { + ClientLancache.detectLancacheServer() + } + + if (ClientLancache.useLanCacheServer) { + logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") + + // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely + // be served much faster than over the internet. Steam internally has this behavior as well. + if (maxDownloads == 8) { + maxDownloads = 25 + } + } + + // Set some configuration values, first. + config = config.copy( + downloadManifestOnly = item.downloadManifestOnly, + installPath = item.installDirectory?.toPath(), + installToGameNameDirectory = item.installToGameNameDirectory, + ) + + // Sequential looping. + when (item) { + is PubFileItem -> { + if (item.pubfile == INVALID_MANIFEST_ID) { + logger?.debug("Invalid Pub File ID for ${item.appId}") + return@runBlocking + } + + logger?.debug("Downloading PUB File for ${item.appId}") + + notifyListeners { it.onDownloadStarted(item) } + downloadPubFile(item.appId, item.pubfile) + } + + is UgcItem -> { + if (item.ugcId == INVALID_MANIFEST_ID) { + logger?.debug("Invalid UGC ID for ${item.appId}") + return@runBlocking + } + + logger?.debug("Downloading UGC File for ${item.appId}") + + notifyListeners { it.onDownloadStarted(item) } + downloadUGC(item.appId, item.ugcId) + } + + is AppItem -> { + val branch = item.branch ?: DEFAULT_BRANCH + config = config.copy(betaPassword = item.branchPassword) + + if (!config.betaPassword.isNullOrBlank() && branch.isBlank()) { + logger?.error("Error: Cannot specify 'branchpassword' when 'branch' is not specified.") + return@runBlocking + } + + config = config.copy(downloadAllPlatforms = item.downloadAllPlatforms) + + val os = item.os + + if (config.downloadAllPlatforms && !os.isNullOrBlank()) { + logger?.error("Error: Cannot specify `os` when `all-platforms` is specified.") + return@runBlocking + } + + config = config.copy(downloadAllArchs = item.downloadAllArchs) + + val arch = item.osArch + + if (config.downloadAllArchs && !arch.isNullOrBlank()) { + logger?.error("Error: Cannot specify `osarch` when `all-archs` is specified.") + return@runBlocking + } + + config = config.copy(downloadAllLanguages = item.downloadAllLanguages) + + val language = item.language + + if (config.downloadAllLanguages && !language.isNullOrBlank()) { + logger?.error("Error: Cannot specify `language` when `all-languages` is specified.") + return@runBlocking + } + + val lv = item.lowViolence + + val depotManifestIds = mutableListOf>() + val isUGC = false + + val depotIdList = item.depot + val manifestIdList = item.manifest + + if (manifestIdList.isNotEmpty()) { + if (depotIdList.size != manifestIdList.size) { + logger?.error("Error: `manifest` requires one id for every `depot` specified") + return@runBlocking + } + val zippedDepotManifest = depotIdList.zip(manifestIdList) { depotId, manifestId -> + Pair(depotId, manifestId) + } + depotManifestIds.addAll(zippedDepotManifest) + } else { + depotManifestIds.addAll( + depotIdList.map { depotId -> + Pair(depotId, INVALID_MANIFEST_ID) + } + ) + } + + logger?.debug("Downloading App for ${item.appId}") + + notifyListeners { it.onDownloadStarted(item) } + downloadApp( + appId = item.appId, + depotManifestIds = depotManifestIds, + branch = branch, + os = os, + arch = arch, + language = language, + lv = lv, + isUgc = isUGC, + ) + } + } + + notifyListeners { it.onDownloadCompleted(item) } + } + } catch (e: IOException) { + logger?.error("Error downloading item ${item.appId}: ${e.message}", e) + + notifyListeners { it.onDownloadFailed(item, e) } + + throw e + } + } + + return@future true + } + + override fun close() { + isStarted.set(false) + + HttpClient.close() + + items.clear() + processingChannel.close() + + lastFileProgressUpdate.clear() + listeners.clear() + + steam3?.close() + steam3 = null + + cdnClientPool?.close() + cdnClientPool = null + + logger = null + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt new file mode 100644 index 00000000..5a514926 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt @@ -0,0 +1,7 @@ +package `in`.dragonbra.javasteam.depotdownloader + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class ContentDownloaderException(value: String) : Exception(value) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt new file mode 100644 index 00000000..2f746eb2 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -0,0 +1,53 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import okio.FileSystem +import okio.Path + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +@Serializable +data class DepotConfigStore( + val installedManifestIDs: MutableMap = mutableMapOf(), +) { + companion object { + private var instance: DepotConfigStore? = null + + private var filePath: Path? = null + + private val json = Json { prettyPrint = true } + + val isLoaded: Boolean + get() = instance != null + + fun loadFromFile(path: Path) { + // require(!isLoaded) { "Config already loaded" } + + instance = if (FileSystem.SYSTEM.exists(path)) { + FileSystem.SYSTEM.read(path) { + json.decodeFromString(readUtf8()) + } + } else { + DepotConfigStore() + } + + filePath = path + } + + fun save() { + val currentInstance = requireNotNull(instance) { "Saved config before loading" } + val currentPath = requireNotNull(filePath) { "File path not set" } + + currentPath.parent?.let { FileSystem.SYSTEM.createDirectories(it) } + + FileSystem.SYSTEM.write(currentPath) { + writeUtf8(json.encodeToString(currentInstance)) + } + } + + fun getInstance(): DepotConfigStore = requireNotNull(instance) { "Config not loaded" } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt new file mode 100644 index 00000000..03908dc2 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -0,0 +1,46 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.util.Versions +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.engine.cio.endpoint +import io.ktor.client.plugins.UserAgent +import kotlinx.coroutines.isActive + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +object HttpClient { + + private var _httpClient: HttpClient? = null + + val httpClient: HttpClient + get() { + if (_httpClient?.isActive != true) { + _httpClient = HttpClient(CIO) { + install(UserAgent) { + agent = "DepotDownloader/${Versions.getVersion()}" + } + engine { + maxConnectionsCount = 10 + endpoint { + maxConnectionsPerRoute = 5 + pipelineMaxSize = 20 + keepAliveTime = 5000 + connectTimeout = 5000 + requestTimeout = 30000 + } + } + } + } + return _httpClient!! + } + + fun close() { + if (httpClient.isActive) { + _httpClient?.close() + _httpClient = null + } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt new file mode 100644 index 00000000..c3ce3c22 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -0,0 +1,35 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem +import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress +import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress + +/** + * Listener interface for download events. + * + * @author Lossy + * @since Oct 1, 2025 + */ +interface IDownloadListener { + // Queue management + fun onItemAdded(item: DownloadItem, index: Int) {} + fun onItemRemoved(item: DownloadItem, index: Int) {} + fun onQueueCleared(previousItems: List) {} + + // Download lifecycle + fun onDownloadStarted(item: DownloadItem) {} + fun onDownloadCompleted(item: DownloadItem) {} + fun onDownloadFailed(item: DownloadItem, error: Throwable) {} + + // Progress tracking + fun onOverallProgress(progress: OverallProgress) {} + fun onDepotProgress(depotId: Int, progress: DepotProgress) {} + fun onFileProgress(depotId: Int, fileName: String, progress: FileProgress) {} + + // Status updates + fun onStatusUpdate(message: String) {} + + // Configuration + fun onAndroidEmulation(value: Boolean) {} +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt new file mode 100644 index 00000000..2b80f563 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -0,0 +1,340 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.enums.EResult +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesPublishedfileSteamclient +import `in`.dragonbra.javasteam.rpc.service.PublishedFile +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest +import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.SteamCloud +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.callback.UGCDetailsCallback +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.CDNAuthToken +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent +import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages +import `in`.dragonbra.javasteam.steam.handlers.steamuser.SteamUser +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.types.KeyValue +import `in`.dragonbra.javasteam.types.PublishedFileID +import `in`.dragonbra.javasteam.types.UGCHandle +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class Steam3Session( + private val steamClient: SteamClient, + debug: Boolean, +) : Closeable { + + private var logger: Logger? = null + + internal val appTokens = ConcurrentHashMap() + internal val packageTokens = ConcurrentHashMap() + internal val depotKeys = ConcurrentHashMap() + internal val cdnAuthTokens = ConcurrentHashMap, CompletableDeferred>() + internal val appInfo = ConcurrentHashMap() + internal val packageInfo = ConcurrentHashMap() + internal val appBetaPasswords = ConcurrentHashMap() + + private var unifiedMessages: SteamUnifiedMessages? = null + internal var steamUser: SteamUser? = null + internal var steamContent: SteamContent? = null + internal var steamApps: SteamApps? = null + internal var steamCloud: SteamCloud? = null + internal var steamPublishedFile: PublishedFile? = null + + init { + if (debug) { + logger = LogManager.getLogger(Steam3Session::class.java) + } + + unifiedMessages = requireNotNull(steamClient.getHandler()) + steamApps = requireNotNull(steamClient.getHandler()) + steamCloud = requireNotNull(steamClient.getHandler()) + steamContent = requireNotNull(steamClient.getHandler()) + steamPublishedFile = requireNotNull(unifiedMessages?.createService()) + steamUser = requireNotNull(steamClient.getHandler()) + } + + override fun close() { + logger?.debug("Closing...") + + unifiedMessages = null + steamUser = null + steamContent = null + steamApps = null + steamCloud = null + steamPublishedFile = null + + cdnAuthTokens.values.forEach { it.cancel() } + cdnAuthTokens.clear() + + depotKeys.values.forEach { it.fill(0) } + depotKeys.clear() + appBetaPasswords.values.forEach { it.fill(0) } + appBetaPasswords.clear() + + appTokens.clear() + packageTokens.clear() + appInfo.clear() + packageInfo.clear() + + logger = null + } + + suspend fun requestAppInfo(appId: Int, bForce: Boolean = false) { + if (appInfo.containsKey(appId) && !bForce) { + return + } + + val appTokens = steamApps!!.picsGetAccessTokens(appId).await() + + if (appTokens.appTokensDenied.contains(appId)) { + logger?.error("Insufficient privileges to get access token for app $appId") + } + + appTokens.appTokens.forEach { tokenDict -> + this.appTokens[tokenDict.key] = tokenDict.value + } + + val request = PICSRequest(appId) + + this.appTokens[appId]?.let { token -> + request.accessToken = token + } + + val appInfoMultiple = steamApps!!.picsGetProductInfo(request).await() + + logger?.debug( + "requestAppInfo($appId, $bForce) with \n" + + "${appTokens.appTokens.size} appTokens, \n" + + "${appTokens.appTokensDenied.size} appTokensDenied, \n" + + "${appTokens.packageTokens.size} packageTokens, and \n" + + "${appTokens.packageTokensDenied} packageTokensDenied. \n" + + "picsGetProductInfo result size: ${appInfoMultiple.results.size}" + ) + + appInfoMultiple.results.forEach { appInfo -> + appInfo.apps.forEach { appValue -> + val app = appValue.value + this.appInfo[app.id] = app + } + appInfo.unknownApps.forEach { app -> + this.appInfo[app] = null + } + } + } + + // TODO race condition (??) + private val packageInfoMutex = Mutex() + suspend fun requestPackageInfo(packageIds: List) { + packageInfoMutex.withLock { + // I have a silly race condition??? + val packages = packageIds.filter { !packageInfo.containsKey(it) } + + if (packages.isEmpty()) return + + val packageRequests = arrayListOf() + + packages.forEach { pkg -> + val request = PICSRequest(id = pkg) + + packageTokens[pkg]?.let { token -> + request.accessToken = token + } + + packageRequests.add(request) + } + + val packageInfoMultiple = steamApps!!.picsGetProductInfo(emptyList(), packageRequests).await() + + logger?.debug( + "requestPackageInfo(packageIds =${packageIds.size}) \n" + + "picsGetProductInfo result size: ${packageInfoMultiple.results.size} " + ) + + packageInfoMultiple.results.forEach { pkgInfo -> + pkgInfo.packages.forEach { pkgValue -> + val pkg = pkgValue.value + packageInfo[pkg.id] = pkg + } + pkgInfo.unknownPackages.forEach { pkgValue -> + packageInfo[pkgValue] = null + } + } + } + } + + suspend fun requestFreeAppLicense(appId: Int): Boolean { + try { + val resultInfo = steamApps!!.requestFreeLicense(appId).await() + + logger?.debug("requestFreeAppLicense($appId) has result ${resultInfo.result}") + + return resultInfo.grantedApps.contains(appId) + } catch (e: Exception) { + logger?.error("Failed to request FreeOnDemand license for app $appId: ${e.message}") + return false + } + } + + suspend fun requestDepotKey(depotId: Int, appId: Int = 0) { + if (depotKeys.containsKey(depotId)) { + return + } + + val depotKey = steamApps!!.getDepotDecryptionKey(depotId, appId).await() + + logger?.debug( + "requestDepotKey($depotId, $appId) " + + "Got depot key for ${depotKey.depotID} result: ${depotKey.result}" + ) + + if (depotKey.result != EResult.OK) { + return + } + + depotKeys[depotKey.depotID] = depotKey.depotKey + } + + suspend fun getDepotManifestRequestCode( + depotId: Int, + appId: Int, + manifestId: Long, + branch: String, + ): ULong = withContext(Dispatchers.IO) { + val requestCode = steamContent!!.getManifestRequestCode( + depotId = depotId, + appId = appId, + manifestId = manifestId, + branch = branch, + branchPasswordHash = null, + parentScope = this // TODO am I passing this right? + ).await().toULong() + + if (requestCode == 0UL) { + logger?.error("No manifest request code was returned for depot $depotId from app $appId, manifest $manifestId") + + if (steamClient.isDisconnected) { + logger?.debug("Suggestion: Try logging in with -username as old manifests may not be available for anonymous accounts.") + } + } else { + logger?.debug("Got manifest request code for depot $depotId from app $appId, manifest $manifestId, result: $requestCode") + } + + logger?.debug( + "getDepotManifestRequestCode($depotId, $appId, $manifestId, $branch) " + + "got request code $requestCode" + ) + + return@withContext requestCode + } + + suspend fun requestCDNAuthToken(appId: Int, depotId: Int, server: Server) = withContext(Dispatchers.IO) { + val cdnKey = depotId to server.host!! + + if (cdnAuthTokens.containsKey(cdnKey)) { + return@withContext + } + + val completion = CompletableDeferred() + + val existing = cdnAuthTokens.putIfAbsent(cdnKey, completion) + if (existing != null) { + return@withContext + } + + logger?.debug("Requesting CDN auth token for ${server.host}") + + try { + val cdnAuth = steamContent!!.getCDNAuthToken(appId, depotId, server.host!!, this).await() + + logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") + + if (cdnAuth.result != EResult.OK) { + cdnAuthTokens.remove(cdnKey) // Remove failed promise + completion.completeExceptionally(Exception("Failed to get CDN auth token: ${cdnAuth.result}")) + return@withContext + } + + completion.complete(cdnAuth) + } catch (e: Exception) { + logger?.error(e) + cdnAuthTokens.remove(cdnKey) // Remove failed promise + completion.completeExceptionally(e) + } + } + + suspend fun checkAppBetaPassword(appId: Int, password: String) { + val appPassword = steamApps!!.checkAppBetaPassword(appId, password).await() + + logger?.debug( + "checkAppBetaPassword($appId, )," + + "retrieved ${appPassword.betaPasswords.size} beta keys with result: ${appPassword.result}" + ) + + appPassword.betaPasswords.forEach { entry -> + this.appBetaPasswords[entry.key] = entry.value + } + } + + suspend fun getPrivateBetaDepotSection(appId: Int, branch: String): KeyValue { + // Should be filled by CheckAppBetaPassword + val branchPassword = appBetaPasswords[branch] ?: return KeyValue() + + // Should be filled by RequestAppInfo + val accessToken = appTokens[appId] ?: 0L + + val privateBeta = steamApps!!.picsGetPrivateBeta(appId, accessToken, branch, branchPassword).await() + + logger?.debug("getPrivateBetaDepotSection($appId, $branch) result: ${privateBeta.result}") + + return privateBeta.depotSection + } + + @Throws(ContentDownloaderException::class) + suspend fun getPublishedFileDetails( + appId: Int, + pubFile: PublishedFileID, + ): SteammessagesPublishedfileSteamclient.PublishedFileDetails? { + val pubFileRequest = + SteammessagesPublishedfileSteamclient.CPublishedFile_GetDetails_Request.newBuilder().apply { + this.appid = appId + this.addPublishedfileids(pubFile.toLong()) + }.build() + + val details = steamPublishedFile!!.getDetails(pubFileRequest).await() + + logger?.debug("requestUGCDetails($appId, $pubFile) result: ${details.result}") + + if (details.result == EResult.OK) { + return details.body.publishedfiledetailsBuilderList.firstOrNull()?.build() + } + + throw ContentDownloaderException("EResult ${details.result.code()} (${details.result}) while retrieving file details for pubfile $pubFile.") + } + + suspend fun getUGCDetails(ugcHandle: UGCHandle): UGCDetailsCallback? { + val callback = steamCloud!!.requestUGCDetails(ugcHandle).await() + + logger?.debug("requestUGCDetails($ugcHandle) result: ${callback.result}") + + if (callback.result == EResult.OK) { + return callback + } else if (callback.result == EResult.FileNotFound) { + return null + } + + throw ContentDownloaderException("EResult ${callback.result.code()} (${callback.result}) while retrieving UGC details for ${ugcHandle.value}.") + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt new file mode 100644 index 00000000..b69a0047 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -0,0 +1,207 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.util.Adler32 +import okio.FileHandle +import okio.FileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer +import org.apache.commons.lang3.SystemUtils +import java.io.IOException +import java.security.MessageDigest + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +object Util { + + @JvmOverloads + @JvmStatic + fun getSteamOS(androidEmulation: Boolean = false): String { + if (SystemUtils.IS_OS_WINDOWS) { + return "windows" + } + if (SystemUtils.IS_OS_MAC_OSX) { + return "macos" + } + if (SystemUtils.IS_OS_LINUX) { + return "linux" + } + if (SystemUtils.IS_OS_FREE_BSD) { + // Return linux as freebsd steam client doesn't exist yet + return "linux" + } + + // Hack for PC emulation on android. (Pluvia, GameNative, GameHub) + if (androidEmulation && SystemUtils.IS_OS_ANDROID) { + return "windows" + } + + return "unknown" + } + + @JvmStatic + fun getSteamArch(): String { + val arch = System.getProperty("os.arch")?.lowercase() ?: "" + return when { + arch.contains("64") -> "64" + arch.contains("86") -> "32" + arch.contains("amd64") -> "64" + arch.contains("x86_64") -> "64" + arch.contains("aarch64") -> "64" + arch.contains("arm64") -> "64" + else -> "32" + } + } + + @JvmStatic + fun saveManifestToFile(directory: Path, manifest: DepotManifest): Boolean = try { + val filename = directory / "${manifest.depotID}_${manifest.manifestGID}.manifest" + manifest.saveToFile(filename.toString()) + + val shaFile = "$filename.sha".toPath() + FileSystem.SYSTEM.write(shaFile) { + write(fileSHAHash(filename)) + } + + true + } catch (e: Exception) { + false + } + + @JvmStatic + fun loadManifestFromFile( + directory: Path, + depotId: Int, + manifestId: Long, + badHashWarning: Boolean, + ): DepotManifest? { + // Try loading Steam format manifest first. + val filename = directory / "${depotId}_$manifestId.manifest" + + if (FileSystem.SYSTEM.exists(filename)) { + val expectedChecksum = try { + FileSystem.SYSTEM.read(filename / ".sha") { + readByteArray() + } + } catch (e: IOException) { + null + } + + val currentChecksum = fileSHAHash(filename) + + if (expectedChecksum != null && expectedChecksum.contentEquals(currentChecksum)) { + return DepotManifest.loadFromFile(filename.toString()) + } else if (badHashWarning) { + println("Manifest $manifestId on disk did not match the expected checksum.") + } + } + + return null + } + + @JvmStatic + fun fileSHAHash(filename: Path): ByteArray { + val digest = MessageDigest.getInstance("SHA-1") + + FileSystem.SYSTEM.source(filename).use { source -> + source.buffer().use { bufferedSource -> + val buffer = ByteArray(8192) + var bytesRead: Int + + while (bufferedSource.read(buffer).also { bytesRead = it } != -1) { + digest.update(buffer, 0, bytesRead) + } + } + } + + return digest.digest() + } + + /** + * Validate a file against Steam3 Chunk data + * + * @param handle FileHandle to read from + * @param chunkData Array of ChunkData to validate against + * @return List of ChunkData that are needed + * @throws IOException If there's an error reading the file + */ + @Throws(IOException::class) + fun validateSteam3FileChecksums(handle: FileHandle, chunkData: List): List { + val neededChunks = mutableListOf() + + for (data in chunkData) { + val chunk = ByteArray(data.uncompressedLength) + val read = handle.read(data.offset, chunk, 0, data.uncompressedLength) + + val tempChunk = if (read > 0 && read < data.uncompressedLength) { + chunk.copyOf(read) + } else { + chunk + } + + val adler = Adler32.calculate(tempChunk) + if (adler != data.checksum) { + neededChunks.add(data) + } + } + + return neededChunks + } + + @JvmStatic + fun dumpManifestToTextFile(depot: DepotDownloadInfo, manifest: DepotManifest) { + val txtManifest = depot.installDir / "manifest_${depot.depotId}_${depot.manifestId}.txt" + + FileSystem.SYSTEM.write(txtManifest) { + writeUtf8("Content Manifest for Depot ${depot.depotId}\n") + writeUtf8("\n") + writeUtf8("Manifest ID / date : ${depot.manifestId} / ${manifest.creationTime}\n") + + val uniqueChunks = manifest.files + .flatMap { it.chunks } + .mapNotNull { it.chunkID } + .toSet() + + writeUtf8("Total number of files : ${manifest.files.size}\n") + writeUtf8("Total number of chunks : ${uniqueChunks.size}\n") + writeUtf8("Total bytes on disk : ${manifest.totalUncompressedSize}\n") + writeUtf8("Total bytes compressed : ${manifest.totalCompressedSize}\n") + writeUtf8("\n") + writeUtf8("\n") + + writeUtf8(" Size Chunks File SHA Flags Name\n") + manifest.files.forEach { file -> + val sha1Hash = file.fileHash.toHexString().lowercase() + writeUtf8( + "%14d %6d %s %5x %s\n".format( + file.totalSize, + file.chunks.size, + sha1Hash, + EDepotFileFlag.code(file.flags), + file.fileName + ) + ) + } + } + } + + @JvmStatic + fun formatBytes(bytes: Long): String { + val units = arrayOf("B", "KB", "MB", "GB") + var size = bytes.toDouble() + var unitIndex = 0 + + while (size >= 1024 && unitIndex < units.size - 1) { + size /= 1024 + unitIndex++ + } + + return "%.2f %s".format(size, units[unitIndex]) + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt new file mode 100644 index 00000000..05fbb247 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt @@ -0,0 +1,9 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.types.ChunkData + +/** + * @author Oxters + * @since Oct 29, 2024 + */ +data class ChunkMatch(val oldChunk: ChunkData, val newChunk: ChunkData) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt new file mode 100644 index 00000000..256575ec --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt @@ -0,0 +1,41 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import okio.Path + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotDownloadInfo( + val depotId: Int, + val appId: Int, + val manifestId: Long, + val branch: String, + val installDir: Path, + val depotKey: ByteArray, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is DepotDownloadInfo) return false + + if (depotId != other.depotId) return false + if (appId != other.appId) return false + if (manifestId != other.manifestId) return false + if (branch != other.branch) return false + if (installDir != other.installDir) return false + if (!depotKey.contentEquals(other.depotKey)) return false + + return true + } + + override fun hashCode(): Int { + var result = depotId + result = 31 * result + appId + result = 31 * result + manifestId.hashCode() + result = 31 * result + branch.hashCode() + result = 31 * result + installDir.hashCode() + result = 31 * result + depotKey.contentHashCode() + return result + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt new file mode 100644 index 00000000..bef4590b --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt @@ -0,0 +1,20 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.types.FileData +import okio.Path + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotFilesData( + val depotDownloadInfo: DepotDownloadInfo, + val depotCounter: DepotDownloadCounter, + val stagingDir: Path, + val manifest: DepotManifest, + val previousManifest: DepotManifest?, + val filteredFiles: MutableList, + val allFileNames: HashSet, +) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt new file mode 100644 index 00000000..8d509e27 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt @@ -0,0 +1,26 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +// https://kotlinlang.org/docs/coding-conventions.html#source-file-organization + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class GlobalDownloadCounter( + var completeDownloadSize: Long = 0, + var totalBytesCompressed: Long = 0, + var totalBytesUncompressed: Long = 0, +) + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotDownloadCounter( + var completeDownloadSize: Long = 0, + var sizeDownloaded: Long = 0, + var depotBytesCompressed: Long = 0, + var depotBytesUncompressed: Long = 0, +) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt new file mode 100644 index 00000000..a204c4e1 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -0,0 +1,62 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.depotdownloader.ContentDownloader + +// https://kotlinlang.org/docs/coding-conventions.html#source-file-organization + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +abstract class DownloadItem( + val appId: Int, + val installDirectory: String?, + val installToGameNameDirectory: Boolean, + val downloadManifestOnly: Boolean, +) + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class UgcItem @JvmOverloads constructor( + appId: Int, + val ugcId: Long = ContentDownloader.INVALID_MANIFEST_ID, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class PubFileItem @JvmOverloads constructor( + appId: Int, + val pubfile: Long = ContentDownloader.INVALID_MANIFEST_ID, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class AppItem @JvmOverloads constructor( + appId: Int, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + val branch: String? = null, + val branchPassword: String? = null, + val downloadAllPlatforms: Boolean = false, + val os: String? = null, + val downloadAllArchs: Boolean = false, + val osArch: String? = null, + val downloadAllLanguages: Boolean = false, + val language: String? = null, + val lowViolence: Boolean = false, + val depot: List = emptyList(), + val manifest: List = emptyList(), + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt new file mode 100644 index 00000000..f8e8035d --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt @@ -0,0 +1,72 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +/** + * Overall download progress across all items. + * + * @author Lossy + * @since Oct 1, 2025 + */ +data class OverallProgress( + val currentItem: Int, + val totalItems: Int, + val totalBytesDownloaded: Long, + val totalBytesExpected: Long, + val status: DownloadStatus, +) { + val percentComplete: Double + get() = if (totalBytesExpected > 0) { + (totalBytesDownloaded.toDouble() / totalBytesExpected) * 100.0 + } else { + 0.0 + } +} + +/** + * Progress for a specific depot + * + * @author Lossy + * @since Oct 1, 2025 + */ +data class DepotProgress( + val depotId: Int, + val filesCompleted: Int, + val totalFiles: Int, + val bytesDownloaded: Long, + val totalBytes: Long, + val status: DownloadStatus, +) { + val percentComplete: Double + get() = if (totalBytes > 0) { + (bytesDownloaded.toDouble() / totalBytes) * 100.0 + } else { + 0.0 + } +} + +/** + * Progress for a specific file + * + * @author Lossy + * @since Oct 1, 2025 + */ +data class FileProgress( + val fileName: String, + val bytesDownloaded: Long, + val totalBytes: Long, + val chunksCompleted: Int, + val totalChunks: Int, + val status: DownloadStatus, +) { + val percentComplete: Double + get() = if (totalBytes > 0) { + (bytesDownloaded.toDouble() / totalBytes) * 100.0 + } else { + 0.0 + } +} + +enum class DownloadStatus { + PREPARING, + DOWNLOADING, + COMPLETED, +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt new file mode 100644 index 00000000..2b6a539c --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt @@ -0,0 +1,16 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import kotlinx.coroutines.sync.Mutex +import okio.FileHandle +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class FileStreamData( + var fileHandle: FileHandle?, + val fileLock: Mutex = Mutex(), + var chunksToDownload: AtomicInteger = AtomicInteger(0), +) diff --git a/javasteam-samples/build.gradle.kts b/javasteam-samples/build.gradle.kts index 9deffead..1e1aa774 100644 --- a/javasteam-samples/build.gradle.kts +++ b/javasteam-samples/build.gradle.kts @@ -14,12 +14,13 @@ java { dependencies { implementation(rootProject) implementation(project(":javasteam-cs")) + implementation(project(":javasteam-depotdownloader")) implementation(libs.bouncyCastle) implementation(libs.gson) implementation(libs.kotlin.coroutines) implementation(libs.okHttp) - implementation(libs.protobuf.java) // To access protobufs directly as shown in Sample #2 + implementation(libs.protobuf.java) // Protobuf access implementation(libs.qrCode) implementation(libs.zstd) // Content Downloading. implementation(libs.xz) // Content Downloading. diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index e22d7e08..b73f408a 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -1,9 +1,17 @@ package in.dragonbra.javasteamsamples._023_downloadapp; +import in.dragonbra.javasteam.depotdownloader.ContentDownloader; +import in.dragonbra.javasteam.depotdownloader.IDownloadListener; +import in.dragonbra.javasteam.depotdownloader.data.*; import in.dragonbra.javasteam.enums.EResult; -import in.dragonbra.javasteam.steam.contentdownloader.ContentDownloader; +import in.dragonbra.javasteam.steam.authentication.AuthPollResult; +import in.dragonbra.javasteam.steam.authentication.AuthSessionDetails; +import in.dragonbra.javasteam.steam.authentication.AuthenticationException; +import in.dragonbra.javasteam.steam.authentication.UserConsoleAuthenticator; +import in.dragonbra.javasteam.steam.handlers.steamapps.License; import in.dragonbra.javasteam.steam.handlers.steamapps.SteamApps; import in.dragonbra.javasteam.steam.handlers.steamapps.callback.FreeLicenseCallback; +import in.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback; import in.dragonbra.javasteam.steam.handlers.steamuser.LogOnDetails; import in.dragonbra.javasteam.steam.handlers.steamuser.SteamUser; import in.dragonbra.javasteam.steam.handlers.steamuser.callback.LoggedOffCallback; @@ -14,8 +22,17 @@ import in.dragonbra.javasteam.steam.steamclient.callbacks.DisconnectedCallback; import in.dragonbra.javasteam.util.log.DefaultLogListener; import in.dragonbra.javasteam.util.log.LogManager; +import org.jetbrains.annotations.NotNull; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; -import java.io.File; /** * @author Oxters @@ -32,15 +49,18 @@ * called Rocky Mayhem */ @SuppressWarnings("FieldCanBeLocal") -public class SampleDownloadApp implements Runnable { +public class SampleDownloadApp implements Runnable, IDownloadListener { private final int ROCKY_MAYHEM_APP_ID = 1303350; - private final int ROCKY_MAYHEM_DEPOT_ID = 1303351; + + private final String DEFAULT_INSTALL_DIRECTORY = "steamapps"; private SteamClient steamClient; private CallbackManager manager; + private SteamUser steamUser; + private SteamApps steamApps; private boolean isRunning; @@ -49,12 +69,13 @@ public class SampleDownloadApp implements Runnable { private final String pass; - private final String twoFactor; + private List subscriptions; + + private List licenseList; - public SampleDownloadApp(String user, String pass, String twoFactor) { + public SampleDownloadApp(String user, String pass) { this.user = user; this.pass = pass; - this.twoFactor = twoFactor; } public static void main(String[] args) { @@ -65,68 +86,111 @@ public static void main(String[] args) { LogManager.addListener(new DefaultLogListener()); - String twoFactor = null; - if (args.length == 3) - twoFactor = args[2]; - new SampleDownloadApp(args[0], args[1], twoFactor).run(); + new SampleDownloadApp(args[0], args[1]).run(); } @Override public void run() { - // create our steamclient instance + // Most everything has been described in earlier samples. + // Anything pertaining to this sample will be commented. + steamClient = new SteamClient(); - // create the callback manager which will route callbacks to function calls manager = new CallbackManager(steamClient); - // get the steamuser handler, which is used for logging on after successfully connecting steamUser = steamClient.getHandler(SteamUser.class); - steamApps = steamClient.getHandler(SteamApps.class); - // register a few callbacks we're interested in - // these are registered upon creation to a callback manager, which will then route the callbacks - // to the functions specified - manager.subscribe(ConnectedCallback.class, this::onConnected); - manager.subscribe(DisconnectedCallback.class, this::onDisconnected); + steamApps = steamClient.getHandler(SteamApps.class); - manager.subscribe(LoggedOnCallback.class, this::onLoggedOn); - manager.subscribe(LoggedOffCallback.class, this::onLoggedOff); + subscriptions = new ArrayList<>(); - manager.subscribe(FreeLicenseCallback.class, this::onFreeLicense); + subscriptions.add(manager.subscribe(ConnectedCallback.class, this::onConnected)); + subscriptions.add(manager.subscribe(DisconnectedCallback.class, this::onDisconnected)); + subscriptions.add(manager.subscribe(LoggedOnCallback.class, this::onLoggedOn)); + subscriptions.add(manager.subscribe(LoggedOffCallback.class, this::onLoggedOff)); + subscriptions.add(manager.subscribe(LicenseListCallback.class, this::onLicenseList)); + subscriptions.add(manager.subscribe(FreeLicenseCallback.class, this::onFreeLicense)); isRunning = true; System.out.println("Connecting to steam..."); - // initiate the connection steamClient.connect(); - // create our callback handling loop while (isRunning) { - // in order for the callbacks to get routed, they need to be handled by the manager manager.runWaitCallbacks(1000L); } + + for (var subscription : subscriptions) { + try { + subscription.close(); + } catch (IOException e) { + System.out.println("Couldn't close a callback."); + } + } } private void onConnected(ConnectedCallback callback) { System.out.println("Connected to Steam! Logging in " + user + "..."); - LogOnDetails details = new LogOnDetails(); - details.setUsername(user); - details.setPassword(pass); - if (twoFactor != null) { - details.setTwoFactorCode(twoFactor); - } + AuthSessionDetails authDetails = new AuthSessionDetails(); + authDetails.username = user; + authDetails.password = pass; + authDetails.deviceFriendlyName = "JavaSteam - Sample 023"; + authDetails.persistentSession = true; + + authDetails.authenticator = new UserConsoleAuthenticator(); + + try { + var path = Paths.get("refreshtoken.txt"); + + String accountName; + String refreshToken; + if (!Files.exists(path)) { + System.out.println("No existing refresh token found. Beginning Authentication"); + + var authSession = steamClient.getAuthentication().beginAuthSessionViaCredentials(authDetails).get(); + + AuthPollResult pollResponse = authSession.pollingWaitForResult().get(); - // Set LoginID to a non-zero value if you have another client connected using the same account, - // the same private ip, and same public ip. - details.setLoginID(149); + accountName = pollResponse.getAccountName(); + refreshToken = pollResponse.getRefreshToken(); - steamUser.logOn(details); + // Save out refresh token for automatic login on next sample run. + Files.writeString(path, pollResponse.getRefreshToken()); + } else { + System.out.println("Existing refresh token found"); + var token = Files.readString(path); + + accountName = user; + refreshToken = token; + } + + LogOnDetails details = new LogOnDetails(); + details.setUsername(accountName); + details.setAccessToken(refreshToken); + details.setShouldRememberPassword(true); + + details.setLoginID(149); + + System.out.println("Logging in..."); + + steamUser.logOn(details); + } catch (Exception e) { + if (e instanceof AuthenticationException) { + System.err.println("An Authentication error has occurred. " + e.getMessage()); + } else if (e instanceof CancellationException) { + System.err.println("An Cancellation exception was raised. Usually means a timeout occurred. " + e.getMessage()); + } else { + System.err.println("An error occurred:" + e.getMessage()); + } + + steamUser.logOff(); + } } private void onDisconnected(DisconnectedCallback callback) { - System.out.println("Disconnected from Steam"); + System.out.println("Disconnected from Steam, UserInitiated: " + callback.isUserInitiated()); if (callback.isUserInitiated()) { isRunning = false; @@ -143,9 +207,6 @@ private void onDisconnected(DisconnectedCallback callback) { private void onLoggedOn(LoggedOnCallback callback) { if (callback.getResult() != EResult.OK) { if (callback.getResult() == EResult.AccountLogonDenied) { - // if we receive AccountLogonDenied or one of its flavors (AccountLogonDeniedNoMailSent, etc.) - // then the account we're logging into is SteamGuard protected - // see sample 5 for how SteamGuard can be handled System.out.println("Unable to logon to Steam: This account is SteamGuard protected."); isRunning = false; @@ -163,9 +224,21 @@ private void onLoggedOn(LoggedOnCallback callback) { // now that we are logged in, we can request a free license for Rocky Mayhem steamApps.requestFreeLicense(ROCKY_MAYHEM_APP_ID); + } + private void onLicenseList(LicenseListCallback callback) { + if (callback.getResult() != EResult.OK) { + System.out.println("Failed to obtain licenses the account owns."); + steamClient.disconnect(); + return; + } + + licenseList = callback.getLicenseList(); + + System.out.println("Got " + licenseList.size() + " licenses from account!"); } + @SuppressWarnings("ExtractMethodRecommender") private void onFreeLicense(FreeLicenseCallback callback) { if (callback.getResult() != EResult.OK) { System.out.println("Failed to get a free license for Rocky Mayhem"); @@ -173,24 +246,72 @@ private void onFreeLicense(FreeLicenseCallback callback) { return; } - // we have successfully received a free license for Rocky Mayhem so now we can start the download process - // note: it is okay to see some errors about ContentDownloader failing to download a chunk, it will retry and continue. - new File("steamapps/staging/").mkdirs(); - var contentDownloader = new ContentDownloader(steamClient); - contentDownloader.downloadApp( - ROCKY_MAYHEM_APP_ID, - ROCKY_MAYHEM_DEPOT_ID, - "steamapps/", - "steamapps/staging/", - "public", - 8, - progress -> System.out.println("Download progress: " + progress) - ).thenAccept(success -> { + // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. + // You will need to subscribe to LicenseListCallback to obtain your app licenses. + try (var depotDownloader = new ContentDownloader(steamClient, licenseList, false)) { + + depotDownloader.addListener(this); + + // An app id is required at minimum for all item types. + var pubItem = new PubFileItem( + /* appId */ 0, + /* pubfile */ 0, + /* installToGameNameDirectory */ false, + /* installDirectory */ null, + /* downloadManifestOnly */ false + ); // TODO find actual pub item + + var ugcItem = new UgcItem( + /* appId */0, + /* ugcId */ 0, + /* installToGameNameDirectory */ false, + /* installDirectory */ null, + /* downloadManifestOnly */ false + ); // TODO find actual ugc item + + var appItem = new AppItem( + /* appId */ 204360, + /* installToGameNameDirectory */ true, + /* installDirectory */ DEFAULT_INSTALL_DIRECTORY, + /* branch */ "public", + /* branchPassword */ "", + /* downloadAllPlatforms */ false, + /* os */ "windows", + /* downloadAllArchs */ false, + /* osArch */ "64", + /* downloadAllLanguages */ false, + /* language */ "english", + /* lowViolence */ false, + /* depot */ List.of(), + /* manifest */ List.of(), + /* downloadManifestOnly */ false + ); + + var appItem2 = new AppItem(225840, true); + var appItem3 = new AppItem(3527290, true); + var appItem4 = new AppItem(ROCKY_MAYHEM_APP_ID, true); + + var downloadList = List.of(pubItem, ugcItem, appItem); + + // Add specified games to the queue. Add, Remove, Move, and general array manipulation methods are available. + // depotDownloader.addAll(downloadList); // TODO + depotDownloader.addAll(List.of(appItem)); + + // Start downloading your items. Array manipulation is now disabled. You can still add to the list. + var success = depotDownloader.start().get(); // Future + if (success) { System.out.println("Download completed successfully"); } - steamClient.disconnect(); - }); + + depotDownloader.removeListener(this); + } catch (IllegalStateException | InterruptedException | ExecutionException e) { + System.out.println("Something happened"); + System.err.println(e.getMessage()); + } finally { + steamUser.logOff(); + System.out.println("Done Downloading"); + } } private void onLoggedOff(LoggedOffCallback callback) { @@ -198,4 +319,96 @@ private void onLoggedOff(LoggedOffCallback callback) { isRunning = false; } + + // Depot Downloader Callbacks. + + @Override + public void onItemAdded(@NotNull DownloadItem item, int index) { + System.out.println("Depot Downloader: Item Added: " + item.getAppId() + ", index: " + index); + System.out.println(" ---- "); + } + + @Override + public void onItemRemoved(@NotNull DownloadItem item, int index) { + System.out.println("Depot Downloader: Item Removed: " + item.getAppId() + ", index: " + index); + System.out.println(" ---- "); + } + + @Override + public void onQueueCleared(@NotNull List previousItems) { + System.out.println("Depot Downloader: Queue size of " + previousItems.size() + " cleared"); + System.out.println(" ---- "); + } + + @Override + public void onDownloadStarted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download started for item: " + item.getAppId()); + System.out.println(" ---- "); + } + + @Override + public void onDownloadCompleted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download completed for item: " + item.getAppId()); + System.out.println(" ---- "); + } + + @Override + public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { + System.out.println("Depot Downloader: Download failed for item: " + item.getAppId()); + System.err.println(error.getMessage()); + System.out.println(" ---- "); + } + + @Override + public void onOverallProgress(@NotNull OverallProgress progress) { + System.out.println("Depot Downloader: Overall Progress"); + System.out.println("currentItem: " + progress.getCurrentItem()); + System.out.println("totalItems: " + progress.getTotalItems()); + System.out.println("totalBytesDownloaded: " + progress.getTotalBytesDownloaded()); + System.out.println("totalBytesExpected: " + progress.getTotalBytesExpected()); + System.out.println("status: " + progress.getStatus()); + System.out.println("percentComplete: " + progress.getPercentComplete()); + System.out.println(" ---- "); + } + + @Override + public void onDepotProgress(int depotId, @NotNull DepotProgress progress) { + System.out.println("Depot Downloader: Depot Progress"); + System.out.println("depotId: " + depotId); + System.out.println("depotId: " + progress.getDepotId()); + System.out.println("filesCompleted: " + progress.getFilesCompleted()); + System.out.println("totalFiles: " + progress.getTotalFiles()); + System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); + System.out.println("totalBytes: " + progress.getTotalBytes()); + System.out.println("status: " + progress.getStatus()); + System.out.println("percentComplete: " + progress.getPercentComplete()); + System.out.println(" ---- "); + } + + @Override + public void onFileProgress(int depotId, @NotNull String fileName, @NotNull FileProgress progress) { + System.out.println("Depot Downloader: File Progress"); + System.out.println("depotId: " + depotId); + System.out.println("fileName: " + fileName); + System.out.println("fileName: " + progress.getFileName()); + System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); + System.out.println("totalBytes: " + progress.getTotalBytes()); + System.out.println("chunksCompleted: " + progress.getChunksCompleted()); + System.out.println("totalChunks: " + progress.getTotalChunks()); + System.out.println("status: " + progress.getStatus()); + System.out.println("percentComplete: " + progress.getPercentComplete()); + System.out.println(" ---- "); + } + + @Override + public void onStatusUpdate(@NotNull String message) { + System.out.println("Depot Downloader: Status Message: " + message); + System.out.println(" ---- "); + } + + @Override + public void onAndroidEmulation(boolean value) { + System.out.println("Depot Downloader: Android Emulation: " + value); + System.out.println(" ---- "); + } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 1dc5b8bc..fa1f8579 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,7 @@ rootProject.name = "javasteam" -include(":javasteam-samples") include(":javasteam-cs") +include(":javasteam-deadlock") +include(":javasteam-depotdownloader") +include(":javasteam-dota2") +include(":javasteam-samples") include(":javasteam-tf") -include("javasteam-deadlock") -include("javasteam-dota2") diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index 9f7591f4..f77917e4 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -6,22 +6,23 @@ import `in`.dragonbra.javasteam.types.ChunkData import `in`.dragonbra.javasteam.types.DepotManifest import `in`.dragonbra.javasteam.util.SteamKitWebRequestException import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger -import `in`.dragonbra.javasteam.util.stream.MemoryStream import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.future.future +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout import okhttp3.HttpUrl import okhttp3.OkHttpClient import okhttp3.Request -import java.io.ByteArrayOutputStream +import okhttp3.coroutines.executeAsync +import java.io.ByteArrayInputStream import java.io.Closeable import java.io.IOException -import java.util.concurrent.* -import java.util.zip.DataFormatException +import java.util.concurrent.CompletableFuture import java.util.zip.ZipInputStream /** @@ -32,64 +33,76 @@ import java.util.zip.ZipInputStream */ class Client(steamClient: SteamClient) : Closeable { - private val httpClient: OkHttpClient = steamClient.configuration.httpClient - - private val defaultScope = CoroutineScope(Dispatchers.IO) + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) companion object { - private val logger: Logger = LogManager.getLogger() + private val logger: Logger = LogManager.getLogger(Client::class.java) /** * Default timeout to use when making requests */ - var requestTimeout = 10000L + var requestTimeout = 10_000L /** * Default timeout to use when reading the response body */ - var responseBodyTimeout = 60000L + var responseBodyTimeout = 60_000L - @JvmStatic - @JvmOverloads - fun buildCommand( + private fun buildCommand( server: Server, command: String, query: String? = null, proxyServer: Server? = null, ): HttpUrl { - // TODO look into this to mimic SK's method. Should be able to remove if/else and only have the if. - val httpUrl: HttpUrl + val scheme = if (server.protocol == Server.ConnectionProtocol.HTTP) "http" else "https" + var host = server.vHost ?: server.host ?: "" + var port = server.port + var path = command + if (proxyServer != null && proxyServer.useAsProxy && proxyServer.proxyRequestPathTemplate != null) { - httpUrl = HttpUrl.Builder() - .scheme(if (proxyServer.protocol == Server.ConnectionProtocol.HTTP) "http" else "https") - .host(proxyServer.vHost) - .port(proxyServer.port) - .addPathSegment(server.vHost) - .addPathSegments(command) - .run { - query?.let { this.query(it) } ?: this - }.build() - } else { - httpUrl = HttpUrl.Builder() - .scheme(if (server.protocol == Server.ConnectionProtocol.HTTP) "http" else "https") - .host(server.vHost) - .port(server.port) - .addPathSegments(command) - .run { - query?.let { this.query(it) } ?: this - }.build() + val pathTemplate = proxyServer.proxyRequestPathTemplate!! + .replace("%host%", host) + .replace("%path%", "/$command") + + host = proxyServer.vHost ?: proxyServer.host ?: "" + port = proxyServer.port + path = pathTemplate + } + + val urlBuilder = HttpUrl.Builder() + .scheme(scheme) + .host(host) + .port(port) + .addPathSegments(path.trimStart('/')) + + query?.let { queryString -> + if (queryString.isNotEmpty()) { + val params = queryString.split("&") + for (param in params) { + val keyValue = param.split("=", limit = 2) + if (keyValue.size == 2) { + urlBuilder.addQueryParameter(keyValue[0], keyValue[1]) + } else if (keyValue.size == 1 && keyValue[0].isNotEmpty()) { + urlBuilder.addQueryParameter(keyValue[0], "") + } + } + } } - return httpUrl + return urlBuilder.build() } } + private val httpClient: OkHttpClient = steamClient.configuration.httpClient + /** * Disposes of this object. */ override fun close() { + scope.cancel() httpClient.connectionPool.evictAll() + httpClient.dispatcher.executorService.shutdown() } /** @@ -106,7 +119,6 @@ class Client(steamClient: SteamClient) : Closeable { * @exception IllegalArgumentException [server] was null. * @exception IOException A network error occurred when performing the request. * @exception SteamKitWebRequestException A network error occurred when performing the request. - * @exception DataFormatException When the data received is not as expected */ suspend fun downloadManifest( depotId: Int, @@ -116,8 +128,9 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): DepotManifest { + ): DepotManifest = withContext(Dispatchers.IO) { val manifestVersion = 5 + val url = if (manifestRequestCode > 0U) { "depot/$depotId/manifest/$manifestId/$manifestVersion/$manifestRequestCode" } else { @@ -128,8 +141,12 @@ class Client(steamClient: SteamClient) : Closeable { .url(buildCommand(server, url, cdnAuthToken, proxyServer)) .build() - return withTimeout(requestTimeout) { - val response = httpClient.newCall(request).execute() + logger.debug("Request URL is: $request") + + try { + val response = withTimeout(requestTimeout) { + httpClient.newCall(request).executeAsync() + } if (!response.isSuccessful) { throw SteamKitWebRequestException( @@ -138,93 +155,38 @@ class Client(steamClient: SteamClient) : Closeable { ) } - val depotManifest = withTimeout(responseBodyTimeout) { - val contentLength = response.header("Content-Length")?.toIntOrNull() + return@withContext withTimeout(responseBodyTimeout) { + response.use { resp -> + val responseBody = resp.body?.bytes() + ?: throw SteamKitWebRequestException("Response body is null") - if (contentLength == null) { - logger.debug("Manifest response does not have Content-Length, falling back to unbuffered read.") - } + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") + } - response.body.byteStream().use { inputStream -> - ByteArrayOutputStream().use { bs -> - val bytesRead = inputStream.copyTo(bs, contentLength ?: DEFAULT_BUFFER_SIZE) + // Decompress the zipped manifest data + ZipInputStream(ByteArrayInputStream(responseBody)).use { zipInputStream -> + zipInputStream.nextEntry + ?: throw SteamKitWebRequestException("Expected the zip to contain at least one file") - if (bytesRead != contentLength?.toLong()) { - throw DataFormatException("Length mismatch after downloading depot manifest! (was $bytesRead, but should be $contentLength)") - } + val manifestData = zipInputStream.readBytes() - val contentBytes = bs.toByteArray() - - MemoryStream(contentBytes).use { ms -> - ZipInputStream(ms).use { zip -> - var entryCount = 0 - while (zip.nextEntry != null) { - entryCount++ - } - if (entryCount > 1) { - logger.debug("Expected the zip to contain only one file") - } - } - } + val depotManifest = DepotManifest.deserialize(ByteArrayInputStream(manifestData)) - // Decompress the zipped manifest data - MemoryStream(contentBytes).use { ms -> - ZipInputStream(ms).use { zip -> - zip.nextEntry - DepotManifest.deserialize(zip) - } + if (depotKey != null) { + depotManifest.decryptFilenames(depotKey) } + + depotManifest } } } - - depotKey?.let { key -> - // if we have the depot key, decrypt the manifest filenames - depotManifest.decryptFilenames(key) - } - - depotManifest + } catch (e: Exception) { + logger.error("Failed to download manifest ${request.url}: ${e.message}", e) + throw e } } - /** - * Java Compat: - * Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided. - * @param depotId The id of the depot being accessed. - * @param manifestId The unique identifier of the manifest to be downloaded. - * @param manifestRequestCode The manifest request code for the manifest that is being downloaded. - * @param server The content server to connect to. - * @param depotKey The depot decryption key for the depot that will be downloaded. - * This is used for decrypting filenames (if needed) in depot manifests. - * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. - * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. - * @return A [DepotManifest] instance that contains information about the files present within a depot. - * @exception IllegalArgumentException [server] was null. - * @exception IOException A network error occurred when performing the request. - * @exception SteamKitWebRequestException A network error occurred when performing the request. - * @exception DataFormatException When the data received is not as expected - */ - @JvmOverloads - fun downloadManifestFuture( - depotId: Int, - manifestId: Long, - manifestRequestCode: Long, - server: Server, - depotKey: ByteArray? = null, - proxyServer: Server? = null, - cdnAuthToken: String? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadManifest( - depotId = depotId, - manifestId = manifestId, - manifestRequestCode = manifestRequestCode.toULong(), - server = server, - depotKey = depotKey, - proxyServer = proxyServer, - cdnAuthToken = cdnAuthToken, - ) - } - /** * Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided. * This function will also validate the length of the downloaded chunk with the value of [ChunkData.compressedLength], @@ -252,7 +214,7 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): Int { + ): Int = withContext(Dispatchers.IO) { require(chunk.chunkID != null) { "Chunk must have a ChunkID." } if (depotKey == null) { @@ -268,82 +230,118 @@ class Client(steamClient: SteamClient) : Closeable { val chunkID = Strings.toHex(chunk.chunkID) val url = "depot/$depotId/chunk/$chunkID" - val request: Request = if (ClientLancache.useLanCacheServer) { - ClientLancache.buildLancacheRequest(server, url, cdnAuthToken) + val request = if (ClientLancache.useLanCacheServer) { + ClientLancache.buildLancacheRequest(server = server, command = url, query = cdnAuthToken) } else { - Request.Builder().url(buildCommand(server, url, cdnAuthToken, proxyServer)).build() + val url = buildCommand(server = server, command = url, query = cdnAuthToken, proxyServer = proxyServer) + Request.Builder() + .url(url) + .build() } - withTimeout(requestTimeout) { - httpClient.newCall(request).execute() - }.use { response -> - if (!response.isSuccessful) { - throw SteamKitWebRequestException( - "Response status code does not indicate success: ${response.code} (${response.message})", - response - ) + try { + val response = withTimeout(requestTimeout) { + httpClient.newCall(request).executeAsync() } - var contentLength = chunk.compressedLength + response.use { resp -> + if (!resp.isSuccessful) { + throw SteamKitWebRequestException( + "Response status code does not indicate success: ${resp.code} (${resp.message})", + resp + ) + } - response.header("Content-Length")?.toLongOrNull()?.let { responseContentLength -> - contentLength = responseContentLength.toInt() + val contentLength = resp.body.contentLength().toInt() - // assert that lengths match only if the chunk has a length assigned. - if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { - throw IllegalStateException("Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})") + if (contentLength == 0) { + chunk.compressedLength } - } ?: run { - if (contentLength > 0) { - logger.debug("Response does not have Content-Length, falling back to chunk.compressedLength.") - } else { + + // Validate content length + if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { throw SteamKitWebRequestException( - "Response does not have Content-Length and chunk.compressedLength is not set.", - response + "Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})" ) } - } - // If no depot key is provided, stream into the destination buffer without renting - if (depotKey == null) { - val bytesRead = withTimeout(responseBodyTimeout) { - response.body.byteStream().use { input -> - input.readNBytesCompat(destination, 0, contentLength) - } + val responseBody = resp.body.bytes() + + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") } - if (bytesRead != contentLength) { - throw IOException("Length mismatch after downloading depot chunk! (was $bytesRead, but should be $contentLength)") + if (responseBody.size != contentLength) { + throw SteamKitWebRequestException( + "Length mismatch after downloading depot chunk! (was ${responseBody.size}, but should be $contentLength)" + ) } - return contentLength - } + if (depotKey == null) { + System.arraycopy(responseBody, 0, destination, 0, contentLength) + return@withContext contentLength + } - // We have to stream into a temporary buffer because a decryption will need to be performed - val buffer = ByteArray(contentLength) + return@withContext DepotChunk.process(chunk, responseBody, destination, depotKey) + } + } catch (e: Exception) { + logger.error("Failed to download a depot chunk ${request.url}: ${e.message}", e) + throw e + } + } - try { - val bytesRead = withTimeout(responseBodyTimeout) { - response.body.byteStream().use { input -> - input.readNBytesCompat(buffer, 0, contentLength) - } - } + // region Java Compatibility - if (bytesRead != contentLength) { - throw IOException("Length mismatch after downloading encrypted depot chunk! (was $bytesRead, but should be $contentLength)") - } + /** + * Java-compatible version of downloadManifest that returns a CompletableFuture. + * Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided. + * @param depotId The id of the depot being accessed. + * @param manifestId The unique identifier of the manifest to be downloaded. + * @param manifestRequestCode The manifest request code for the manifest that is being downloaded. + * @param server The content server to connect to. + * @param depotKey The depot decryption key for the depot that will be downloaded. + * This is used for decrypting filenames (if needed) in depot manifests. + * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. + * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. + * @return A CompletableFuture that will complete with a [DepotManifest] instance that contains information about the files present within a depot. + * @exception IllegalArgumentException [server] was null. + * @exception IOException A network error occurred when performing the request. + * @exception SteamKitWebRequestException A network error occurred when performing the request. + */ + @JvmOverloads + fun downloadManifestFuture( + depotId: Int, + manifestId: Long, + manifestRequestCode: Long, + server: Server, + depotKey: ByteArray? = null, + proxyServer: Server? = null, + cdnAuthToken: String? = null, + ): CompletableFuture { + val future = CompletableFuture() - // process the chunk immediately - return DepotChunk.process(chunk, buffer, destination, depotKey) - } catch (ex: Exception) { - logger.error("Failed to download a depot chunk ${request.url}", ex) - throw ex + scope.launch { + try { + val result = downloadManifest( + depotId = depotId, + manifestId = manifestId, + manifestRequestCode = manifestRequestCode.toULong(), + server = server, + depotKey = depotKey, + proxyServer = proxyServer, + cdnAuthToken = cdnAuthToken + ) + future.complete(result) + } catch (e: Exception) { + future.completeExceptionally(e) } } + + return future } /** - * Java Compat: + * Java-compatible version of downloadDepotChunk that returns a CompletableFuture. * Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided. * This function will also validate the length of the downloaded chunk with the value of [ChunkData.compressedLength], * if it has been assigned a value. @@ -357,7 +355,7 @@ class Client(steamClient: SteamClient) : Closeable { * This is used to process the chunk data. * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. - * @return The total number of bytes written to [destination]. + * @return A CompletableFuture that will complete with the total number of bytes written to [destination]. * @exception IllegalArgumentException Thrown if the chunk's [ChunkData.chunkID] was null or if the [destination] buffer is too small. * @exception IllegalStateException Thrown if the downloaded data does not match the expected length. * @exception SteamKitWebRequestException A network error occurred when performing the request. @@ -371,15 +369,28 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadDepotChunk( - depotId = depotId, - chunk = chunk, - server = server, - destination = destination, - depotKey = depotKey, - proxyServer = proxyServer, - cdnAuthToken = cdnAuthToken, - ) + ): CompletableFuture { + val future = CompletableFuture() + + scope.launch { + try { + val bytesWritten = downloadDepotChunk( + depotId = depotId, + chunk = chunk, + server = server, + destination = destination, + depotKey = depotKey, + proxyServer = proxyServer, + cdnAuthToken = cdnAuthToken + ) + future.complete(bytesWritten) + } catch (e: Exception) { + future.completeExceptionally(e) + } + } + + return future } + + // endregion } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt index eb3238ae..0eb9e699 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt @@ -1,80 +1,126 @@ package `in`.dragonbra.javasteam.steam.cdn -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.future.future -import okhttp3.HttpUrl +import kotlinx.coroutines.withContext +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Request import java.net.Inet4Address import java.net.Inet6Address import java.net.InetAddress -import java.util.concurrent.* /** * @author Lossy * @since 31/12/2024 */ -@Suppress("unused") object ClientLancache { - - private const val TRIGGER_DOMAIN: String = "lancache.steamcontent.com" - /** - * When set to true, will attempt to download from a Lancache instance on the LAN rather than going out to Steam's CDNs. + * When set to true, will attempt to download from a Lancache instance on the LAN + * rather than going out to Steam's CDNs. */ var useLanCacheServer: Boolean = false + private set + + private const val TRIGGER_DOMAIN = "lancache.steamcontent.com" /** - * Attempts to automatically resolve a Lancache on the local network. - * If detected, SteamKit will route all downloads through the cache rather than through Steam's CDN. + * Attempts to automatically resolve a Lancache on the local network. If detected, + * SteamKit will route all downloads through the cache rather than through Steam's CDN. * Will try to detect the Lancache through the poisoned DNS entry for lancache.steamcontent.com * - * This is a modified version from the original source : + * This is a modified version from the original source: * https://github.com/tpill90/lancache-prefill-common/blob/main/dotnet/LancacheIpResolver.cs */ - @JvmStatic - @JvmOverloads - fun detectLancacheServer(dispatcher: CoroutineDispatcher = Dispatchers.IO): CompletableFuture = - CoroutineScope(dispatcher).future { - val ipAddresses = InetAddress.getAllByName(TRIGGER_DOMAIN) - .filter { it is Inet4Address || it is Inet6Address } + suspend fun detectLancacheServer() { + withContext(Dispatchers.IO) { + try { + val addresses = InetAddress.getAllByName(TRIGGER_DOMAIN) + val ipAddresses = addresses.filter { address -> + address is Inet4Address || address is Inet6Address + } - useLanCacheServer = ipAddresses.any { isPrivateAddress(it) } + useLanCacheServer = ipAddresses.any { isPrivateAddress(it) } + } catch (_: Exception) { + useLanCacheServer = false + } } + } /** * Determines if an IP address is a private address, as specified in RFC1918 + * * @param toTest The IP address that will be tested - * @return true if the IP is a private address, false if it isn't private + * @return Returns true if the IP is a private address, false if it isn't private */ @JvmStatic fun isPrivateAddress(toTest: InetAddress): Boolean { - if (toTest.isLoopbackAddress) return true + if (toTest.isLoopbackAddress) { + return true + } + val bytes = toTest.address - return when (toTest) { - is Inet4Address -> when (bytes[0].toInt() and 0xFF) { + + // IPv4 + if (toTest is Inet4Address) { + // Convert signed byte to unsigned for comparison + val firstOctet = bytes[0].toInt() and 0xFF + + return when (firstOctet) { 10 -> true - 172 -> (bytes[1].toInt() and 0xFF) in 16..31 - 192 -> bytes[1].toInt() and 0xFF == 168 + 172 -> { + val secondOctet = bytes[1].toInt() and 0xFF + secondOctet in 16..<32 + } + + 192 -> { + val secondOctet = bytes[1].toInt() and 0xFF + secondOctet == 168 + } + else -> false } - is Inet6Address -> (bytes[0].toInt() and 0xFE) == 0xFC || toTest.isLinkLocalAddress - else -> false } + + // IPv6 + if (toTest is Inet6Address) { + // Check for Unique Local Address (fc00::/7) and link-local + val firstByte = bytes[0].toInt() and 0xFF + return (firstByte and 0xFE) == 0xFC || toTest.isLinkLocalAddress + } + + return false } - fun buildLancacheRequest(server: Server, command: String, query: String?): Request = Request.Builder() - .url( - HttpUrl.Builder() - .scheme("http") - .host("lancache.steamcontent.com") - .port(80) - .addPathSegments(command) - .query(query) - .build() - ) - .header("Host", server.host) - .header("User-Agent", "Valve/Steam HTTP Client 1.0") - .build() + /** + * Builds an HTTP request for Lancache with proper headers + * + * @param server The server to route the request through + * @param command The API command/path + * @param query Optional query parameters + * @return OkHttp Request object configured for Lancache + */ + fun buildLancacheRequest(server: Server, command: String, query: String? = null): Request { + val urlBuilder = "http://lancache.steamcontent.com:80".toHttpUrl().newBuilder() + .addPathSegments(command.trimStart('/')) + + query?.let { queryString -> + if (queryString.isNotEmpty()) { + val params = queryString.split("&") + for (param in params) { + val keyValue = param.split("=", limit = 2) + if (keyValue.size == 2) { + urlBuilder.addQueryParameter(keyValue[0], keyValue[1]) + } else if (keyValue.size == 1 && keyValue[0].isNotEmpty()) { + urlBuilder.addQueryParameter(keyValue[0], "") + } + } + } + } + + return Request.Builder() + .url(urlBuilder.build()) + .header("Host", server.host ?: "") + // User agent must match the Steam client in order for Lancache to correctly identify and cache Valve's CDN content + .header("User-Agent", "Valve/Steam HTTP Client 1.0") + .build() + } } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt deleted file mode 100644 index ee377dd2..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt +++ /dev/null @@ -1,142 +0,0 @@ -package `in`.dragonbra.javasteam.steam.cdn - -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Job -import kotlinx.coroutines.async -import kotlinx.coroutines.cancel -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import java.util.concurrent.ConcurrentLinkedDeque -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -/** - * [ClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed - */ -class ClientPool(internal val steamClient: SteamClient, private val appId: Int, private val parentScope: CoroutineScope) { - - companion object { - private const val SERVER_ENDPOINT_MIN_SIZE = 8 - } - - val cdnClient: Client = Client(steamClient) - - var proxyServer: Server? = null - private set - - private val activeConnectionPool = ConcurrentLinkedDeque() - - private val availableServerEndpoints = ConcurrentLinkedQueue() - - private val populatePoolEvent = CountDownLatch(1) - - private val monitorJob: Job - - private val logger: Logger = LogManager.getLogger() - - init { - monitorJob = parentScope.launch { connectionPoolMonitor().await() } - } - - fun shutdown() { - monitorJob.cancel() - } - - private fun fetchBootstrapServerList(): Deferred?> = parentScope.async { - return@async try { - steamClient.getHandler(SteamContent::class.java)?.getServersForSteamPipe(parentScope = parentScope)?.await() - } catch (ex: Exception) { - logger.error("Failed to retrieve content server list", ex) - - null - } - } - - private fun connectionPoolMonitor() = parentScope.async { - var didPopulate = false - - while (isActive) { - populatePoolEvent.await(1, TimeUnit.SECONDS) - - if (availableServerEndpoints.size < SERVER_ENDPOINT_MIN_SIZE && steamClient.isConnected) { - val servers = fetchBootstrapServerList().await() - - if (servers.isNullOrEmpty()) { - logger.error("Servers is empty or null, exiting connection pool monitor") - parentScope.cancel() - return@async - } - - proxyServer = servers.find { it.useAsProxy } - - val weightedCdnServers = servers - .filter { server -> - val isEligibleForApp = server.allowedAppIds.isEmpty() || appId in server.allowedAppIds - isEligibleForApp && (server.type == "SteamCache" || server.type == "CDN") - } - .sortedBy { it.weightedLoad } - - for (server in weightedCdnServers) { - repeat(server.numEntries) { - availableServerEndpoints.offer(server) - } - } - - didPopulate = true - } else if (availableServerEndpoints.isEmpty() && !steamClient.isConnected && didPopulate) { - logger.error("Available server endpoints is empty and steam is not connected, exiting connection pool monitor") - - parentScope.cancel() - - return@async - } - } - } - - private fun buildConnection(): Deferred = parentScope.async { - return@async try { - if (availableServerEndpoints.size < SERVER_ENDPOINT_MIN_SIZE) { - populatePoolEvent.countDown() - } - - var output: Server? = null - - while (isActive && availableServerEndpoints.poll().also { output = it } == null) { - delay(1000) - } - - output - } catch (e: Exception) { - logger.error("Failed to build connection", e) - - null - } - } - - internal fun getConnection(): Deferred = parentScope.async { - return@async try { - val server = activeConnectionPool.poll() ?: buildConnection().await() - server - } catch (e: Exception) { - logger.error("Failed to get/build connection", e) - - null - } - } - - internal fun returnConnection(server: Server?) { - server?.let { activeConnectionPool.push(it) } - } - - @Suppress("unused") - internal fun returnBrokenConnection(server: Server?) { - // Broken connections are not returned to the pool - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt index 6285449f..db744e3c 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt @@ -51,15 +51,16 @@ object DepotChunk { require(iv.size == ivBytesRead) { "Failed to decrypt depot chunk iv (${iv.size} != $ivBytesRead)" } + val aes = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) + aes.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) + // With CBC and padding, the decrypted size will always be smaller val buffer = ByteArray(data.size - iv.size) - val cbcCipher = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) - cbcCipher.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) val writtenDecompressed: Int try { - val written = cbcCipher.doFinal(data, iv.size, data.size - iv.size, buffer) + val written = aes.doFinal(data, iv.size, data.size - iv.size, buffer) // Per SK: // Steam client checks for like 20 bytes for pkzip, and 22 bytes for vzip, diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt index 7b9aa104..82a2035f 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt @@ -1,24 +1,35 @@ package `in`.dragonbra.javasteam.steam.cdn +import java.net.InetSocketAddress + /** * Represents a single Steam3 'Steampipe' content server. */ -class Server @JvmOverloads constructor( - protocol: ConnectionProtocol = ConnectionProtocol.HTTP, - host: String, - vHost: String, - port: Int, - type: String? = null, - sourceID: Int = 0, - cellID: Int = 0, - load: Int = 0, - weightedLoad: Float = 0f, - numEntries: Int = 0, - steamChinaOnly: Boolean = false, - useAsProxy: Boolean = false, - proxyRequestPathTemplate: String? = null, - allowedAppIds: IntArray = IntArray(0), -) { +class Server { + + companion object { + /** + * Creates a Server from an InetSocketAddress. + */ + @JvmStatic + fun fromInetSocketAddress(endPoint: InetSocketAddress): Server = Server().apply { + protocol = if (endPoint.port == 443) ConnectionProtocol.HTTPS else ConnectionProtocol.HTTP + host = endPoint.address.hostAddress + vHost = endPoint.address.hostAddress + port = endPoint.port + } + + /** + * Creates a Server from hostname and port. + */ + @JvmStatic + fun fromHostAndPort(hostname: String, port: Int): Server = Server().apply { + protocol = if (port == 443) ConnectionProtocol.HTTPS else ConnectionProtocol.HTTP + host = hostname + vHost = hostname + this.port = port + } + } /** * The protocol used to connect to this server @@ -38,86 +49,85 @@ class Server @JvmOverloads constructor( /** * Gets the supported connection protocol of the server. */ - var protocol = protocol + var protocol: ConnectionProtocol = ConnectionProtocol.HTTP internal set /** * Gets the hostname of the server. */ - var host = host + var host: String? = null internal set /** * Gets the virtual hostname of the server. */ - var vHost = vHost + var vHost: String? = null internal set /** * Gets the port of the server. */ - var port = port + var port: Int = 0 internal set /** * Gets the type of the server. */ - var type = type + var type: String? = null internal set /** * Gets the SourceID this server belongs to. */ - @Suppress("unused") - var sourceID = sourceID + var sourceId: Int = 0 internal set /** * Gets the CellID this server belongs to. */ - var cellID = cellID + var cellId: Int = 0 internal set /** * Gets the load value associated with this server. */ - var load = load + var load: Int = 0 internal set /** * Gets the weighted load. */ - var weightedLoad = weightedLoad + var weightedLoad: Float = 0F internal set /** * Gets the number of entries this server is worth. */ - var numEntries = numEntries + var numEntries: Int = 0 internal set /** * Gets the flag whether this server is for Steam China only. */ - var steamChinaOnly = steamChinaOnly + var steamChinaOnly: Boolean = false internal set /** * Gets the download proxy status. */ - var useAsProxy = useAsProxy + var useAsProxy: Boolean = false internal set /** * Gets the transformation template applied to request paths. */ - var proxyRequestPathTemplate = proxyRequestPathTemplate + var proxyRequestPathTemplate: String? = null internal set /** * Gets the list of app ids this server can be used with. */ - var allowedAppIds = allowedAppIds + var allowedAppIds: IntArray = intArrayOf() internal set /** diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt deleted file mode 100644 index 2004de2b..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.ChunkData - -data class ChunkMatch( - val oldChunk: ChunkData, - val newChunk: ChunkData, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt deleted file mode 100644 index 2a638d65..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt +++ /dev/null @@ -1,724 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.enums.EDepotFileFlag -import `in`.dragonbra.javasteam.enums.EResult -import `in`.dragonbra.javasteam.steam.cdn.ClientPool -import `in`.dragonbra.javasteam.steam.cdn.Server -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest -import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient -import `in`.dragonbra.javasteam.types.ChunkData -import `in`.dragonbra.javasteam.types.DepotManifest -import `in`.dragonbra.javasteam.types.FileData -import `in`.dragonbra.javasteam.types.KeyValue -import `in`.dragonbra.javasteam.util.Adler32 -import `in`.dragonbra.javasteam.util.SteamKitWebRequestException -import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.Utils -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.future.future -import kotlinx.coroutines.isActive -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit -import java.io.File -import java.io.FileInputStream -import java.io.FileOutputStream -import java.io.RandomAccessFile -import java.nio.ByteBuffer -import java.nio.file.Paths -import java.time.Instant -import java.time.temporal.ChronoUnit -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentLinkedQueue - -@Suppress("unused", "SpellCheckingInspection") -class ContentDownloader(val steamClient: SteamClient) { - - companion object { - private const val HTTP_UNAUTHORIZED = 401 - private const val HTTP_FORBIDDEN = 403 - private const val HTTP_NOT_FOUND = 404 - private const val SERVICE_UNAVAILABLE = 503 - - internal const val INVALID_APP_ID = Int.MAX_VALUE - internal const val INVALID_MANIFEST_ID = Long.MAX_VALUE - - private val logger: Logger = LogManager.getLogger(ContentDownloader::class.java) - } - - private val defaultScope = CoroutineScope(Dispatchers.IO) - - private fun requestDepotKey( - appId: Int, - depotId: Int, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.getDepotDecryptionKey(depotId, appId)?.await() - - return@async Pair(callback?.result ?: EResult.Fail, callback?.depotKey) - } - - private fun getDepotManifestId( - app: PICSProductInfo, - depotId: Int, - branchId: String, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val depot = app.keyValues["depots"][depotId.toString()] - if (depot == KeyValue.INVALID) { - logger.error("Could not find depot $depotId of ${app.id}") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val manifest = depot["manifests"][branchId] - if (manifest != KeyValue.INVALID) { - return@async Pair(app.id, manifest["gid"].asLong()) - } - - val depotFromApp = depot["depotfromapp"].asInteger(INVALID_APP_ID) - if (depotFromApp == app.id || depotFromApp == INVALID_APP_ID) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val innerApp = getAppInfo(depotFromApp, parentScope).await() - if (innerApp == null) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - return@async getDepotManifestId(innerApp, depotId, branchId, parentScope).await() - } - - private fun getAppDirName(app: PICSProductInfo): String { - val installDirKeyValue = app.keyValues["config"]["installdir"] - - return if (installDirKeyValue != KeyValue.INVALID) installDirKeyValue.value!! else app.id.toString() - } - - private fun getAppInfo( - appId: Int, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.picsGetProductInfo(PICSRequest(appId))?.await() - val apps = callback?.results?.flatMap { it.apps.values } - - if (apps.isNullOrEmpty()) { - logger.error("Received empty apps list in PICSProductInfo response for $appId") - return@async null - } - - if (apps.size > 1) { - logger.debug("Received ${apps.size} apps from PICSProductInfo for $appId, using first result") - } - - return@async apps.first() - } - - /** - * Kotlin coroutines version - */ - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope = defaultScope, - ): Deferred = parentScope.async { - downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = onDownloadProgress, - scope = parentScope - ) - } - - /** - * Java-friendly version that returns a CompletableFuture - */ - @JvmOverloads - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - progressCallback: ProgressCallback? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = progressCallback?.let { callback -> { progress -> callback.onProgress(progress) } }, - scope = defaultScope - ) - } - - private suspend fun downloadAppInternal( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - scope: CoroutineScope, - ): Boolean { - if (!scope.isActive) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - return false - } - - val cdnPool = ClientPool(steamClient, appId, scope) - - val shiftedAppId: Int - val manifestId: Long - val appInfo = getAppInfo(appId, scope).await() - - if (appInfo == null) { - logger.error("Could not retrieve PICSProductInfo of $appId") - return false - } - - getDepotManifestId(appInfo, depotId, branch, scope).await().apply { - shiftedAppId = first - manifestId = second - } - - val depotKeyResult = requestDepotKey(shiftedAppId, depotId, scope).await() - - if (depotKeyResult.first != EResult.OK || depotKeyResult.second == null) { - logger.error("Depot key request for $appId failed with result ${depotKeyResult.first}") - return false - } - - val depotKey = depotKeyResult.second!! - - var newProtoManifest = steamClient.configuration.depotManifestProvider.fetchManifest(depotId, manifestId) - var oldProtoManifest = steamClient.configuration.depotManifestProvider.fetchLatestManifest(depotId) - - if (oldProtoManifest?.manifestGID == manifestId) { - oldProtoManifest = null - } - - // In case we have an early exit, this will force equiv of verifyall next run. - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, INVALID_MANIFEST_ID) - - try { - if (newProtoManifest == null) { - newProtoManifest = - downloadFilesManifestOf(shiftedAppId, depotId, manifestId, branch, depotKey, cdnPool, scope).await() - } else { - logger.debug("Already have manifest $manifestId for depot $depotId.") - } - - if (newProtoManifest == null) { - logger.error("Failed to retrieve files manifest for app: $shiftedAppId depot: $depotId manifest: $manifestId branch: $branch") - return false - } - - if (!scope.isActive) { - return false - } - - val downloadCounter = GlobalDownloadCounter() - val installDir = Paths.get(installPath, getAppDirName(appInfo)).toString() - val stagingDir = Paths.get(stagingPath, getAppDirName(appInfo)).toString() - val depotFileData = DepotFilesData( - depotDownloadInfo = DepotDownloadInfo(depotId, shiftedAppId, manifestId, branch, installDir, depotKey), - depotCounter = DepotDownloadCounter( - completeDownloadSize = newProtoManifest.totalUncompressedSize - ), - stagingDir = stagingDir, - manifest = newProtoManifest, - previousManifest = oldProtoManifest - ) - - downloadDepotFiles(cdnPool, downloadCounter, depotFileData, maxDownloads, onDownloadProgress, scope).await() - - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, manifestId) - - cdnPool.shutdown() - - // delete the staging directory of this app - File(stagingDir).deleteRecursively() - - logger.debug( - "Depot $depotId - Downloaded ${depotFileData.depotCounter.depotBytesCompressed} " + - "bytes (${depotFileData.depotCounter.depotBytesUncompressed} bytes uncompressed)" - ) - - return true - } catch (e: CancellationException) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - - return false - } catch (e: Exception) { - logger.error("Error occurred while downloading app $shiftedAppId", e) - - return false - } - } - - private fun downloadDepotFiles( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - maxDownloads: Int, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!parentScope.isActive) { - return@async - } - - depotFilesData.manifest.files.forEach { file -> - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - if (file.flags.contains(EDepotFileFlag.Directory)) { - File(fileFinalPath).mkdirs() - File(fileStagingPath).mkdirs() - } else { - // Some manifests don't explicitly include all necessary directories - File(fileFinalPath).parentFile.mkdirs() - File(fileStagingPath).parentFile.mkdirs() - } - } - - logger.debug("Downloading depot ${depotFilesData.depotDownloadInfo.depotId}") - - val files = depotFilesData.manifest.files.filter { !it.flags.contains(EDepotFileFlag.Directory) }.toTypedArray() - val networkChunkQueue = ConcurrentLinkedQueue>() - - val downloadSemaphore = Semaphore(maxDownloads) - files.map { file -> - async { - downloadSemaphore.withPermit { - downloadDepotFile(depotFilesData, file, networkChunkQueue, onDownloadProgress, parentScope).await() - } - } - }.awaitAll() - - networkChunkQueue.map { (fileStreamData, fileData, chunk) -> - async { - downloadSemaphore.withPermit { - downloadSteam3DepotFileChunk( - cdnPool = cdnPool, - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = fileData, - fileStreamData = fileStreamData, - chunk = chunk, - onDownloadProgress = onDownloadProgress, - parentScope = parentScope - ).await() - } - } - }.awaitAll() - - // Check for deleted files if updating the depot. - depotFilesData.previousManifest?.apply { - val previousFilteredFiles = files.asSequence().map { it.fileName }.toMutableSet() - - // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names - previousFilteredFiles.removeAll(depotFilesData.manifest.files.map { it.fileName }.toSet()) - - for (existingFileName in previousFilteredFiles) { - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, existingFileName).toString() - - if (!File(fileFinalPath).exists()) { - continue - } - - File(fileFinalPath).delete() - logger.debug("Deleted $fileFinalPath") - } - } - } - - private fun downloadDepotFile( - depotFilesData: DepotFilesData, - file: FileData, - networkChunkQueue: ConcurrentLinkedQueue>, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depotDownloadCounter = depotFilesData.depotCounter - val oldManifestFile = depotFilesData.previousManifest?.files?.find { it.fileName == file.fileName } - - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - // This may still exist if the previous run exited before cleanup - File(fileStagingPath).takeIf { it.exists() }?.delete() - - val neededChunks: MutableList - val fi = File(fileFinalPath) - val fileDidExist = fi.exists() - - if (!fileDidExist) { - // create new file. need all chunks - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - } - - neededChunks = file.chunks.toMutableList() - } else { - // open existing - if (oldManifestFile != null) { - neededChunks = mutableListOf() - - val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) - if (!hashMatches) { - logger.debug("Validating $fileFinalPath") - - val matchingChunks = mutableListOf() - - for (chunk in file.chunks) { - val oldChunk = oldManifestFile.chunks.find { it.chunkID.contentEquals(chunk.chunkID) } - if (oldChunk != null) { - matchingChunks.add(ChunkMatch(oldChunk, chunk)) - } else { - neededChunks.add(chunk) - } - } - - val orderedChunks = matchingChunks.sortedBy { it.oldChunk.offset } - - val copyChunks = mutableListOf() - - FileInputStream(fileFinalPath).use { fsOld -> - for (match in orderedChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - val adler = Adler32.calculate(tmp) - if (adler != match.oldChunk.checksum) { - neededChunks.add(match.newChunk) - } else { - copyChunks.add(match) - } - } - } - - if (neededChunks.isNotEmpty()) { - File(fileFinalPath).renameTo(File(fileStagingPath)) - - FileInputStream(fileStagingPath).use { fsOld -> - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - - for (match in copyChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - fs.channel.position(match.newChunk.offset) - fs.write(tmp) - } - } - } - - File(fileStagingPath).delete() - } - } - } else { - // No old manifest or file not in old manifest. We must validate. - RandomAccessFile(fileFinalPath, "rw").use { fs -> - if (fi.length() != file.totalSize) { - fs.channel.truncate(file.totalSize) - } - - logger.debug("Validating $fileFinalPath") - neededChunks = Utils.validateSteam3FileChecksums( - fs, - file.chunks.sortedBy { it.offset }.toTypedArray() - ) - } - } - - if (neededChunks.isEmpty()) { - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += file.totalSize - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - - return@async - } - - val sizeOnDisk = file.totalSize - neededChunks.sumOf { it.uncompressedLength.toLong() } - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += sizeOnDisk - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - } - - val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) - if (fileIsExecutable && - (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) - ) { - File(fileFinalPath).setExecutable(true) - } else if (!fileIsExecutable && oldManifestFile != null && oldManifestFile.flags.contains(EDepotFileFlag.Executable)) { - File(fileFinalPath).setExecutable(false) - } - - val fileStreamData = FileStreamData( - fileStream = null, - fileLock = Semaphore(1), - chunksToDownload = neededChunks.size - ) - - for (chunk in neededChunks) { - networkChunkQueue.add(Triple(fileStreamData, file, chunk)) - } - } - - private fun downloadSteam3DepotFileChunk( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - file: FileData, - fileStreamData: FileStreamData, - chunk: ChunkData, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depot = depotFilesData.depotDownloadInfo - val depotDownloadCounter = depotFilesData.depotCounter - - val chunkID = Strings.toHex(chunk.chunkID) - - var outputChunkData = ByteArray(chunk.uncompressedLength) - var writtenBytes = 0 - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - outputChunkData = ByteArray(chunk.uncompressedLength) - writtenBytes = cdnPool.cdnClient.downloadDepotChunk( - depotId = depot.depotId, - chunk = chunk, - server = connection!!, - destination = outputChunkData, - depotKey = depot.depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - when (e.statusCode) { - HTTP_UNAUTHORIZED, HTTP_FORBIDDEN -> { - logger.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") - break - } - - else -> logger.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") - } - } catch (e: NoClassDefFoundError) { - // Zstd is a 'compileOnly' dependency. - throw CancellationException(e.message) - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered unexpected error downloading chunk $chunkID", e) - } - } while (isActive && writtenBytes <= 0) - - if (writtenBytes <= 0) { - logger.error("Failed to find any server with chunk $chunkID for depot ${depot.depotId}. Aborting.") - throw CancellationException("Failed to download chunk") - } - - try { - fileStreamData.fileLock.acquire() - - if (fileStreamData.fileStream == null) { - val fileFinalPath = Paths.get(depot.installDir, file.fileName).toString() - val randomAccessFile = RandomAccessFile(fileFinalPath, "rw") - fileStreamData.fileStream = randomAccessFile.channel - } - - fileStreamData.fileStream?.position(chunk.offset) - fileStreamData.fileStream?.write(ByteBuffer.wrap(outputChunkData, 0, writtenBytes)) - } finally { - fileStreamData.fileLock.release() - } - - val remainingChunks = synchronized(fileStreamData) { - --fileStreamData.chunksToDownload - } - if (remainingChunks <= 0) { - fileStreamData.fileStream?.close() - } - - var sizeDownloaded: Long - synchronized(depotDownloadCounter) { - sizeDownloaded = depotDownloadCounter.sizeDownloaded + outputChunkData.size - depotDownloadCounter.sizeDownloaded = sizeDownloaded - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - - onDownloadProgress?.invoke( - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - ) - } - - private fun downloadFilesManifestOf( - appId: Int, - depotId: Int, - manifestId: Long, - branch: String, - depotKey: ByteArray, - cdnPool: ClientPool, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - if (!isActive) { - return@async null - } - - var depotManifest: DepotManifest? = null - var manifestRequestCode = 0UL - var manifestRequestCodeExpiration = Instant.MIN - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - if (connection == null) continue - - val now = Instant.now() - - // In order to download this manifest, we need the current manifest request code - // The manifest request code is only valid for a specific period of time - if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { - val steamContent = steamClient.getHandler(SteamContent::class.java)!! - - manifestRequestCode = steamContent.getManifestRequestCode( - depotId = depotId, - appId = appId, - manifestId = manifestId, - branch = branch, - parentScope = parentScope - ).await() - - // This code will hopefully be valid for one period following the issuing period - manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) - - // If we could not get the manifest code, this is a fatal error - if (manifestRequestCode == 0UL) { - throw CancellationException("No manifest request code was returned for manifest $manifestId in depot $depotId") - } - } - - depotManifest = cdnPool.cdnClient.downloadManifest( - depotId = depotId, - manifestId = manifestId, - manifestRequestCode = manifestRequestCode, - server = connection, - depotKey = depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: CancellationException) { - logger.error("Connection timeout downloading depot manifest $depotId $manifestId") - - return@async null - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - val statusName = when (e.statusCode) { - HTTP_UNAUTHORIZED -> HTTP_UNAUTHORIZED::class.java.name - HTTP_FORBIDDEN -> HTTP_FORBIDDEN::class.java.name - HTTP_NOT_FOUND -> HTTP_NOT_FOUND::class.java.name - SERVICE_UNAVAILABLE -> SERVICE_UNAVAILABLE::class.java.name - else -> null - } - - logger.error( - "Downloading of manifest $manifestId failed for depot $depotId with " + - if (statusName != null) { - "response of $statusName(${e.statusCode})" - } else { - "status code of ${e.statusCode}" - } - ) - - return@async null - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered error downloading manifest for depot $depotId $manifestId", e) - - return@async null - } - } while (isActive && depotManifest == null) - - if (depotManifest == null) { - throw CancellationException("Unable to download manifest $manifestId for depot $depotId") - } - - val newProtoManifest = depotManifest - steamClient.configuration.depotManifestProvider.updateManifest(newProtoManifest) - - return@async newProtoManifest - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt deleted file mode 100644 index dd31905a..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -data class DepotDownloadCounter( - var completeDownloadSize: Long = 0, - var sizeDownloaded: Long = 0, - var depotBytesCompressed: Long = 0, - var depotBytesUncompressed: Long = 0, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt deleted file mode 100644 index ff31b973..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt +++ /dev/null @@ -1,11 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -@Suppress("ArrayInDataClass") -data class DepotDownloadInfo( - val depotId: Int, - val appId: Int, - val manifestId: Long, - val branch: String, - val installDir: String, - val depotKey: ByteArray?, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt deleted file mode 100644 index f2e01726..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt +++ /dev/null @@ -1,11 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -data class DepotFilesData( - val depotDownloadInfo: DepotDownloadInfo, - val depotCounter: DepotDownloadCounter, - val stagingDir: String, - val manifest: DepotManifest, - val previousManifest: DepotManifest?, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt deleted file mode 100644 index ec7201b6..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt +++ /dev/null @@ -1,210 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import `in`.dragonbra.javasteam.util.stream.MemoryStream -import java.io.ByteArrayOutputStream -import java.io.File -import java.io.IOException -import java.nio.ByteBuffer -import java.nio.file.Files -import java.nio.file.Path -import java.util.zip.CRC32 -import java.util.zip.ZipEntry -import java.util.zip.ZipInputStream -import java.util.zip.ZipOutputStream - -/** - * Depot manifest provider that stores depot manifests in a zip file. - * @constructor Instantiates a [FileManifestProvider] object. - * @param file the file that will store the depot manifests - * - * @author Oxters - * @since 2024-11-07 - */ -@Suppress("unused") -class FileManifestProvider(private val file: Path) : IManifestProvider { - - /** - * Instantiates a [FileManifestProvider] object. - * @param file the file that will store the depot manifests. - */ - constructor(file: File) : this(file.toPath()) - - /** - * Instantiates a [FileManifestProvider] object. - * @param filename the filename that will store the depot manifests. - */ - constructor(filename: String) : this(Path.of(filename)) - - init { - require(file.fileName.toString().isNotBlank()) { "FileName must not be blank" } - } - - companion object { - private val logger: Logger = LogManager.getLogger(FileManifestProvider::class.java) - - private fun getLatestEntryName(depotID: Int): String = "$depotID${File.separator}latest" - - private fun getEntryName(depotID: Int, manifestID: Long): String = "$depotID${File.separator}$manifestID.bin" - - private fun seekToEntry(zipStream: ZipInputStream, entryName: String): ZipEntry? { - var zipEntry: ZipEntry? - - do { - zipEntry = zipStream.nextEntry - if (zipEntry?.name.equals(entryName, true)) { - break - } - } while (zipEntry != null) - - return zipEntry - } - - private fun copyZip(from: ZipInputStream, to: ZipOutputStream, vararg excludeEntries: String) { - var entry = from.nextEntry - - while (entry != null) { - if (!excludeEntries.contains(entry.name) && (entry.isDirectory || (!entry.isDirectory && entry.size > 0))) { - to.putNextEntry(entry) - - if (!entry.isDirectory) { - val entryBytes = ByteArray(entry.size.toInt()) - - from.readNBytesCompat(entryBytes, 0, entryBytes.size) - to.write(entryBytes) - } - - to.closeEntry() - } - - entry = from.nextEntry - } - } - - private fun zipUncompressed(zip: ZipOutputStream, entryName: String, bytes: ByteArray) { - val entry = ZipEntry(entryName).apply { - method = ZipEntry.STORED - size = bytes.size.toLong() - compressedSize = bytes.size.toLong() - crc = CRC32().run { - update(bytes) - value - } - } - - zip.putNextEntry(entry) - zip.write(bytes) - zip.closeEntry() - } - } - - override fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? = runCatching { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zip -> - seekToEntry(zip, getEntryName(depotID, manifestID))?.let { - if (it.size > 0) { - DepotManifest.deserialize(zip) - } else { - null - } - } - } - } - }.fold( - onSuccess = { it }, - onFailure = { error -> - when (error) { - is NoSuchFileException -> logger.debug("File doesn't exist") - else -> logger.error("Unknown error occurred", error) - } - - null - } - ) - - override fun fetchLatestManifest(depotID: Int): DepotManifest? = runCatching { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zip -> - seekToEntry(zip, getLatestEntryName(depotID))?.let { idEntry -> - if (idEntry.size > 0) { - ByteBuffer.wrap(zip.readNBytesCompat(idEntry.size.toInt())).getLong() - } else { - null - } - } - } - }?.let { manifestId -> - fetchManifest(depotID, manifestId) - } - }.fold( - onSuccess = { it }, - onFailure = { error -> - when (error) { - is NoSuchFileException -> logger.debug("File doesn't exist") - else -> logger.error("Unknown error occurred", error) - } - - null - } - ) - - override fun setLatestManifestId(depotID: Int, manifestID: Long) { - ByteArrayOutputStream().use { bs -> - ZipOutputStream(bs).use { zip -> - // copy old file only if it exists - if (Files.exists(file)) { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zs -> - copyZip(zs, zip, getLatestEntryName(depotID)) - } - } - } - // write manifest id as uncompressed data - ByteBuffer.allocate(Long.SIZE_BYTES).apply { - putLong(manifestID) - zipUncompressed(zip, getLatestEntryName(depotID), array()) - } - } - // save all data to the file - try { - Files.newOutputStream(file).use { fos -> - fos.write(bs.toByteArray()) - } - } catch (e: IOException) { - logger.error("Failed to write manifest ID to file ${file.fileName}", e) - } - } - } - - override fun updateManifest(manifest: DepotManifest) { - ByteArrayOutputStream().use { bs -> - ZipOutputStream(bs).use { zip -> - // copy old file only if it exists - if (Files.exists(file)) { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zs -> - copyZip(zs, zip, getEntryName(manifest.depotID, manifest.manifestGID)) - } - } - } - // add manifest as uncompressed data - MemoryStream().use { ms -> - manifest.serialize(ms.asOutputStream()) - - zipUncompressed(zip, getEntryName(manifest.depotID, manifest.manifestGID), ms.toByteArray()) - } - } - // save all data to the file - try { - Files.newOutputStream(file).use { fos -> - fos.write(bs.toByteArray()) - } - } catch (e: IOException) { - logger.error("Failed to write manifest to file ${file.fileName}", e) - } - } - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt deleted file mode 100644 index 87970b90..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt +++ /dev/null @@ -1,10 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import kotlinx.coroutines.sync.Semaphore -import java.nio.channels.FileChannel - -data class FileStreamData( - var fileStream: FileChannel?, - val fileLock: Semaphore, - var chunksToDownload: Int, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt deleted file mode 100644 index 76929969..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt +++ /dev/null @@ -1,6 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -data class GlobalDownloadCounter( - var totalBytesCompressed: Long = 0, - var totalBytesUncompressed: Long = 0, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt deleted file mode 100644 index 05373c88..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt +++ /dev/null @@ -1,36 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -/** - * An interface for persisting depot manifests for Steam content downloading - * - * @author Oxters - * @since 2024-11-06 - */ -interface IManifestProvider { - - /** - * Ask a provider to fetch a specific depot manifest - * @return A [Pair] object with a [DepotManifest] and its checksum if it exists otherwise null - */ - fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? - - /** - * Ask a provider to fetch the most recent manifest used of a depot - * @return A [Pair] object with a [DepotManifest] and its checksum if it exists otherwise null - */ - fun fetchLatestManifest(depotID: Int): DepotManifest? - - /** - * Ask a provider to set the most recent manifest ID used of a depot - */ - fun setLatestManifestId(depotID: Int, manifestID: Long) - - /** - * Update the persistent depot manifest - * @param manifest The depot manifest - * @return The checksum of the depot manifest - */ - fun updateManifest(manifest: DepotManifest) -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt deleted file mode 100644 index 8f680953..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt +++ /dev/null @@ -1,28 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -/** - * @author Oxters - * @since 2024-11-06 - */ -class MemoryManifestProvider : IManifestProvider { - - private val depotManifests = mutableMapOf>() - - private val latestManifests = mutableMapOf() - - override fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? = - depotManifests[depotID]?.get(manifestID) - - override fun fetchLatestManifest(depotID: Int): DepotManifest? = - latestManifests[depotID]?.let { fetchManifest(depotID, it) } - - override fun setLatestManifestId(depotID: Int, manifestID: Long) { - latestManifests[depotID] = manifestID - } - - override fun updateManifest(manifest: DepotManifest) { - depotManifests.getOrPut(manifest.depotID) { mutableMapOf() }[manifest.manifestGID] = manifest - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt deleted file mode 100644 index afe5adc3..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -/** - * Interface for Java to implement for progress updates - */ -fun interface ProgressCallback { - fun onProgress(progress: Float) -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt similarity index 74% rename from src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt rename to src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt index 807670f5..1d9068aa 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt @@ -1,14 +1,14 @@ -package `in`.dragonbra.javasteam.steam.cdn +package `in`.dragonbra.javasteam.steam.handlers.steamcontent import `in`.dragonbra.javasteam.enums.EResult -import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetCDNAuthToken_Response +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.callback.ServiceMethodResponse import java.util.Date /** * This is received when a CDN auth token is received */ -class AuthToken(message: ServiceMethodResponse) { +class CDNAuthToken(message: ServiceMethodResponse) { /** * Result of the operation diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt index c38bdd3c..5e66c4e6 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt @@ -5,9 +5,9 @@ import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystem import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetManifestRequestCode_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetServersForSteamPipe_Request import `in`.dragonbra.javasteam.rpc.service.ContentServerDirectory -import `in`.dragonbra.javasteam.steam.cdn.AuthToken import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.CDNAuthToken import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages import `in`.dragonbra.javasteam.steam.webapi.ContentServerDirectoryService import kotlinx.coroutines.CoroutineScope @@ -38,7 +38,7 @@ class SteamContent : ClientMsgHandler() { parentScope: CoroutineScope, ): Deferred> = parentScope.async { val request = CContentServerDirectory_GetServersForSteamPipe_Request.newBuilder().apply { - this.cellId = cellId ?: client.cellID?.toInt() ?: 0 + this.cellId = cellId ?: client.cellID ?: 0 maxNumServers?.let { this.maxServers = it } }.build() @@ -65,7 +65,7 @@ class SteamContent : ClientMsgHandler() { branch: String? = null, branchPasswordHash: String? = null, parentScope: CoroutineScope, - ): Deferred = parentScope.async { + ): Deferred = parentScope.async { var localBranch = branch var localBranchPasswordHash = branchPasswordHash @@ -89,24 +89,24 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getManifestRequestCode(request).await() val response = message.body.build() - return@async response.manifestRequestCode.toULong() + return@async response.manifestRequestCode } /** * Request product information for an app or package - * Results are returned in a [AuthToken]. + * Results are returned in a [CDNAuthToken]. * * @param app App id requested. * @param depot Depot id requested. * @param hostName CDN host name being requested. - * @return The [AuthToken] containing the result. + * @return The [CDNAuthToken] containing the result. */ fun getCDNAuthToken( app: Int, depot: Int, hostName: String, parentScope: CoroutineScope, - ): Deferred = parentScope.async { + ): Deferred = parentScope.async { val request = CContentServerDirectory_GetCDNAuthToken_Request.newBuilder().apply { this.appId = app this.depotId = depot @@ -115,7 +115,7 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getCDNAuthToken(request).await() - return@async AuthToken(message) + return@async CDNAuthToken(message) } /** diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt index 0bb0a2af..9c91907a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import okhttp3.OkHttpClient import java.util.* @@ -96,14 +95,6 @@ interface ISteamConfigurationBuilder { */ fun withServerListProvider(provider: IServerListProvider): ISteamConfigurationBuilder - /** - * Configures the depot manifest provider for this [SteamConfiguration]. - * - * @param provider The depot manifest provider to use. - * @return A builder with modified configuration. - */ - fun withManifestProvider(provider: IManifestProvider): ISteamConfigurationBuilder - /** * Configures the Universe that this [SteamConfiguration] belongs to. * diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt index 13212928..44a7c41a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import `in`.dragonbra.javasteam.steam.discovery.SmartCMServerList import `in`.dragonbra.javasteam.steam.steamclient.SteamClient @@ -79,12 +78,6 @@ class SteamConfiguration internal constructor(private val state: SteamConfigurat val serverListProvider: IServerListProvider get() = state.serverListProvider - /** - * The depot manifest provider to use. - */ - val depotManifestProvider: IManifestProvider - get() = state.depotManifestProvider - /** * The Universe to connect to. This should always be [EUniverse.Public] unless you work at Valve and are using this internally. If this is you, hello there. */ diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt index 6ce754d2..c275477a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt @@ -4,8 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider -import `in`.dragonbra.javasteam.steam.contentdownloader.MemoryManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import `in`.dragonbra.javasteam.steam.discovery.MemoryServerListProvider import `in`.dragonbra.javasteam.steam.webapi.WebAPI @@ -72,11 +70,6 @@ class SteamConfigurationBuilder : ISteamConfigurationBuilder { return this } - override fun withManifestProvider(provider: IManifestProvider): ISteamConfigurationBuilder { - state.depotManifestProvider = provider - return this - } - override fun withUniverse(universe: EUniverse): ISteamConfigurationBuilder { state.universe = universe return this @@ -108,7 +101,6 @@ class SteamConfigurationBuilder : ISteamConfigurationBuilder { httpClient = OkHttpClient(), protocolTypes = EnumSet.of(ProtocolTypes.TCP, ProtocolTypes.WEB_SOCKET), serverListProvider = MemoryServerListProvider(), - depotManifestProvider = MemoryManifestProvider(), universe = EUniverse.Public, webAPIBaseAddress = WebAPI.DEFAULT_BASE_ADDRESS, cellID = 0, diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt index fbce399a..00bf1ff4 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import okhttp3.OkHttpClient import java.util.* @@ -22,7 +21,6 @@ data class SteamConfigurationState( var httpClient: OkHttpClient, var protocolTypes: EnumSet, var serverListProvider: IServerListProvider, - var depotManifestProvider: IManifestProvider, var universe: EUniverse, var webAPIBaseAddress: String, var webAPIKey: String?, diff --git a/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt b/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt index 54565d32..d30c5548 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt @@ -8,6 +8,7 @@ import `in`.dragonbra.javasteam.steam.cdn.Server */ object ContentServerDirectoryService { + @JvmStatic internal fun convertServerList( response: CContentServerDirectory_GetServersForSteamPipe_Response, ): List = response.serversList.map { child -> @@ -18,21 +19,21 @@ object ContentServerDirectoryService { Server.ConnectionProtocol.HTTP } - Server( - protocol = protocol, - host = child.host, - vHost = child.vhost, - port = if (protocol == Server.ConnectionProtocol.HTTPS) 443 else 80, - type = child.type, - sourceID = child.sourceId, - cellID = child.cellId, - load = child.load, - weightedLoad = child.weightedLoad, - numEntries = child.numEntriesInClientList, - steamChinaOnly = child.steamChinaOnly, - useAsProxy = child.useAsProxy, - proxyRequestPathTemplate = child.proxyRequestPathTemplate, - allowedAppIds = child.allowedAppIdsList.toIntArray() - ) + Server().apply { + this.protocol = protocol + this.host = child.host + this.vHost = child.vhost + this.port = if (protocol == Server.ConnectionProtocol.HTTPS) 443 else 80 + this.type = child.type + this.sourceId = child.sourceId + this.cellId = child.cellId + this.load = child.load + this.weightedLoad = child.weightedLoad + this.numEntries = child.numEntriesInClientList + this.steamChinaOnly = child.steamChinaOnly + this.useAsProxy = child.useAsProxy + this.proxyRequestPathTemplate = child.proxyRequestPathTemplate + this.allowedAppIds = child.allowedAppIdsList.toIntArray() + } } } diff --git a/src/main/java/in/dragonbra/javasteam/types/PubFile.kt b/src/main/java/in/dragonbra/javasteam/types/PubFile.kt new file mode 100644 index 00000000..fe50ce11 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/types/PubFile.kt @@ -0,0 +1,108 @@ +package `in`.dragonbra.javasteam.types + +/** + * The base class used for wrapping common ULong types, to introduce type safety and distinguish between common types. + */ +@Suppress("unused") +abstract class UInt64Handle : Any { + + /** + * Gets or sets the value. + */ + protected open var value: ULong = 0UL + + /** + * @constructor Initializes a new instance of the [UInt64Handle] class. + */ + protected constructor() + + /** + * Initializes a new instance of the [UInt64Handle] class. + * @param value The value to initialize this handle to. + */ + protected constructor(value: ULong) { + this.value = value + } + + /** + * Returns a hash code for this instance. + * @return A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table. + */ + override fun hashCode(): Int = value.hashCode() + + /** + * Determines whether the specified object is equal to this instance. + * @param other The object to compare with this instance. + * @return true if the specified object is equal to this instance; otherwise, false. + */ + override fun equals(other: Any?): Boolean { + if (other is UInt64Handle) { + return other.value == value + } + + return false + } + + /** + * Returns a string that represents this instance. + * @return A string that represents this instance. + */ + override fun toString(): String = value.toString() + + /** + * TODO + */ + fun toLong(): Long = value.toLong() + + /** + * Indicates whether the current object is equal to another object of the same type. + * @param other An object to compare with this object. + * @return true if the current object is equal to the other parameter; otherwise, false. + */ + fun equals(other: UInt64Handle?): Boolean { + if (other == null) { + return false + } + return value == other.value + } +} + +/** + * Represents a handle to a published file on the Steam workshop. + */ +class PublishedFileID : UInt64Handle { + + /** + * Initializes a new instance of the PublishedFileID class. + * @param fileId The file id. + */ + constructor(fileId: Long = Long.MAX_VALUE) : super(fileId.toULong()) + + companion object { + /** + * Implements the operator ==. + * @param a The first published file. + * @param b The second published file. + * @return The result of the operator. + */ + fun equals(a: PublishedFileID?, b: PublishedFileID?): Boolean { + if (a === b) { + return true + } + + if (a == null || b == null) { + return false + } + + return a.value == b.value + } + + /** + * Implements the operator !=. + * @param a The first published file. + * @param b The second published file. + * @return The result of the operator. + */ + fun notEquals(a: PublishedFileID?, b: PublishedFileID?): Boolean = !equals(a, b) + } +} diff --git a/src/main/java/in/dragonbra/javasteam/util/Utils.java b/src/main/java/in/dragonbra/javasteam/util/Utils.java index ecd8d3f1..f614c5c8 100644 --- a/src/main/java/in/dragonbra/javasteam/util/Utils.java +++ b/src/main/java/in/dragonbra/javasteam/util/Utils.java @@ -193,40 +193,4 @@ public static long crc32(byte[] bytes) { checksum.update(bytes, 0, bytes.length); return checksum.getValue(); } - - /** - * Validate a file against Steam3 Chunk data - * - * @param fs FileInputStream to read from - * @param chunkData Array of ChunkData to validate against - * @return List of ChunkData that are needed - * @throws IOException If there's an error reading the file - * @throws ClosedChannelException If this channel is closed - * @throws IllegalArgumentException If the new position is negative - */ - @SuppressWarnings("resource") - public static List validateSteam3FileChecksums(RandomAccessFile fs, ChunkData[] chunkData) throws IOException { - List neededChunks = new ArrayList<>(); - int read; - - for (ChunkData data : chunkData) { - byte[] chunk = new byte[data.getUncompressedLength()]; - fs.getChannel().position(data.getOffset()); - read = fs.read(chunk, 0, data.getUncompressedLength()); - - byte[] tempChunk; - if (read > 0 && read < data.getUncompressedLength()) { - tempChunk = Arrays.copyOf(chunk, read); - } else { - tempChunk = chunk; - } - - int adler = Adler32.calculate(tempChunk); - if (adler != data.getChecksum()) { - neededChunks.add(data); - } - } - - return neededChunks; - } } diff --git a/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java b/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java index 30e9bd95..2bf4ef4a 100644 --- a/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java +++ b/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java @@ -42,7 +42,7 @@ public void throwsSteamKitWebExceptionOnUnsuccessfulWebResponseForManifest() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); // JVM will throw ExecutionException Exception exception = Assertions.assertThrows(ExecutionException.class, () -> @@ -64,7 +64,7 @@ public void throwsSteamKitWebExceptionOnUnsuccessfulWebResponseForChunk() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0L, 0, 0); // JVM will throw ExecutionException @@ -87,7 +87,7 @@ public void throwsWhenNoChunkIDIsSet() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(); // JVM will throw ExecutionException @@ -111,7 +111,7 @@ public void throwsWhenDestinationBufferSmaller() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0, 32, 64); // JVM will throw ExecutionException @@ -135,7 +135,7 @@ public void throwsWhenDestinationBufferSmallerWithDepotKey() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0, 32, 64); // JVM will throw ExecutionException From d547520aaaccebb0eba1d218a90786193d05272b Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 11:14:13 -0500 Subject: [PATCH 02/21] Clean up dependencies --- gradle/libs.versions.toml | 8 ++++++++ javasteam-depotdownloader/build.gradle.kts | 7 +++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4a35e6ca..5079c202 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -21,6 +21,10 @@ publishPlugin = "2.0.0" # https://mvnrepository.com/artifact/io.github.gradle-ne xz = "1.10" # https://mvnrepository.com/artifact/org.tukaani/xz zstd = "1.5.7-4" # https://search.maven.org/artifact/com.github.luben/zstd-jni +# Depot Downloader +kotlin-serialization-json = "1.9.0" # https://mvnrepository.com/artifact/org.jetbrains.kotlinx/kotlinx-serialization-json-jvm +okio = "3.16.0" # https://mvnrepository.com/artifact/com.squareup.okio/okio + # Testing Lib versions commons-io = "2.20.0" # https://mvnrepository.com/artifact/commons-io/commons-io commonsCodec = "1.19.0" # https://mvnrepository.com/artifact/commons-codec/commons-codec @@ -47,6 +51,10 @@ protobuf-protoc = { module = "com.google.protobuf:protoc", version.ref = "protob xz = { module = "org.tukaani:xz", version.ref = "xz" } zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } +# Depot Downloader +kotlin-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlin-serialization-json"} +okio = { module = "com.squareup.okio:okio", version.ref = "okio"} + # Tests test-commons-codec = { module = "commons-codec:commons-codec", version.ref = "commonsCodec" } test-commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" } diff --git a/javasteam-depotdownloader/build.gradle.kts b/javasteam-depotdownloader/build.gradle.kts index 5ad11c36..df2c31b1 100644 --- a/javasteam-depotdownloader/build.gradle.kts +++ b/javasteam-depotdownloader/build.gradle.kts @@ -67,14 +67,13 @@ tasks.withType { } dependencies { - implementation(rootProject) // TODO verify if this causes something like a circular dependency. - - implementation("com.squareup.okio:okio:3.16.0") - implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + implementation(rootProject) implementation(libs.bundles.ktor) implementation(libs.commons.lang3) implementation(libs.kotlin.coroutines) + implementation(libs.kotlin.serialization.json) implementation(libs.kotlin.stdib) + implementation(libs.okio) implementation(libs.protobuf.java) } From 16f8f8a94281289cd41a0397a1e8248b033e171a Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 13:03:49 -0500 Subject: [PATCH 03/21] Finalize some methods, and some cleanup. --- .../depotdownloader/CDNClientPool.kt | 19 +---- .../depotdownloader/ContentDownloader.kt | 82 +++++++++++-------- .../depotdownloader/DepotConfigStore.kt | 4 +- .../javasteam/depotdownloader/HttpClient.kt | 42 +++++----- .../javasteam/depotdownloader/Util.kt | 22 ++++- 5 files changed, 94 insertions(+), 75 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 6ea50137..81f341aa 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -7,9 +7,6 @@ import `in`.dragonbra.javasteam.steam.steamclient.SteamClient import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.cancel import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlin.jvm.Throws @@ -18,7 +15,7 @@ import kotlin.jvm.Throws * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. * @param steamClient an instance of [SteamClient] * @param appId the selected app id to ensure an endpoint supports the download. - * @param scope (optional) the [CoroutineScope] to use. + * @param scope the [CoroutineScope] to use. * @param debug enable or disable logging through [LogManager] * * @author Oxters @@ -28,18 +25,10 @@ import kotlin.jvm.Throws class CDNClientPool( private val steamClient: SteamClient, private val appId: Int, - private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), + private val scope: CoroutineScope, debug: Boolean = false, ) : AutoCloseable { - companion object { - fun init( - steamClient: SteamClient, - appId: Int, - debug: Boolean, - ): CDNClientPool = CDNClientPool(steamClient = steamClient, appId = appId, debug = debug) - } - private var logger: Logger? = null private val servers: ArrayList = arrayListOf() @@ -63,8 +52,6 @@ class CDNClientPool( } override fun close() { - scope.cancel() - servers.clear() cdnClient = null @@ -134,7 +121,7 @@ class CDNClientPool( if (servers[nextServer % servers.count()] == server) { nextServer++ - // TODO: Add server to ContentServerPenalty + // TODO: (SK) Add server to ContentServerPenalty } } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt index 2571e2c0..0f85c9a6 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -112,10 +112,10 @@ class ContentDownloader @JvmOverloads constructor( // What is a PriorityQueue? private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private val items = CopyOnWriteArrayList(ArrayList()) private val listeners = CopyOnWriteArrayList() - private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private var logger: Logger? = null private val isStarted: AtomicBoolean = AtomicBoolean(false) private val processingChannel = Channel(Channel.UNLIMITED) @@ -242,7 +242,7 @@ class ContentDownloader @JvmOverloads constructor( filesystem.createDirectories(fileFinalPath.parent!!) filesystem.createDirectories(fileStagingPath.parent!!) - HttpClient.httpClient.use { client -> + HttpClient.getClient(maxDownloads).use { client -> logger?.debug("Starting download of $fileName...") val response = client.get(url) @@ -300,7 +300,7 @@ class ContentDownloader @JvmOverloads constructor( var depotManifestIds = depotManifestIds.toMutableList() val steamUser = requireNotNull(steam3!!.steamUser) - cdnClientPool = CDNClientPool.init(steamClient, appId, debug) + cdnClientPool = CDNClientPool(steamClient, appId, scope, debug) // Load our configuration data containing the depots currently installed var configPath = config.installPath @@ -1578,9 +1578,12 @@ class ContentDownloader @JvmOverloads constructor( return } - items.add(0, item) - - notifyListeners { it.onItemAdded(item, 0) } + try { + items.add(0, item) + notifyListeners { it.onItemAdded(item, 0) } + } catch (e: Exception) { + logger?.error(e) + } } fun addAt(index: Int, item: DownloadItem): Boolean { @@ -1604,11 +1607,16 @@ class ContentDownloader @JvmOverloads constructor( return null } - return if (items.isNotEmpty()) { - val item = items.removeAt(0) - notifyListeners { it.onItemRemoved(item, 0) } - item - } else { + return try { + if (items.isNotEmpty()) { + val item = items.removeAt(0) + notifyListeners { it.onItemRemoved(item, 0) } + item + } else { + null + } + } catch (e: IndexOutOfBoundsException) { + logger?.error(e) null } } @@ -1619,12 +1627,17 @@ class ContentDownloader @JvmOverloads constructor( return null } - return if (items.isNotEmpty()) { - val lastIndex = items.size - 1 - val item = items.removeAt(lastIndex) - notifyListeners { it.onItemRemoved(item, lastIndex) } - item - } else { + return try { + if (items.isNotEmpty()) { + val lastIndex = items.size - 1 + val item = items.removeAt(lastIndex) + notifyListeners { it.onItemRemoved(item, lastIndex) } + item + } else { + null + } + } catch (e: IndexOutOfBoundsException) { + logger?.error(e) null } } @@ -1636,11 +1649,16 @@ class ContentDownloader @JvmOverloads constructor( } val index = items.indexOf(item) - return if (index >= 0) { - items.removeAt(index) - notifyListeners { it.onItemRemoved(item, index) } - true - } else { + return try { + if (index >= 0) { + items.removeAt(index) + notifyListeners { it.onItemRemoved(item, index) } + true + } else { + false + } + } catch (e: IndexOutOfBoundsException) { + logger?.error(e) false } } @@ -1720,6 +1738,16 @@ class ContentDownloader @JvmOverloads constructor( remainingItems.set(initialItems.size) initialItems.forEach { processingChannel.send(it) } + if (ClientLancache.useLanCacheServer) { + logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") + + // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely + // be served much faster than over the internet. Steam internally has this behavior as well. + if (maxDownloads == 8) { + maxDownloads = 25 + } + } + repeat(remainingItems.get()) { // Process exactly this many ensureActive() @@ -1733,16 +1761,6 @@ class ContentDownloader @JvmOverloads constructor( ClientLancache.detectLancacheServer() } - if (ClientLancache.useLanCacheServer) { - logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") - - // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely - // be served much faster than over the internet. Steam internally has this behavior as well. - if (maxDownloads == 8) { - maxDownloads = 25 - } - } - // Set some configuration values, first. config = config.copy( downloadManifestOnly = item.downloadManifestOnly, diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index 2f746eb2..56dea76e 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -20,9 +20,6 @@ data class DepotConfigStore( private val json = Json { prettyPrint = true } - val isLoaded: Boolean - get() = instance != null - fun loadFromFile(path: Path) { // require(!isLoaded) { "Config already loaded" } @@ -48,6 +45,7 @@ data class DepotConfigStore( } } + @Throws(IllegalArgumentException::class) fun getInstance(): DepotConfigStore = requireNotNull(instance) { "Config not loaded" } } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 03908dc2..3d626d55 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -13,34 +13,34 @@ import kotlinx.coroutines.isActive */ object HttpClient { - private var _httpClient: HttpClient? = null + private var httpClient: HttpClient? = null - val httpClient: HttpClient - get() { - if (_httpClient?.isActive != true) { - _httpClient = HttpClient(CIO) { - install(UserAgent) { - agent = "DepotDownloader/${Versions.getVersion()}" - } - engine { - maxConnectionsCount = 10 - endpoint { - maxConnectionsPerRoute = 5 - pipelineMaxSize = 20 - keepAliveTime = 5000 - connectTimeout = 5000 - requestTimeout = 30000 - } + fun getClient(maxConnections: Int = 8): HttpClient { + if (httpClient?.isActive != true) { + httpClient = HttpClient(CIO) { + install(UserAgent) { + agent = "DepotDownloader/${Versions.getVersion()}" + } + engine { + maxConnectionsCount = maxConnections + endpoint { + maxConnectionsPerRoute = (maxConnections / 2).coerceAtLeast(1) + pipelineMaxSize = maxConnections * 2 + keepAliveTime = 5000 + connectTimeout = 5000 + requestTimeout = 30000 } } } - return _httpClient!! } + return httpClient!! + } + fun close() { - if (httpClient.isActive) { - _httpClient?.close() - _httpClient = null + if (httpClient?.isActive == true) { + httpClient?.close() + httpClient = null } } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt index b69a0047..8f178765 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -5,6 +5,8 @@ import `in`.dragonbra.javasteam.enums.EDepotFileFlag import `in`.dragonbra.javasteam.types.ChunkData import `in`.dragonbra.javasteam.types.DepotManifest import `in`.dragonbra.javasteam.util.Adler32 +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger import okio.FileHandle import okio.FileSystem import okio.Path @@ -13,6 +15,7 @@ import okio.buffer import org.apache.commons.lang3.SystemUtils import java.io.IOException import java.security.MessageDigest +import java.security.NoSuchAlgorithmException /** * @author Lossy @@ -20,6 +23,8 @@ import java.security.MessageDigest */ object Util { + private val logger: Logger = LogManager.getLogger() + @JvmOverloads @JvmStatic fun getSteamOS(androidEmulation: Boolean = false): String { @@ -47,7 +52,12 @@ object Util { @JvmStatic fun getSteamArch(): String { - val arch = System.getProperty("os.arch")?.lowercase() ?: "" + val arch = try { + System.getProperty("os.arch")?.lowercase() ?: "" + } catch (e: Exception) { + logger.error(e) + "" + } return when { arch.contains("64") -> "64" arch.contains("86") -> "32" @@ -60,6 +70,7 @@ object Util { } @JvmStatic + @Throws(IOException::class) fun saveManifestToFile(directory: Path, manifest: DepotManifest): Boolean = try { val filename = directory / "${manifest.depotID}_${manifest.manifestGID}.manifest" manifest.saveToFile(filename.toString()) @@ -70,11 +81,13 @@ object Util { } true - } catch (e: Exception) { + } catch (e: IOException) { + logger.error(e) false } @JvmStatic + @Throws(NoSuchAlgorithmException::class, IllegalArgumentException::class, IOException::class) fun loadManifestFromFile( directory: Path, depotId: Int, @@ -98,7 +111,7 @@ object Util { if (expectedChecksum != null && expectedChecksum.contentEquals(currentChecksum)) { return DepotManifest.loadFromFile(filename.toString()) } else if (badHashWarning) { - println("Manifest $manifestId on disk did not match the expected checksum.") + logger.debug("Manifest $manifestId on disk did not match the expected checksum.") } } @@ -106,6 +119,7 @@ object Util { } @JvmStatic + @Throws(NoSuchAlgorithmException::class, IllegalArgumentException::class, IOException::class) fun fileSHAHash(filename: Path): ByteArray { val digest = MessageDigest.getInstance("SHA-1") @@ -131,6 +145,7 @@ object Util { * @return List of ChunkData that are needed * @throws IOException If there's an error reading the file */ + @JvmStatic @Throws(IOException::class) fun validateSteam3FileChecksums(handle: FileHandle, chunkData: List): List { val neededChunks = mutableListOf() @@ -155,6 +170,7 @@ object Util { } @JvmStatic + @Throws(IOException::class) fun dumpManifestToTextFile(depot: DepotDownloadInfo, manifest: DepotManifest) { val txtManifest = depot.installDir / "manifest_${depot.depotId}_${depot.manifestId}.txt" From 9aaad5b081be85d11c6ab46a2b636ddb6c21024b Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 15:10:49 -0500 Subject: [PATCH 04/21] Fix nullable value in ConcurrentHashMap --- .../depotdownloader/ContentDownloader.kt | 4 ++-- .../javasteam/depotdownloader/Steam3Session.kt | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt index 0f85c9a6..09d201a0 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -696,7 +696,7 @@ class ContentDownloader @JvmOverloads constructor( return null } - val app = steam3!!.appInfo[appId] ?: return null + val app = steam3!!.appInfo[appId]?.value ?: return null val appInfo = app.keyValues val sectionKey = when (section) { @@ -729,7 +729,7 @@ class ContentDownloader @JvmOverloads constructor( steam3!!.requestPackageInfo(licenseQuery) licenseQuery.forEach { license -> - steam3!!.packageInfo[license]?.let { pkg -> + steam3!!.packageInfo[license]?.value?.let { pkg -> val appIds = pkg.keyValues["appids"].children.map { it.asInteger() } val depotIds = pkg.keyValues["depotids"].children.map { it.asInteger() } if (depotId in appIds) { diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 2b80f563..053e4928 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -42,8 +42,8 @@ class Steam3Session( internal val packageTokens = ConcurrentHashMap() internal val depotKeys = ConcurrentHashMap() internal val cdnAuthTokens = ConcurrentHashMap, CompletableDeferred>() - internal val appInfo = ConcurrentHashMap() - internal val packageInfo = ConcurrentHashMap() + internal val appInfo = ConcurrentHashMap>() + internal val packageInfo = ConcurrentHashMap>() internal val appBetaPasswords = ConcurrentHashMap() private var unifiedMessages: SteamUnifiedMessages? = null @@ -53,6 +53,9 @@ class Steam3Session( internal var steamCloud: SteamCloud? = null internal var steamPublishedFile: PublishedFile? = null + // ConcurrentHashMap can't have nullable Keys or Values + internal data class Optional(val value: T?) + init { if (debug) { logger = LogManager.getLogger(Steam3Session::class.java) @@ -127,10 +130,10 @@ class Steam3Session( appInfoMultiple.results.forEach { appInfo -> appInfo.apps.forEach { appValue -> val app = appValue.value - this.appInfo[app.id] = app + this.appInfo[app.id] = Optional(app) } appInfo.unknownApps.forEach { app -> - this.appInfo[app] = null + this.appInfo[app] = Optional(null) } } } @@ -166,10 +169,10 @@ class Steam3Session( packageInfoMultiple.results.forEach { pkgInfo -> pkgInfo.packages.forEach { pkgValue -> val pkg = pkgValue.value - packageInfo[pkg.id] = pkg + packageInfo[pkg.id] = Optional(pkg) } pkgInfo.unknownPackages.forEach { pkgValue -> - packageInfo[pkgValue] = null + packageInfo[pkgValue] = Optional(null) } } } From daacc5c35b90f6e7bf8319b7f91dc16681335752 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 21:16:38 -0500 Subject: [PATCH 05/21] Some tidying --- .../depotdownloader/CDNClientPool.kt | 19 ++++----- .../depotdownloader/ContentDownloader.kt | 2 +- .../depotdownloader/DepotConfigStore.kt | 2 - .../javasteam/depotdownloader/HttpClient.kt | 2 +- .../depotdownloader/Steam3Session.kt | 41 +++++++++---------- .../javasteam/depotdownloader/Util.kt | 1 + 6 files changed, 31 insertions(+), 36 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 81f341aa..33f0e264 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -2,18 +2,15 @@ package `in`.dragonbra.javasteam.depotdownloader import `in`.dragonbra.javasteam.steam.cdn.Client import `in`.dragonbra.javasteam.steam.cdn.Server -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlin.jvm.Throws /** * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. - * @param steamClient an instance of [SteamClient] + * @param steamSession an instance of [Steam3Session] * @param appId the selected app id to ensure an endpoint supports the download. * @param scope the [CoroutineScope] to use. * @param debug enable or disable logging through [LogManager] @@ -23,7 +20,7 @@ import kotlin.jvm.Throws * @since Nov 7, 2024 */ class CDNClientPool( - private val steamClient: SteamClient, + private val steamSession: Steam3Session, private val appId: Int, private val scope: CoroutineScope, debug: Boolean = false, @@ -44,7 +41,7 @@ class CDNClientPool( private set init { - cdnClient = Client(steamClient) + cdnClient = Client(steamSession.steamClient) if (debug) { logger = LogManager.getLogger(CDNClientPool::class.java) @@ -66,8 +63,8 @@ class CDNClientPool( servers.clear() } - val serversForSteamPipe = steamClient.getHandler()!!.getServersForSteamPipe( - cellId = steamClient.cellID ?: 0, + val serversForSteamPipe = steamSession.steamContent!!.getServersForSteamPipe( + cellId = steamSession.steamClient.cellID ?: 0, maxNumServers = maxNumServers, parentScope = scope ).await() @@ -85,8 +82,8 @@ class CDNClientPool( servers.addAll(weightedCdnServers) - // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } - logger?.debug("Found ${servers.size} Servers: \n") + // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } + logger?.debug("Found ${servers.size} Servers") if (servers.isEmpty()) { throw Exception("Failed to retrieve any download servers.") @@ -108,7 +105,7 @@ class CDNClientPool( logger?.debug("Returning connection: $server") - // nothing to do, maybe remove from ContentServerPenalty? + // (SK) nothing to do, maybe remove from ContentServerPenalty? } suspend fun returnBrokenConnection(server: Server?) = mutex.withLock { diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt index 09d201a0..c539e6ce 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -300,7 +300,7 @@ class ContentDownloader @JvmOverloads constructor( var depotManifestIds = depotManifestIds.toMutableList() val steamUser = requireNotNull(steam3!!.steamUser) - cdnClientPool = CDNClientPool(steamClient, appId, scope, debug) + cdnClientPool = CDNClientPool(steam3!!, appId, scope, debug) // Load our configuration data containing the depots currently installed var configPath = config.installPath diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index 56dea76e..b7297734 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -21,8 +21,6 @@ data class DepotConfigStore( private val json = Json { prettyPrint = true } fun loadFromFile(path: Path) { - // require(!isLoaded) { "Config already loaded" } - instance = if (FileSystem.SYSTEM.exists(path)) { FileSystem.SYSTEM.read(path) { json.decodeFromString(readUtf8()) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 3d626d55..340f85a9 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -19,7 +19,7 @@ object HttpClient { if (httpClient?.isActive != true) { httpClient = HttpClient(CIO) { install(UserAgent) { - agent = "DepotDownloader/${Versions.getVersion()}" + agent = "JavaSteam-DepotDownloader/${Versions.getVersion()}" } engine { maxConnectionsCount = maxConnections diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 053e4928..3f7e7099 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -32,12 +32,14 @@ import java.util.concurrent.ConcurrentHashMap * @since Oct 1, 2025 */ class Steam3Session( - private val steamClient: SteamClient, + internal val steamClient: SteamClient, debug: Boolean, ) : Closeable { private var logger: Logger? = null + private var isAborted: Boolean = false // Stubbed, no way to set true/false yet. + internal val appTokens = ConcurrentHashMap() internal val packageTokens = ConcurrentHashMap() internal val depotKeys = ConcurrentHashMap() @@ -96,7 +98,7 @@ class Steam3Session( } suspend fun requestAppInfo(appId: Int, bForce: Boolean = false) { - if (appInfo.containsKey(appId) && !bForce) { + if ((appInfo.containsKey(appId) && !bForce) || isAborted) { return } @@ -145,7 +147,9 @@ class Steam3Session( // I have a silly race condition??? val packages = packageIds.filter { !packageInfo.containsKey(it) } - if (packages.isEmpty()) return + if (packages.isEmpty() || isAborted) { + return + } val packageRequests = arrayListOf() @@ -192,7 +196,7 @@ class Steam3Session( } suspend fun requestDepotKey(depotId: Int, appId: Int = 0) { - if (depotKeys.containsKey(depotId)) { + if (depotKeys.containsKey(depotId) || isAborted) { return } @@ -216,6 +220,10 @@ class Steam3Session( manifestId: Long, branch: String, ): ULong = withContext(Dispatchers.IO) { + if (isAborted) { + return@withContext 0UL + } + val requestCode = steamContent!!.getManifestRequestCode( depotId = depotId, appId = appId, @@ -229,7 +237,7 @@ class Steam3Session( logger?.error("No manifest request code was returned for depot $depotId from app $appId, manifest $manifestId") if (steamClient.isDisconnected) { - logger?.debug("Suggestion: Try logging in with -username as old manifests may not be available for anonymous accounts.") + logger?.debug("Suggestion: Try logging in with a username as old manifests may not be available for anonymous accounts.") } } else { logger?.debug("Got manifest request code for depot $depotId from app $appId, manifest $manifestId, result: $requestCode") @@ -252,30 +260,21 @@ class Steam3Session( val completion = CompletableDeferred() - val existing = cdnAuthTokens.putIfAbsent(cdnKey, completion) - if (existing != null) { + if (isAborted || cdnAuthTokens.putIfAbsent(cdnKey, completion) != null) { return@withContext } logger?.debug("Requesting CDN auth token for ${server.host}") - try { - val cdnAuth = steamContent!!.getCDNAuthToken(appId, depotId, server.host!!, this).await() - - logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") + val cdnAuth = steamContent!!.getCDNAuthToken(appId, depotId, server.host!!, this).await() - if (cdnAuth.result != EResult.OK) { - cdnAuthTokens.remove(cdnKey) // Remove failed promise - completion.completeExceptionally(Exception("Failed to get CDN auth token: ${cdnAuth.result}")) - return@withContext - } + logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") - completion.complete(cdnAuth) - } catch (e: Exception) { - logger?.error(e) - cdnAuthTokens.remove(cdnKey) // Remove failed promise - completion.completeExceptionally(e) + if (cdnAuth.result != EResult.OK) { + return@withContext } + + completion.complete(cdnAuth) } suspend fun checkAppBetaPassword(appId: Int, password: String) { diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt index 8f178765..9d2bc32f 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -103,6 +103,7 @@ object Util { readByteArray() } } catch (e: IOException) { + logger.error(e) null } From 18109cad26d8ddfbc07065f580b5becab18d5443 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 21:55:34 -0500 Subject: [PATCH 06/21] Add maven and update workflows --- .github/workflows/javasteam-build-push.yml | 4 +++ javasteam-depotdownloader/build.gradle.kts | 39 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/javasteam-build-push.yml b/.github/workflows/javasteam-build-push.yml index 70c27cb9..cbc6ad95 100644 --- a/.github/workflows/javasteam-build-push.yml +++ b/.github/workflows/javasteam-build-push.yml @@ -51,3 +51,7 @@ jobs: with: name: javasteam-tf path: javasteam-tf/build/libs + - uses: actions/upload-artifact@v4 + with: + name: javasteam-depotdownloader + path: javasteam-depotdownloader/build/libs diff --git a/javasteam-depotdownloader/build.gradle.kts b/javasteam-depotdownloader/build.gradle.kts index df2c31b1..e22ddc43 100644 --- a/javasteam-depotdownloader/build.gradle.kts +++ b/javasteam-depotdownloader/build.gradle.kts @@ -28,7 +28,6 @@ protobuf.protoc { artifact = libs.protobuf.protoc.get().toString() } - java { sourceCompatibility = JavaVersion.toVersion(libs.versions.java.get()) targetCompatibility = JavaVersion.toVersion(libs.versions.java.get()) @@ -77,3 +76,41 @@ dependencies { implementation(libs.okio) implementation(libs.protobuf.java) } + +/* Artifact publishing */ +publishing { + publications { + create("mavenJava") { + from(components["java"]) + pom { + name = "JavaSteam-depotdownloader" + packaging = "jar" + description = "Depot Downloader for JavaSteam." + url = "https://github.com/Longi94/JavaSteam" + inceptionYear = "2025" + scm { + connection = "scm:git:git://github.com/Longi94/JavaSteam.git" + developerConnection = "scm:git:ssh://github.com:Longi94/JavaSteam.git" + url = "https://github.com/Longi94/JavaSteam/tree/master" + } + licenses { + license { + name = "MIT License" + url = "https://www.opensource.org/licenses/mit-license.php" + } + } + developers { + developer { + id = "Longi" + name = "Long Tran" + email = "lngtrn94@gmail.com" + } + } + } + } + } +} + +signing { + sign(publishing.publications["mavenJava"]) +} From 9d54fa7b642f43cc242a4cf88923f4a057270be7 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 5 Oct 2025 01:52:09 -0500 Subject: [PATCH 07/21] Rename ContentDownloader and ContentDownloaderException to DepotDownloader and DepotDownloaderException. Add kDoc. Change DepotDownloader to use a fifo based system to download content in order of add. --- .../depotdownloader/CDNClientPool.kt | 14 +- .../ContentDownloaderException.kt | 7 - .../depotdownloader/DepotConfigStore.kt | 7 + ...ontentDownloader.kt => DepotDownloader.kt} | 489 ++++++------------ .../DepotDownloaderException.kt | 11 + .../javasteam/depotdownloader/HttpClient.kt | 5 + .../depotdownloader/IDownloadListener.kt | 78 ++- .../depotdownloader/Steam3Session.kt | 14 +- .../depotdownloader/data/ChunkMatch.kt | 7 + .../depotdownloader/data/DepotDownloadInfo.kt | 12 + .../depotdownloader/data/DepotFilesData.kt | 12 + .../depotdownloader/data/DownloadCounters.kt | 17 + .../depotdownloader/data/DownloadItems.kt | 34 +- .../depotdownloader/data/DownloadProgress.kt | 45 +- .../depotdownloader/data/FileStreamData.kt | 8 + .../_023_downloadapp/SampleDownloadApp.java | 123 ++--- 16 files changed, 443 insertions(+), 440 deletions(-) delete mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt rename javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/{ContentDownloader.kt => DepotDownloader.kt} (82%) create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 33f0e264..58ac5366 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -9,11 +9,15 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock /** - * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. - * @param steamSession an instance of [Steam3Session] - * @param appId the selected app id to ensure an endpoint supports the download. - * @param scope the [CoroutineScope] to use. - * @param debug enable or disable logging through [LogManager] + * Manages a pool of CDN server connections for efficient content downloading. + * This class maintains a list of available CDN servers, automatically selects appropriate + * servers based on load and app compatibility, and handles connection rotation when + * servers fail or become unavailable. + * + * @param steamSession The Steam3 session for server communication + * @param appId The application ID to download - used to filter compatible CDN servers + * @param scope The coroutine scope for async operations + * @param debug If true, enables debug logging * * @author Oxters * @author Lossy diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt deleted file mode 100644 index 5a514926..00000000 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt +++ /dev/null @@ -1,7 +0,0 @@ -package `in`.dragonbra.javasteam.depotdownloader - -/** - * @author Lossy - * @since Oct 1, 2025 - */ -class ContentDownloaderException(value: String) : Exception(value) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index b7297734..0bfebe14 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -6,6 +6,13 @@ import okio.FileSystem import okio.Path /** + * Singleton storage for tracking installed depot manifests. + * Persists manifest IDs to disk to enable incremental updates and avoid + * re-downloading unchanged content. The configuration is serialized as JSON + * and must be loaded via [loadFromFile] before use. + * + * @property installedManifestIDs Map of depot IDs to their currently installed manifest IDs + * * @author Lossy * @since Oct 1, 2025 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt similarity index 82% rename from javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt rename to javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index c539e6ce..481a6a6a 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -52,7 +52,6 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.ensureActive -import kotlinx.coroutines.future.future import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex @@ -70,31 +69,48 @@ import java.lang.IllegalStateException import java.time.Instant import java.time.temporal.ChronoUnit import java.util.concurrent.* -import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger import kotlin.collections.mutableListOf import kotlin.collections.set import kotlin.text.toLongOrNull /** - * [ContentDownloader] is a JavaSteam module that is able to download Games, Workshop Items, and other content from Steam. - * @param steamClient an instance of [SteamClient] - * @param licenses a list of licenses the logged-in user has. This is provided by [LicenseListCallback] - * @param debug enable or disable logging through [LogManager] - * @param useLanCache try and detect a local Steam Cache server. - * @param maxDownloads the number of simultaneous downloads. + * Downloads games, workshop items, and other Steam content via depot manifests. + * + * This class provides a queue-based FIFO download system that processes items sequentially. + * Items are added via [add] or [addAll] and processed automatically in order. The processing + * loop starts immediately upon construction and waits for items to be queued. + * + * ## Download Process + * 1. Validates account access and obtains depot keys + * 2. Downloads and caches depot manifests + * 3. Allocates disk space for files + * 4. Downloads chunks concurrently (configured by [maxDownloads]) + * 5. Verifies checksums and moves files to final location + * + * ## Thread Safety + * Methods [add], [addAll], [addListener], and [removeListener] are thread-safe. + * Multiple concurrent calls are supported. + * + * @param steamClient Connected Steam client instance with valid login session + * @param licenses User's license list from [LicenseListCallback]. Required to determine which depots the account has access to. + * @param debug Enables detailed logging of all operations via [LogManager] + * @param useLanCache Attempts to detect and use local Steam cache servers (e.g., LANCache) for faster downloads on local networks + * @param maxDownloads Number of concurrent chunk downloads. Automatically increased to 25 when a LAN cache is detected. Default: 8 + * @param androidEmulation Forces "Windows" as the default OS filter. Used when running Android games in PC emulators that expect Windows builds. * * @author Oxters * @author Lossy * @since Oct 29, 2024 */ @Suppress("unused") -class ContentDownloader @JvmOverloads constructor( +class DepotDownloader @JvmOverloads constructor( private val steamClient: SteamClient, - private val licenses: List, // To be provided from [LicenseListCallback] - private val debug: Boolean = false, // Enable debugging features, such as logging - private val useLanCache: Boolean = false, // Try and detect a lan cache server. - private var maxDownloads: Int = 8, // Max concurrent downloads + private val licenses: List, + private val debug: Boolean = false, + private val useLanCache: Boolean = false, + private var maxDownloads: Int = 8, + private val androidEmulation: Boolean = false, ) : Closeable { companion object { @@ -109,17 +125,13 @@ class ContentDownloader @JvmOverloads constructor( val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" } - // What is a PriorityQueue? - private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - private val items = CopyOnWriteArrayList(ArrayList()) + private val activeDownloads = AtomicInteger(0) private val listeners = CopyOnWriteArrayList() private var logger: Logger? = null - private val isStarted: AtomicBoolean = AtomicBoolean(false) - private val processingChannel = Channel(Channel.UNLIMITED) - private val remainingItems = AtomicInteger(0) + private var processingChannel = Channel(Channel.UNLIMITED) private val lastFileProgressUpdate = ConcurrentHashMap() private val progressUpdateInterval = 500L // ms @@ -127,7 +139,7 @@ class ContentDownloader @JvmOverloads constructor( private var cdnClientPool: CDNClientPool? = null - private var config: Config = Config() + private var config: Config = Config(androidEmulation = androidEmulation) // region [REGION] Private data classes. @@ -161,15 +173,21 @@ class ContentDownloader @JvmOverloads constructor( init { if (debug) { - logger = LogManager.getLogger(ContentDownloader::class.java) + logger = LogManager.getLogger(DepotDownloader::class.java) } logger?.debug("DepotDownloader launched with ${licenses.size} for account") steam3 = Steam3Session(steamClient, debug) + + // Launch the processing loop + scope.launch { + processItems() + } } // region [REGION] Downloading Operations + @Throws(IllegalStateException::class) suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { val details = requireNotNull( @@ -321,7 +339,7 @@ class ContentDownloader @JvmOverloads constructor( steam3!!.requestAppInfo(appId, true) } else { val contentName = getAppName(appId) - throw ContentDownloaderException("App $appId ($contentName) is not available from this account.") + throw DepotDownloaderException("App $appId ($contentName) is not available from this account.") } } @@ -405,12 +423,12 @@ class ContentDownloader @JvmOverloads constructor( } if (depotManifestIds.isEmpty() && !hasSpecificDepots) { - throw ContentDownloaderException("Couldn't find any depots to download for app $appId") + throw DepotDownloaderException("Couldn't find any depots to download for app $appId") } if (depotIdsFound.size < depotIdsExpected.size) { val remainingDepotIds = depotIdsExpected.subtract(depotIdsFound.toSet()) - throw ContentDownloaderException("Depot ${remainingDepotIds.joinToString(", ")} not listed for app $appId") + throw DepotDownloaderException("Depot ${remainingDepotIds.joinToString(", ")} not listed for app $appId") } } @@ -704,7 +722,7 @@ class ContentDownloader @JvmOverloads constructor( EAppInfoSection.Extended -> "extended" EAppInfoSection.Config -> "config" EAppInfoSection.Depots -> "depots" - else -> throw ContentDownloaderException("${section.name} not implemented") + else -> throw DepotDownloaderException("${section.name} not implemented") } val sectionKV = appInfo.children.firstOrNull { c -> c.name == sectionKey } @@ -1047,7 +1065,7 @@ class ContentDownloader @JvmOverloads constructor( ) ) { notifyListeners { listener -> - listener.onDepotProgress(depot.depotId, snapshot) + listener.onDepotProgress(snapshot) } } } @@ -1072,7 +1090,7 @@ class ContentDownloader @JvmOverloads constructor( ) } notifyListeners { listener -> - listener.onDepotProgress(depot.depotId, snapshot) + listener.onDepotProgress(snapshot) } } } @@ -1173,7 +1191,7 @@ class ContentDownloader @JvmOverloads constructor( handle.resize(file.totalSize) } } catch (e: IOException) { - throw ContentDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") + throw DepotDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") } neededChunks = ArrayList(file.chunks) @@ -1242,7 +1260,7 @@ class ContentDownloader @JvmOverloads constructor( try { newHandle.resize(file.totalSize) } catch (ex: IOException) { - throw ContentDownloaderException( + throw DepotDownloaderException( "Failed to resize file to expected size $fileFinalPath: ${ex.message}" ) } @@ -1269,7 +1287,7 @@ class ContentDownloader @JvmOverloads constructor( try { handle.resize(file.totalSize) } catch (ex: IOException) { - throw ContentDownloaderException( + throw DepotDownloaderException( "Failed to allocate file $fileFinalPath: ${ex.message}" ) } @@ -1483,9 +1501,8 @@ class ContentDownloader @JvmOverloads constructor( notifyListeners { listener -> listener.onFileProgress( - depotId = depot.depotId, - fileName = file.fileName, progress = FileProgress( + depotId = depot.depotId, fileName = file.fileName, bytesDownloaded = approximateBytesDownloaded, totalBytes = file.totalSize, @@ -1542,356 +1559,166 @@ class ContentDownloader @JvmOverloads constructor( // endregion - // region [REGION] Array Operations - - fun getItems(): List = items.toList() - - fun size(): Int = items.size - - fun isEmpty(): Boolean = items.isEmpty() - - fun get(index: Int): DownloadItem? = items.getOrNull(index) - - fun contains(item: DownloadItem): Boolean = items.contains(item) - - fun indexOf(item: DownloadItem): Int = items.indexOf(item) - - fun addAll(items: List) { - items.forEach(::add) - } + // region [REGION] Queue Operations + /** + * Add a singular item of either [AppItem], [PubFileItem], or [UgcItem] + */ fun add(item: DownloadItem) { - val index = items.size - items.add(item) - - if (isStarted.get()) { - remainingItems.incrementAndGet() - scope.launch { processingChannel.send(item) } - } - - notifyListeners { it.onItemAdded(item, index) } - } - - fun addFirst(item: DownloadItem) { - if (isStarted.get()) { - logger?.debug("Cannot add item when started.") - return - } - - try { - items.add(0, item) - notifyListeners { it.onItemAdded(item, 0) } - } catch (e: Exception) { - logger?.error(e) - } - } - - fun addAt(index: Int, item: DownloadItem): Boolean { - if (isStarted.get()) { - logger?.debug("Cannot addAt item when started.") - return false - } - - return try { - items.add(index, item) - notifyListeners { it.onItemAdded(item, index) } - true - } catch (e: IndexOutOfBoundsException) { - false - } - } - - fun removeFirst(): DownloadItem? { - if (isStarted.get()) { - logger?.debug("Cannot removeFirst item when started.") - return null - } - - return try { - if (items.isNotEmpty()) { - val item = items.removeAt(0) - notifyListeners { it.onItemRemoved(item, 0) } - item - } else { - null - } - } catch (e: IndexOutOfBoundsException) { - logger?.error(e) - null + runBlocking { + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item.appId) } } } - fun removeLast(): DownloadItem? { - if (isStarted.get()) { - logger?.debug("Cannot removeLast item when started.") - return null - } - - return try { - if (items.isNotEmpty()) { - val lastIndex = items.size - 1 - val item = items.removeAt(lastIndex) - notifyListeners { it.onItemRemoved(item, lastIndex) } - item - } else { - null - } - } catch (e: IndexOutOfBoundsException) { - logger?.error(e) - null - } - } - - fun remove(item: DownloadItem): Boolean { - if (isStarted.get()) { - logger?.debug("Cannot remove item when started.") - return false - } - - val index = items.indexOf(item) - return try { - if (index >= 0) { - items.removeAt(index) - notifyListeners { it.onItemRemoved(item, index) } - true - } else { - false + /** + * Add a list items of either [AppItem], [PubFileItem], or [UgcItem] + */ + fun addAll(items: List) { + runBlocking { + items.forEach { item -> + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item.appId) } } - } catch (e: IndexOutOfBoundsException) { - logger?.error(e) - false - } - } - - fun removeAt(index: Int): DownloadItem? { - if (isStarted.get()) { - logger?.debug("Cannot removeAt item when started.") - return null - } - - return try { - val item = items.removeAt(index) - notifyListeners { it.onItemRemoved(item, index) } - item - } catch (e: IndexOutOfBoundsException) { - null - } - } - - fun moveItem(fromIndex: Int, toIndex: Int): Boolean { - if (isStarted.get()) { - logger?.debug("Cannot moveItem item when started.") - return false - } - - return try { - val item = items.removeAt(fromIndex) - items.add(toIndex, item) - true - } catch (e: IndexOutOfBoundsException) { - false } } - fun clear() { - if (isStarted.get()) { - logger?.debug("Cannot clear item when started.") - return - } - - val oldItems = items.toList() - items.clear() - - notifyListeners { it.onQueueCleared(oldItems) } - } - - // endregion - /** - * Some android emulators prefer using "Windows", so this will set downloading to prefer the Windows version. + * Get the current queue size of pending items to be downloaded. */ - fun setAndroidEmulation(value: Boolean) { - if (isStarted.get()) { - logger?.error("Can't set android emulation value once started.") - return - } + fun queueSize(): Int = activeDownloads.get() - config = config.copy(androidEmulation = value) - - notifyListeners { it.onAndroidEmulation(config.androidEmulation) } - } + /** + * Get a boolean value if there are items in queue to be downloaded. + */ + fun isProcessing(): Boolean = activeDownloads.get() > 0 - @Throws(IllegalStateException::class) - fun start(): CompletableFuture = scope.future { - if (isStarted.getAndSet(true)) { - logger?.debug("Downloading already started.") - return@future false - } + // endregion - val initialItems = items.toList() - if (initialItems.isEmpty()) { - logger?.debug("No items to download") - return@future false + private suspend fun processItems() = coroutineScope { + if (useLanCache) { + ClientLancache.detectLancacheServer() } - // Send initial items - remainingItems.set(initialItems.size) - initialItems.forEach { processingChannel.send(it) } - if (ClientLancache.useLanCacheServer) { logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") - - // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely - // be served much faster than over the internet. Steam internally has this behavior as well. if (maxDownloads == 8) { maxDownloads = 25 } } - repeat(remainingItems.get()) { - // Process exactly this many - ensureActive() - - // Obtain the next item in queue. - val item = processingChannel.receive() - + for (item in processingChannel) { try { - runBlocking { - if (useLanCache) { - ClientLancache.detectLancacheServer() - } - - // Set some configuration values, first. - config = config.copy( - downloadManifestOnly = item.downloadManifestOnly, - installPath = item.installDirectory?.toPath(), - installToGameNameDirectory = item.installToGameNameDirectory, - ) - - // Sequential looping. - when (item) { - is PubFileItem -> { - if (item.pubfile == INVALID_MANIFEST_ID) { - logger?.debug("Invalid Pub File ID for ${item.appId}") - return@runBlocking - } + ensureActive() - logger?.debug("Downloading PUB File for ${item.appId}") + // Set configuration values + config = config.copy( + downloadManifestOnly = item.downloadManifestOnly, + installPath = item.installDirectory?.toPath(), + installToGameNameDirectory = item.installToGameNameDirectory, + ) - notifyListeners { it.onDownloadStarted(item) } - downloadPubFile(item.appId, item.pubfile) + when (item) { + is PubFileItem -> { + if (item.pubfile == INVALID_MANIFEST_ID) { + logger?.debug("Invalid Pub File ID for ${item.appId}") + continue } + logger?.debug("Downloading PUB File for ${item.appId}") + notifyListeners { it.onDownloadStarted(item.appId) } + downloadPubFile(item.appId, item.pubfile) + } - is UgcItem -> { - if (item.ugcId == INVALID_MANIFEST_ID) { - logger?.debug("Invalid UGC ID for ${item.appId}") - return@runBlocking - } - - logger?.debug("Downloading UGC File for ${item.appId}") - - notifyListeners { it.onDownloadStarted(item) } - downloadUGC(item.appId, item.ugcId) + is UgcItem -> { + if (item.ugcId == INVALID_MANIFEST_ID) { + logger?.debug("Invalid UGC ID for ${item.appId}") + continue } + logger?.debug("Downloading UGC File for ${item.appId}") + notifyListeners { it.onDownloadStarted(item.appId) } + downloadUGC(item.appId, item.ugcId) + } - is AppItem -> { - val branch = item.branch ?: DEFAULT_BRANCH - config = config.copy(betaPassword = item.branchPassword) - - if (!config.betaPassword.isNullOrBlank() && branch.isBlank()) { - logger?.error("Error: Cannot specify 'branchpassword' when 'branch' is not specified.") - return@runBlocking - } - - config = config.copy(downloadAllPlatforms = item.downloadAllPlatforms) - - val os = item.os - - if (config.downloadAllPlatforms && !os.isNullOrBlank()) { - logger?.error("Error: Cannot specify `os` when `all-platforms` is specified.") - return@runBlocking - } - - config = config.copy(downloadAllArchs = item.downloadAllArchs) + is AppItem -> { + val branch = item.branch ?: DEFAULT_BRANCH + config = config.copy(betaPassword = item.branchPassword) - val arch = item.osArch + if (!config.betaPassword.isNullOrBlank() && branch.isBlank()) { + logger?.error("Error: Cannot specify 'branchpassword' when 'branch' is not specified.") + continue + } - if (config.downloadAllArchs && !arch.isNullOrBlank()) { - logger?.error("Error: Cannot specify `osarch` when `all-archs` is specified.") - return@runBlocking - } + config = config.copy(downloadAllPlatforms = item.downloadAllPlatforms) + val os = item.os - config = config.copy(downloadAllLanguages = item.downloadAllLanguages) + if (config.downloadAllPlatforms && !os.isNullOrBlank()) { + logger?.error("Error: Cannot specify `os` when `all-platforms` is specified.") + continue + } - val language = item.language + config = config.copy(downloadAllArchs = item.downloadAllArchs) + val arch = item.osArch - if (config.downloadAllLanguages && !language.isNullOrBlank()) { - logger?.error("Error: Cannot specify `language` when `all-languages` is specified.") - return@runBlocking - } + if (config.downloadAllArchs && !arch.isNullOrBlank()) { + logger?.error("Error: Cannot specify `osarch` when `all-archs` is specified.") + continue + } - val lv = item.lowViolence + config = config.copy(downloadAllLanguages = item.downloadAllLanguages) + val language = item.language - val depotManifestIds = mutableListOf>() - val isUGC = false + if (config.downloadAllLanguages && !language.isNullOrBlank()) { + logger?.error("Error: Cannot specify `language` when `all-languages` is specified.") + continue + } - val depotIdList = item.depot - val manifestIdList = item.manifest + val depotManifestIds = mutableListOf>() + val depotIdList = item.depot + val manifestIdList = item.manifest - if (manifestIdList.isNotEmpty()) { - if (depotIdList.size != manifestIdList.size) { - logger?.error("Error: `manifest` requires one id for every `depot` specified") - return@runBlocking - } - val zippedDepotManifest = depotIdList.zip(manifestIdList) { depotId, manifestId -> - Pair(depotId, manifestId) - } - depotManifestIds.addAll(zippedDepotManifest) - } else { - depotManifestIds.addAll( - depotIdList.map { depotId -> - Pair(depotId, INVALID_MANIFEST_ID) - } - ) + if (manifestIdList.isNotEmpty()) { + if (depotIdList.size != manifestIdList.size) { + logger?.error("Error: `manifest` requires one id for every `depot` specified") + continue } - - logger?.debug("Downloading App for ${item.appId}") - - notifyListeners { it.onDownloadStarted(item) } - downloadApp( - appId = item.appId, - depotManifestIds = depotManifestIds, - branch = branch, - os = os, - arch = arch, - language = language, - lv = lv, - isUgc = isUGC, - ) + depotManifestIds.addAll(depotIdList.zip(manifestIdList)) + } else { + depotManifestIds.addAll(depotIdList.map { it to INVALID_MANIFEST_ID }) } - } - notifyListeners { it.onDownloadCompleted(item) } + logger?.debug("Downloading App for ${item.appId}") + notifyListeners { it.onDownloadStarted(item.appId) } + downloadApp( + appId = item.appId, + depotManifestIds = depotManifestIds, + branch = branch, + os = os, + arch = arch, + language = language, + lv = item.lowViolence, + isUgc = false, + ) + } } - } catch (e: IOException) { - logger?.error("Error downloading item ${item.appId}: ${e.message}", e) - - notifyListeners { it.onDownloadFailed(item, e) } - throw e + notifyListeners { it.onDownloadCompleted(item.appId) } + } catch (e: Exception) { + logger?.error("Error downloading item ${item.appId}: ${e.message}", e) + notifyListeners { it.onDownloadFailed(item.appId, e) } + } finally { + activeDownloads.decrementAndGet() } } - - return@future true } override fun close() { - isStarted.set(false) + processingChannel.close() - HttpClient.close() + scope.cancel("DepotDownloader Closing") - items.clear() - processingChannel.close() + HttpClient.close() lastFileProgressUpdate.clear() listeners.clear() diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt new file mode 100644 index 00000000..4c3b4bbc --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt @@ -0,0 +1,11 @@ +package `in`.dragonbra.javasteam.depotdownloader + +/** + * Exception thrown when content download operations fail. + * Used to indicate errors during depot downloads, manifest retrieval, + * or other content downloader operations. + * + * @author Lossy + * @since Oct 1, 2025 + */ +class DepotDownloaderException(value: String) : Exception(value) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 340f85a9..75e4f612 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -8,6 +8,11 @@ import io.ktor.client.plugins.UserAgent import kotlinx.coroutines.isActive /** + * Singleton HTTP client for content downloader operations. + * Provides a shared, configured Ktor HTTP client optimized for Steam CDN downloads. + * The client is lazily initialized on first use and reused across all download operations. + * Connection pooling and timeouts are configured based on the maximum concurrent downloads. + * * @author Lossy * @since Oct 1, 2025 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt index c3ce3c22..9b4ab7f2 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -1,35 +1,79 @@ package `in`.dragonbra.javasteam.depotdownloader import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress -import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress /** - * Listener interface for download events. + * Listener interface for receiving download progress and status events. + * + * All methods have default empty implementations, allowing listeners to + * implement only the callbacks they need. * * @author Lossy * @since Oct 1, 2025 */ interface IDownloadListener { - // Queue management - fun onItemAdded(item: DownloadItem, index: Int) {} - fun onItemRemoved(item: DownloadItem, index: Int) {} - fun onQueueCleared(previousItems: List) {} + /** + * Called when an item is added to the download queue. + * + * @param appId The application ID that was queued + */ + fun onItemAdded(appId: Int) {} + + /** + * Called when a download begins processing. + * + * @param appId The application ID being downloaded + */ + fun onDownloadStarted(appId: Int) {} + + /** + * Called when a download completes successfully. + * + * @param appId The application ID that finished downloading + */ + fun onDownloadCompleted(appId: Int) {} - // Download lifecycle - fun onDownloadStarted(item: DownloadItem) {} - fun onDownloadCompleted(item: DownloadItem) {} - fun onDownloadFailed(item: DownloadItem, error: Throwable) {} + /** + * Called when a download fails with an error. + * + * @param appId The application ID that failed + * @param error The exception that caused the failure + */ + fun onDownloadFailed(appId: Int, error: Throwable) {} - // Progress tracking + /** + * Called periodically with overall download progress across all items. + * Reports progress for the entire download queue, including completed + * and remaining items. + * + * @param progress Overall download statistics + */ fun onOverallProgress(progress: OverallProgress) {} - fun onDepotProgress(depotId: Int, progress: DepotProgress) {} - fun onFileProgress(depotId: Int, fileName: String, progress: FileProgress) {} - // Status updates - fun onStatusUpdate(message: String) {} + /** + * Called periodically with progress for a specific depot. + * Reports file allocation and download progress for an individual depot. + * + * @param progress Depot-specific download statistics + */ + fun onDepotProgress(progress: DepotProgress) {} + + /** + * Called periodically with progress for a specific file. + * Reports chunk-level download progress for individual files. + * + * @param progress File-specific download statistics + */ + fun onFileProgress(progress: FileProgress) {} - // Configuration - fun onAndroidEmulation(value: Boolean) {} + /** + * Called with informational status messages during download operations. + * Used for logging or displaying current operations like manifest + * downloads, file validation, and allocation. + * + * @param message Human-readable status message + */ + fun onStatusUpdate(message: String) {} } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 3f7e7099..9a3ebb94 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -28,6 +28,14 @@ import java.io.Closeable import java.util.concurrent.ConcurrentHashMap /** + * Manages Steam protocol session state and API interactions for content downloading. + * + * All Steam API handlers are initialized during construction and must remain valid + * for the lifetime of this session. + * + * @param steamClient The connected Steam client instance + * @param debug If true, enables debug logging for all Steam API operations + * * @author Lossy * @since Oct 1, 2025 */ @@ -304,7 +312,7 @@ class Steam3Session( return privateBeta.depotSection } - @Throws(ContentDownloaderException::class) + @Throws(DepotDownloaderException::class) suspend fun getPublishedFileDetails( appId: Int, pubFile: PublishedFileID, @@ -323,7 +331,7 @@ class Steam3Session( return details.body.publishedfiledetailsBuilderList.firstOrNull()?.build() } - throw ContentDownloaderException("EResult ${details.result.code()} (${details.result}) while retrieving file details for pubfile $pubFile.") + throw DepotDownloaderException("EResult ${details.result.code()} (${details.result}) while retrieving file details for pubfile $pubFile.") } suspend fun getUGCDetails(ugcHandle: UGCHandle): UGCDetailsCallback? { @@ -337,6 +345,6 @@ class Steam3Session( return null } - throw ContentDownloaderException("EResult ${callback.result.code()} (${callback.result}) while retrieving UGC details for ${ugcHandle.value}.") + throw DepotDownloaderException("EResult ${callback.result.code()} (${callback.result}) while retrieving UGC details for ${ugcHandle.value}.") } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt index 05fbb247..dafee0bd 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt @@ -3,6 +3,13 @@ package `in`.dragonbra.javasteam.depotdownloader.data import `in`.dragonbra.javasteam.types.ChunkData /** + * Pairs matching chunks between old and new depot manifests for differential updates. + * Used during file validation to identify chunks that can be reused from existing + * files, avoiding unnecessary re-downloads when only portions of a file have changed. + * + * @property oldChunk Chunk from the previously installed manifest + * @property newChunk Corresponding chunk from the new manifest being downloaded + * * @author Oxters * @since Oct 29, 2024 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt index 256575ec..5b853ce4 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt @@ -3,6 +3,18 @@ package `in`.dragonbra.javasteam.depotdownloader.data import okio.Path /** + * Contains all information required to download a specific depot manifest and its content. + * This class aggregates the depot identification, authentication, and installation details + * needed to perform a complete depot download operation. It is created during the depot + * resolution phase and passed through the download pipeline. + * + * @property depotId The Steam depot identifier + * @property appId The owning application ID (may differ from the app being downloaded if the depot uses `depotfromapp` proxying) + * @property manifestId The specific manifest version to download + * @property branch The branch name this manifest belongs to (e.g., "public", "beta") + * @property installDir The target directory for downloaded files + * @property depotKey The AES decryption key for depot chunks. Cleared on disposal for security. + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt index bef4590b..15d4dda9 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt @@ -5,6 +5,18 @@ import `in`.dragonbra.javasteam.types.FileData import okio.Path /** + * Aggregates all data needed to process and download files for a single depot. + * Created during manifest processing and passed to the download phase. Contains both + * the current manifest and optional previous manifest to enable differential updates. + * + * @property depotDownloadInfo Core depot identification and authentication details + * @property depotCounter Progress tracking counters for this depot's download + * @property stagingDir Temporary directory for in-progress file writes + * @property manifest The current depot manifest being downloaded + * @property previousManifest The previously installed manifest, if any. Used to identify reusable chunks and deleted files. + * @property filteredFiles Files to download after applying platform, language, and user filters. Modified during processing to remove duplicates across depots. + * @property allFileNames Complete set of filenames in this depot, including directories. Used for cross-depot deduplication and cleanup of deleted files. + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt index 8d509e27..380463c2 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt @@ -3,6 +3,14 @@ package `in`.dragonbra.javasteam.depotdownloader.data // https://kotlinlang.org/docs/coding-conventions.html#source-file-organization /** + * Tracks cumulative download statistics across all depots in a download session. + * Used for overall progress reporting and final download summary. All fields are + * accessed under synchronization to ensure thread-safe updates from concurrent downloads. + * + * @property completeDownloadSize Total bytes expected to download across all depots. Adjusted during validation when existing chunks are reused. + * @property totalBytesCompressed Total compressed bytes transferred from CDN servers + * @property totalBytesUncompressed Total uncompressed bytes written to disk + * * @author Oxters * @author Lossy * @since Oct 29, 2024 @@ -14,6 +22,15 @@ data class GlobalDownloadCounter( ) /** + * Tracks download statistics for a single depot. + * Used for depot-level progress reporting. All fields are accessed under synchronization + * to ensure thread-safe updates from concurrent chunk downloads. + * + * @property completeDownloadSize Total bytes expected to download for this depot. Calculated during file allocation phase. + * @property sizeDownloaded Bytes successfully downloaded and written so far + * @property depotBytesCompressed Compressed bytes transferred from CDN for this depot + * @property depotBytesUncompressed Uncompressed bytes written to disk for this depot + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt index a204c4e1..e94ecdd6 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -1,10 +1,14 @@ package `in`.dragonbra.javasteam.depotdownloader.data -import `in`.dragonbra.javasteam.depotdownloader.ContentDownloader - // https://kotlinlang.org/docs/coding-conventions.html#source-file-organization /** + * Base class for downloadable Steam content items. + * @property appId The Steam application ID + * @property installDirectory Optional custom installation directory path + * @property installToGameNameDirectory If true, installs to a directory named after the game + * @property downloadManifestOnly If true, only downloads the manifest file without actual content + * * @author Lossy * @since Oct 1, 2025 */ @@ -16,30 +20,52 @@ abstract class DownloadItem( ) /** + * Represents a Steam Workshop (UGC - User Generated Content) item for download. + * + * @property ugcId The unique UGC item identifier + * * @author Lossy * @since Oct 1, 2025 */ class UgcItem @JvmOverloads constructor( appId: Int, - val ugcId: Long = ContentDownloader.INVALID_MANIFEST_ID, + val ugcId: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, downloadManifestOnly: Boolean = false, ) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) /** + * Represents a Steam published file for download. + * + * @property pubfile The published file identifier + * * @author Lossy * @since Oct 1, 2025 */ class PubFileItem @JvmOverloads constructor( appId: Int, - val pubfile: Long = ContentDownloader.INVALID_MANIFEST_ID, + val pubfile: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, downloadManifestOnly: Boolean = false, ) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) /** + * Represents a Steam application/game for download from a depot. + * + * @property branch The branch name to download from (e.g., "public", "beta") + * @property branchPassword Password for password-protected branches + * @property downloadAllPlatforms If true, downloads depots for all platforms + * @property os Operating system filter (e.g., "windows", "macos", "linux") + * @property downloadAllArchs If true, downloads depots for all architectures + * @property osArch Architecture filter (e.g., "32", "64") + * @property downloadAllLanguages If true, downloads depots for all languages + * @property language Language filter (e.g., "english", "french") + * @property lowViolence If true, downloads low-violence versions where available + * @property depot List of specific depot IDs to download + * @property manifest List of specific manifest IDs corresponding to depot IDs + * * @author Lossy * @since Oct 1, 2025 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt index f8e8035d..2e058560 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt @@ -1,7 +1,16 @@ package `in`.dragonbra.javasteam.depotdownloader.data /** - * Overall download progress across all items. + * Reports overall download progress across all queued items. + * Provides high-level statistics for the entire download session, tracking + * which item is currently processing and cumulative byte transfer. + * + * @property currentItem Number of items completed (1-based) + * @property totalItems Total number of items in the download session + * @property totalBytesDownloaded Cumulative uncompressed bytes downloaded across all depots + * @property totalBytesExpected Total uncompressed bytes expected for all items + * @property status Current download phase + * @property percentComplete Calculated completion percentage (0.0 to 100.0) * * @author Lossy * @since Oct 1, 2025 @@ -22,7 +31,18 @@ data class OverallProgress( } /** - * Progress for a specific depot + * Reports download progress for a specific depot within an item. + * Tracks both file-level progress (allocation/validation) and byte-level + * download progress. During the [DownloadStatus.PREPARING] phase, tracks + * file allocation; during [DownloadStatus.DOWNLOADING], tracks actual transfers. + * + * @property depotId The Steam depot identifier + * @property filesCompleted Number of files fully allocated or downloaded + * @property totalFiles Total files to process in this depot (excludes directories) + * @property bytesDownloaded Uncompressed bytes successfully downloaded + * @property totalBytes Total uncompressed bytes expected for this depot + * @property status Current depot processing phase + * @property percentComplete Calculated completion percentage (0.0 to 100.0) * * @author Lossy * @since Oct 1, 2025 @@ -44,12 +64,24 @@ data class DepotProgress( } /** - * Progress for a specific file + * Reports download progress for an individual file. + * Provides chunk-level granularity for tracking file downloads. Updates are + * throttled to every 500ms to avoid excessive callback overhead. + * + * @property depotId The Steam depot containing this file + * @property fileName Relative path of the file within the depot + * @property bytesDownloaded Approximate uncompressed bytes downloaded (based on chunk completion) + * @property totalBytes Total uncompressed file size + * @property chunksCompleted Number of chunks successfully downloaded and written + * @property totalChunks Total chunks comprising this file + * @property status Current file download status + * @property percentComplete Calculated completion percentage (0.0 to 100.0) * * @author Lossy * @since Oct 1, 2025 */ data class FileProgress( + val depotId: Int, val fileName: String, val bytesDownloaded: Long, val totalBytes: Long, @@ -65,6 +97,13 @@ data class FileProgress( } } +/** + * Represents the current phase of a download operation. + * + * @property PREPARING File allocation and validation phase. Files are being pre-allocated on disk and existing content is being verified. + * @property DOWNLOADING Active chunk download phase. Content is being transferred from CDN. + * @property COMPLETED Download finished successfully. All files written and verified. + */ enum class DownloadStatus { PREPARING, DOWNLOADING, diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt index 2b6a539c..1bc68fac 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt @@ -5,6 +5,14 @@ import okio.FileHandle import java.util.concurrent.atomic.AtomicInteger /** + * Internal state for managing concurrent chunk writes to a single file. + * Coordinates writes from multiple download workers to ensure thread-safe file access + * and tracks when all chunks have been written. + * + * @property fileHandle Shared file handle for all chunk writes. Lazily opened on first write. + * @property fileLock Mutex protecting concurrent access to [fileHandle] + * @property chunksToDownload Atomic counter of remaining chunks. File is closed when this reaches zero. + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index b73f408a..f7abc981 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -1,6 +1,6 @@ package in.dragonbra.javasteamsamples._023_downloadapp; -import in.dragonbra.javasteam.depotdownloader.ContentDownloader; +import in.dragonbra.javasteam.depotdownloader.DepotDownloader; import in.dragonbra.javasteam.depotdownloader.IDownloadListener; import in.dragonbra.javasteam.depotdownloader.data.*; import in.dragonbra.javasteam.enums.EResult; @@ -30,8 +30,8 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Scanner; import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; /** @@ -123,6 +123,7 @@ public void run() { for (var subscription : subscriptions) { try { + System.out.println("Closing: " + subscription.getClass().getName()); subscription.close(); } catch (IOException e) { System.out.println("Couldn't close a callback."); @@ -238,7 +239,6 @@ private void onLicenseList(LicenseListCallback callback) { System.out.println("Got " + licenseList.size() + " licenses from account!"); } - @SuppressWarnings("ExtractMethodRecommender") private void onFreeLicense(FreeLicenseCallback callback) { if (callback.getResult() != EResult.OK) { System.out.println("Failed to get a free license for Rocky Mayhem"); @@ -248,69 +248,72 @@ private void onFreeLicense(FreeLicenseCallback callback) { // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. // You will need to subscribe to LicenseListCallback to obtain your app licenses. - try (var depotDownloader = new ContentDownloader(steamClient, licenseList, false)) { + try (var depotDownloader = new DepotDownloader(steamClient, licenseList, false)) { + // Add this class as a listener of IDownloadListener depotDownloader.addListener(this); // An app id is required at minimum for all item types. var pubItem = new PubFileItem( /* appId */ 0, /* pubfile */ 0, - /* installToGameNameDirectory */ false, - /* installDirectory */ null, - /* downloadManifestOnly */ false + /* (Optional) installToGameNameDirectory */ false, + /* (Optional) installDirectory */ null, + /* (Optional) downloadManifestOnly */ false ); // TODO find actual pub item var ugcItem = new UgcItem( /* appId */0, /* ugcId */ 0, - /* installToGameNameDirectory */ false, - /* installDirectory */ null, - /* downloadManifestOnly */ false + /* (Optional) installToGameNameDirectory */ false, + /* (Optional) installDirectory */ null, + /* (Optional) downloadManifestOnly */ false ); // TODO find actual ugc item var appItem = new AppItem( /* appId */ 204360, - /* installToGameNameDirectory */ true, - /* installDirectory */ DEFAULT_INSTALL_DIRECTORY, - /* branch */ "public", - /* branchPassword */ "", - /* downloadAllPlatforms */ false, - /* os */ "windows", - /* downloadAllArchs */ false, - /* osArch */ "64", - /* downloadAllLanguages */ false, - /* language */ "english", - /* lowViolence */ false, - /* depot */ List.of(), - /* manifest */ List.of(), - /* downloadManifestOnly */ false + /* (Optional) installToGameNameDirectory */ true, + /* (Optional) installDirectory */ DEFAULT_INSTALL_DIRECTORY, + /* (Optional) branch */ "public", + /* (Optional) branchPassword */ "", + /* (Optional) downloadAllPlatforms */ false, + /* (Optional) os */ "windows", + /* (Optional) downloadAllArchs */ false, + /* (Optional) osArch */ "64", + /* (Optional) downloadAllLanguages */ false, + /* (Optional) language */ "english", + /* (Optional) lowViolence */ false, + /* (Optional) depot */ List.of(), + /* (Optional) manifest */ List.of(), + /* (Optional) downloadManifestOnly */ false ); - var appItem2 = new AppItem(225840, true); - var appItem3 = new AppItem(3527290, true); - var appItem4 = new AppItem(ROCKY_MAYHEM_APP_ID, true); + var scanner = new Scanner(System.in); + System.out.print("Enter a game app id: "); + var appId = scanner.nextInt(); - var downloadList = List.of(pubItem, ugcItem, appItem); + // After 'depotDownloader' is constructed, items added are downloaded in a First-In, First-Out queue on the fly. - // Add specified games to the queue. Add, Remove, Move, and general array manipulation methods are available. - // depotDownloader.addAll(downloadList); // TODO - depotDownloader.addAll(List.of(appItem)); + // Add a singular item to process. + depotDownloader.add(new AppItem(appId, true)); - // Start downloading your items. Array manipulation is now disabled. You can still add to the list. - var success = depotDownloader.start().get(); // Future + // You can add a List of items to be processed. + // depotDownloader.add(List.of()); - if (success) { - System.out.println("Download completed successfully"); + // Stay here while content downloads. Note this sample is synchronous so we'll loop here. + while (depotDownloader.isProcessing()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + break; + } } + // Remove this class as a listener of IDownloadListener depotDownloader.removeListener(this); - } catch (IllegalStateException | InterruptedException | ExecutionException e) { - System.out.println("Something happened"); - System.err.println(e.getMessage()); } finally { - steamUser.logOff(); System.out.println("Done Downloading"); + steamUser.logOff(); } } @@ -323,38 +326,26 @@ private void onLoggedOff(LoggedOffCallback callback) { // Depot Downloader Callbacks. @Override - public void onItemAdded(@NotNull DownloadItem item, int index) { - System.out.println("Depot Downloader: Item Added: " + item.getAppId() + ", index: " + index); - System.out.println(" ---- "); - } - - @Override - public void onItemRemoved(@NotNull DownloadItem item, int index) { - System.out.println("Depot Downloader: Item Removed: " + item.getAppId() + ", index: " + index); + public void onItemAdded(int appId) { + System.out.println("Depot Downloader: Item Added: " + appId); System.out.println(" ---- "); } @Override - public void onQueueCleared(@NotNull List previousItems) { - System.out.println("Depot Downloader: Queue size of " + previousItems.size() + " cleared"); + public void onDownloadStarted(int appId) { + System.out.println("Depot Downloader: Download started for item: " + appId); System.out.println(" ---- "); } @Override - public void onDownloadStarted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download started for item: " + item.getAppId()); + public void onDownloadCompleted(int appId) { + System.out.println("Depot Downloader: Download completed for item: " + appId); System.out.println(" ---- "); } @Override - public void onDownloadCompleted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download completed for item: " + item.getAppId()); - System.out.println(" ---- "); - } - - @Override - public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { - System.out.println("Depot Downloader: Download failed for item: " + item.getAppId()); + public void onDownloadFailed(int appId, @NotNull Throwable error) { + System.out.println("Depot Downloader: Download failed for item: " + appId); System.err.println(error.getMessage()); System.out.println(" ---- "); } @@ -372,9 +363,8 @@ public void onOverallProgress(@NotNull OverallProgress progress) { } @Override - public void onDepotProgress(int depotId, @NotNull DepotProgress progress) { + public void onDepotProgress(@NotNull DepotProgress progress) { System.out.println("Depot Downloader: Depot Progress"); - System.out.println("depotId: " + depotId); System.out.println("depotId: " + progress.getDepotId()); System.out.println("filesCompleted: " + progress.getFilesCompleted()); System.out.println("totalFiles: " + progress.getTotalFiles()); @@ -386,10 +376,9 @@ public void onDepotProgress(int depotId, @NotNull DepotProgress progress) { } @Override - public void onFileProgress(int depotId, @NotNull String fileName, @NotNull FileProgress progress) { + public void onFileProgress(@NotNull FileProgress progress) { System.out.println("Depot Downloader: File Progress"); - System.out.println("depotId: " + depotId); - System.out.println("fileName: " + fileName); + System.out.println("depotId: " + progress.getDepotId()); System.out.println("fileName: " + progress.getFileName()); System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); System.out.println("totalBytes: " + progress.getTotalBytes()); @@ -405,10 +394,4 @@ public void onStatusUpdate(@NotNull String message) { System.out.println("Depot Downloader: Status Message: " + message); System.out.println(" ---- "); } - - @Override - public void onAndroidEmulation(boolean value) { - System.out.println("Depot Downloader: Android Emulation: " + value); - System.out.println(" ---- "); - } } From 8177451eaac835c709a012e7490096053aa87383 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 5 Oct 2025 02:23:15 -0500 Subject: [PATCH 08/21] Simplify SampleDownloadApp --- .../_023_downloadapp/SampleDownloadApp.java | 48 +++++++------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index f7abc981..9d25aaf1 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -9,8 +9,6 @@ import in.dragonbra.javasteam.steam.authentication.AuthenticationException; import in.dragonbra.javasteam.steam.authentication.UserConsoleAuthenticator; import in.dragonbra.javasteam.steam.handlers.steamapps.License; -import in.dragonbra.javasteam.steam.handlers.steamapps.SteamApps; -import in.dragonbra.javasteam.steam.handlers.steamapps.callback.FreeLicenseCallback; import in.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback; import in.dragonbra.javasteam.steam.handlers.steamuser.LogOnDetails; import in.dragonbra.javasteam.steam.handlers.steamuser.SteamUser; @@ -42,27 +40,21 @@ *

* this sample introduces the usage of the content downloader API *

- * content downloader lets you download an app from a Steam depot given - * an app ID + * content downloader lets you download an app, pub file, or ugc item given some parameters. *

- * in this case, this sample will demonstrate how to download the free game - * called Rocky Mayhem + * in this case, this sample will ask which game app id you'd like to download. + * You can find the app id of a game by the url of the store page. + * For example "store.steampowered.com/app/1303350/Rocky_Mayhem/", where 1303350 is the app id. */ @SuppressWarnings("FieldCanBeLocal") public class SampleDownloadApp implements Runnable, IDownloadListener { - private final int ROCKY_MAYHEM_APP_ID = 1303350; - - private final String DEFAULT_INSTALL_DIRECTORY = "steamapps"; - private SteamClient steamClient; private CallbackManager manager; private SteamUser steamUser; - private SteamApps steamApps; - private boolean isRunning; private final String user; @@ -100,8 +92,6 @@ public void run() { steamUser = steamClient.getHandler(SteamUser.class); - steamApps = steamClient.getHandler(SteamApps.class); - subscriptions = new ArrayList<>(); subscriptions.add(manager.subscribe(ConnectedCallback.class, this::onConnected)); @@ -109,7 +99,6 @@ public void run() { subscriptions.add(manager.subscribe(LoggedOnCallback.class, this::onLoggedOn)); subscriptions.add(manager.subscribe(LoggedOffCallback.class, this::onLoggedOff)); subscriptions.add(manager.subscribe(LicenseListCallback.class, this::onLicenseList)); - subscriptions.add(manager.subscribe(FreeLicenseCallback.class, this::onFreeLicense)); isRunning = true; @@ -121,9 +110,9 @@ public void run() { manager.runWaitCallbacks(1000L); } + System.out.println("Closing " + subscriptions.size() + " subscriptions."); for (var subscription : subscriptions) { try { - System.out.println("Closing: " + subscription.getClass().getName()); subscription.close(); } catch (IOException e) { System.out.println("Couldn't close a callback."); @@ -223,8 +212,9 @@ private void onLoggedOn(LoggedOnCallback callback) { System.out.println("Successfully logged on!"); - // now that we are logged in, we can request a free license for Rocky Mayhem - steamApps.requestFreeLicense(ROCKY_MAYHEM_APP_ID); + // at this point, we'd be able to perform actions on Steam + + // The sample continues in onLicenseList } private void onLicenseList(LicenseListCallback callback) { @@ -237,15 +227,17 @@ private void onLicenseList(LicenseListCallback callback) { licenseList = callback.getLicenseList(); System.out.println("Got " + licenseList.size() + " licenses from account!"); + + downloadApp(); } - private void onFreeLicense(FreeLicenseCallback callback) { - if (callback.getResult() != EResult.OK) { - System.out.println("Failed to get a free license for Rocky Mayhem"); - steamClient.disconnect(); - return; - } + private void onLoggedOff(LoggedOffCallback callback) { + System.out.println("Logged off of Steam: " + callback.getResult()); + isRunning = false; + } + + private void downloadApp() { // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. // You will need to subscribe to LicenseListCallback to obtain your app licenses. try (var depotDownloader = new DepotDownloader(steamClient, licenseList, false)) { @@ -273,7 +265,7 @@ private void onFreeLicense(FreeLicenseCallback callback) { var appItem = new AppItem( /* appId */ 204360, /* (Optional) installToGameNameDirectory */ true, - /* (Optional) installDirectory */ DEFAULT_INSTALL_DIRECTORY, + /* (Optional) installDirectory */ "steamapps", /* (Optional) branch */ "public", /* (Optional) branchPassword */ "", /* (Optional) downloadAllPlatforms */ false, @@ -317,12 +309,6 @@ private void onFreeLicense(FreeLicenseCallback callback) { } } - private void onLoggedOff(LoggedOffCallback callback) { - System.out.println("Logged off of Steam: " + callback.getResult()); - - isRunning = false; - } - // Depot Downloader Callbacks. @Override From 0fb0e95aad717ae66c6d470d0545d1d8ab007928 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 5 Oct 2025 19:34:54 -0500 Subject: [PATCH 09/21] A litte more tidying --- .../depotdownloader/CDNClientPool.kt | 42 ++++--- .../depotdownloader/DepotDownloader.kt | 83 +++++++------ .../javasteam/depotdownloader/HttpClient.kt | 29 +++-- .../depotdownloader/IDownloadListener.kt | 17 +-- .../depotdownloader/data/DownloadItems.kt | 12 +- .../_023_downloadapp/SampleDownloadApp.java | 112 +++++++++++------- .../in/dragonbra/javasteam/types/FileData.kt | 29 ++++- 7 files changed, 210 insertions(+), 114 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 58ac5366..bd24d98d 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -7,6 +7,8 @@ import `in`.dragonbra.javasteam.util.log.Logger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference /** * Manages a pool of CDN server connections for efficient content downloading. @@ -32,9 +34,9 @@ class CDNClientPool( private var logger: Logger? = null - private val servers: ArrayList = arrayListOf() + private val servers = AtomicReference>(emptyList()) - private var nextServer: Int = 0 + private var nextServer: AtomicInteger = AtomicInteger(0) private val mutex: Mutex = Mutex() @@ -53,7 +55,7 @@ class CDNClientPool( } override fun close() { - servers.clear() + servers.set(emptyList()) cdnClient = null proxyServer = null @@ -63,10 +65,6 @@ class CDNClientPool( @Throws(Exception::class) suspend fun updateServerList(maxNumServers: Int? = null) = mutex.withLock { - if (servers.isNotEmpty()) { - servers.clear() - } - val serversForSteamPipe = steamSession.steamContent!!.getServersForSteamPipe( cellId = steamSession.steamClient.cellID ?: 0, maxNumServers = maxNumServers, @@ -84,27 +82,32 @@ class CDNClientPool( // ContentServerPenalty removed for now. - servers.addAll(weightedCdnServers) + servers.set(weightedCdnServers) + + nextServer.set(0) // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } - logger?.debug("Found ${servers.size} Servers") + logger?.debug("Found ${weightedCdnServers.size} Servers") - if (servers.isEmpty()) { + if (weightedCdnServers.isEmpty()) { throw Exception("Failed to retrieve any download servers.") } } - suspend fun getConnection(): Server = mutex.withLock { - val server = servers[nextServer % servers.count()] + fun getConnection(): Server { + val servers = servers.get() + + val index = nextServer.getAndIncrement() + val server = servers[index % servers.size] logger?.debug("Getting connection $server") return server } - suspend fun returnConnection(server: Server?) = mutex.withLock { + fun returnConnection(server: Server?) { if (server == null) { - return@withLock + return } logger?.debug("Returning connection: $server") @@ -112,15 +115,18 @@ class CDNClientPool( // (SK) nothing to do, maybe remove from ContentServerPenalty? } - suspend fun returnBrokenConnection(server: Server?) = mutex.withLock { + fun returnBrokenConnection(server: Server?) { if (server == null) { - return@withLock + return } logger?.debug("Returning broken connection: $server") - if (servers[nextServer % servers.count()] == server) { - nextServer++ + val servers = servers.get() + val currentIndex = nextServer.get() + + if (servers.isNotEmpty() && servers[currentIndex % servers.size] == server) { + nextServer.incrementAndGet() // TODO: (SK) Add server to ContentServerPenalty } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 481a6a6a..f909e123 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -125,22 +125,30 @@ class DepotDownloader @JvmOverloads constructor( val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" } + private val activeDownloads = AtomicInteger(0) + private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } - private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - private val activeDownloads = AtomicInteger(0) - private val listeners = CopyOnWriteArrayList() - private var logger: Logger? = null - private var processingChannel = Channel(Channel.UNLIMITED) + private val httpClient: HttpClient by lazy { HttpClient(maxConnections = maxDownloads) } + private val lastFileProgressUpdate = ConcurrentHashMap() + + private val listeners = CopyOnWriteArrayList() + private val progressUpdateInterval = 500L // ms - private var steam3: Steam3Session? = null + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private var cdnClientPool: CDNClientPool? = null private var config: Config = Config(androidEmulation = androidEmulation) + private var logger: Logger? = null + + private var processingChannel = Channel(Channel.UNLIMITED) + + private var steam3: Steam3Session? = null + // region [REGION] Private data classes. private data class NetworkChunkItem( @@ -189,7 +197,7 @@ class DepotDownloader @JvmOverloads constructor( // region [REGION] Downloading Operations @Throws(IllegalStateException::class) - suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { + private suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { val details = requireNotNull( steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) ) { "Pub File Null" } @@ -212,7 +220,7 @@ class DepotDownloader @JvmOverloads constructor( } } - suspend fun downloadUGC( + private suspend fun downloadUGC( appId: Int, ugcId: Long, ) { @@ -245,7 +253,7 @@ class DepotDownloader @JvmOverloads constructor( } @Throws(IllegalStateException::class, IOException::class) - suspend fun downloadWebFile(appId: Int, fileName: String, url: String) { + private suspend fun downloadWebFile(appId: Int, fileName: String, url: String) { val (success, installDir) = createDirectories(appId, 0, appId) if (!success) { @@ -260,7 +268,7 @@ class DepotDownloader @JvmOverloads constructor( filesystem.createDirectories(fileFinalPath.parent!!) filesystem.createDirectories(fileStagingPath.parent!!) - HttpClient.getClient(maxDownloads).use { client -> + httpClient.getClient().use { client -> logger?.debug("Starting download of $fileName...") val response = client.get(url) @@ -305,7 +313,7 @@ class DepotDownloader @JvmOverloads constructor( // L4D2 (app) supports LV @Throws(IllegalStateException::class) - suspend fun downloadApp( + private suspend fun downloadApp( appId: Int, depotManifestIds: List>, branch: String, @@ -1210,6 +1218,8 @@ class DepotDownloader @JvmOverloads constructor( val matchingChunks = arrayListOf() file.chunks.forEach { chunk -> + ensureActive() + val oldChunk = oldManifestFile.chunks.firstOrNull { c -> c.chunkID.contentEquals(chunk.chunkID) } @@ -1227,6 +1237,8 @@ class DepotDownloader @JvmOverloads constructor( filesystem.openReadOnly(fileFinalPath).use { handle -> orderedChunks.forEach { match -> + ensureActive() + // Read the chunk data into a byte array val length = match.oldChunk.uncompressedLength val buffer = ByteArray(length) @@ -1266,6 +1278,8 @@ class DepotDownloader @JvmOverloads constructor( } for (match in copyChunks) { + ensureActive() + val tmp = ByteArray(match.oldChunk.uncompressedLength) oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) @@ -1383,7 +1397,8 @@ class DepotDownloader @JvmOverloads constructor( var connection: Server? = null try { - connection = cdnClientPool!!.getConnection() + connection = cdnClientPool?.getConnection() + ?: throw IllegalStateException("ContentDownloader already closed") var cdnToken: String? = null @@ -1566,9 +1581,14 @@ class DepotDownloader @JvmOverloads constructor( */ fun add(item: DownloadItem) { runBlocking { - processingChannel.send(item) - activeDownloads.incrementAndGet() - notifyListeners { it.onItemAdded(item.appId) } + try { + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item) } + } catch (e: Exception) { + logger?.error(e) + throw e + } } } @@ -1577,10 +1597,15 @@ class DepotDownloader @JvmOverloads constructor( */ fun addAll(items: List) { runBlocking { - items.forEach { item -> - processingChannel.send(item) - activeDownloads.incrementAndGet() - notifyListeners { it.onItemAdded(item.appId) } + try { + items.forEach { item -> + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item) } + } + } catch (e: Exception) { + logger?.error(e) + throw e } } } @@ -1622,22 +1647,14 @@ class DepotDownloader @JvmOverloads constructor( when (item) { is PubFileItem -> { - if (item.pubfile == INVALID_MANIFEST_ID) { - logger?.debug("Invalid Pub File ID for ${item.appId}") - continue - } logger?.debug("Downloading PUB File for ${item.appId}") - notifyListeners { it.onDownloadStarted(item.appId) } + notifyListeners { it.onDownloadStarted(item) } downloadPubFile(item.appId, item.pubfile) } is UgcItem -> { - if (item.ugcId == INVALID_MANIFEST_ID) { - logger?.debug("Invalid UGC ID for ${item.appId}") - continue - } logger?.debug("Downloading UGC File for ${item.appId}") - notifyListeners { it.onDownloadStarted(item.appId) } + notifyListeners { it.onDownloadStarted(item) } downloadUGC(item.appId, item.ugcId) } @@ -1689,7 +1706,7 @@ class DepotDownloader @JvmOverloads constructor( } logger?.debug("Downloading App for ${item.appId}") - notifyListeners { it.onDownloadStarted(item.appId) } + notifyListeners { it.onDownloadStarted(item) } downloadApp( appId = item.appId, depotManifestIds = depotManifestIds, @@ -1703,10 +1720,10 @@ class DepotDownloader @JvmOverloads constructor( } } - notifyListeners { it.onDownloadCompleted(item.appId) } + notifyListeners { it.onDownloadCompleted(item) } } catch (e: Exception) { logger?.error("Error downloading item ${item.appId}: ${e.message}", e) - notifyListeners { it.onDownloadFailed(item.appId, e) } + notifyListeners { it.onDownloadFailed(item, e) } } finally { activeDownloads.decrementAndGet() } @@ -1718,7 +1735,7 @@ class DepotDownloader @JvmOverloads constructor( scope.cancel("DepotDownloader Closing") - HttpClient.close() + httpClient.close() lastFileProgressUpdate.clear() listeners.clear() diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 75e4f612..67f35d48 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -6,21 +6,36 @@ import io.ktor.client.engine.cio.CIO import io.ktor.client.engine.cio.endpoint import io.ktor.client.plugins.UserAgent import kotlinx.coroutines.isActive +import java.io.Closeable /** - * Singleton HTTP client for content downloader operations. - * Provides a shared, configured Ktor HTTP client optimized for Steam CDN downloads. - * The client is lazily initialized on first use and reused across all download operations. - * Connection pooling and timeouts are configured based on the maximum concurrent downloads. + * HTTP client wrapper for content downloader operations. + * + * Provides a configured Ktor HTTP client optimized for Steam CDN downloads. + * Each instance maintains its own connection pool based on the specified + * maximum concurrent connections. + * + * @param maxConnections Maximum number of concurrent connections * * @author Lossy * @since Oct 1, 2025 */ -object HttpClient { +class HttpClient( + private val maxConnections: Int, +) : Closeable { private var httpClient: HttpClient? = null - fun getClient(maxConnections: Int = 8): HttpClient { + /** + * Returns the HTTP client instance, creating it lazily on first access. + * + * The client is configured with: + * - Custom User-Agent identifying JavaSteam DepotDownloader + * - Connection pooling based on [maxConnections] + * - 5 second keep-alive and connect timeout + * - 30 second request timeout + */ + fun getClient(): HttpClient { if (httpClient?.isActive != true) { httpClient = HttpClient(CIO) { install(UserAgent) { @@ -42,7 +57,7 @@ object HttpClient { return httpClient!! } - fun close() { + override fun close() { if (httpClient?.isActive == true) { httpClient?.close() httpClient = null diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt index 9b4ab7f2..072ea85a 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -1,6 +1,7 @@ package `in`.dragonbra.javasteam.depotdownloader import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress @@ -17,31 +18,31 @@ interface IDownloadListener { /** * Called when an item is added to the download queue. * - * @param appId The application ID that was queued + * @param item The [DownloadItem] that was queued */ - fun onItemAdded(appId: Int) {} + fun onItemAdded(item: DownloadItem) {} /** * Called when a download begins processing. * - * @param appId The application ID being downloaded + * @param item The [DownloadItem] being downloaded */ - fun onDownloadStarted(appId: Int) {} + fun onDownloadStarted(item: DownloadItem) {} /** * Called when a download completes successfully. * - * @param appId The application ID that finished downloading + * @param item The [DownloadItem] that finished downloading */ - fun onDownloadCompleted(appId: Int) {} + fun onDownloadCompleted(item: DownloadItem) {} /** * Called when a download fails with an error. * - * @param appId The application ID that failed + * @param item The [DownloadItem] that failed * @param error The exception that caused the failure */ - fun onDownloadFailed(appId: Int, error: Throwable) {} + fun onDownloadFailed(item: DownloadItem, error: Throwable) {} /** * Called periodically with overall download progress across all items. diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt index e94ecdd6..1c6baa00 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -2,6 +2,8 @@ package `in`.dragonbra.javasteam.depotdownloader.data // https://kotlinlang.org/docs/coding-conventions.html#source-file-organization +// TODO should these be a builder pattern for Java users? + /** * Base class for downloadable Steam content items. * @property appId The Steam application ID @@ -16,6 +18,7 @@ abstract class DownloadItem( val appId: Int, val installDirectory: String?, val installToGameNameDirectory: Boolean, + val verify: Boolean, // TODO val downloadManifestOnly: Boolean, ) @@ -32,8 +35,9 @@ class UgcItem @JvmOverloads constructor( val ugcId: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, + verify: Boolean = false, downloadManifestOnly: Boolean = false, -) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) /** * Represents a Steam published file for download. @@ -48,8 +52,9 @@ class PubFileItem @JvmOverloads constructor( val pubfile: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, + verify: Boolean = false, downloadManifestOnly: Boolean = false, -) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) /** * Represents a Steam application/game for download from a depot. @@ -84,5 +89,6 @@ class AppItem @JvmOverloads constructor( val lowViolence: Boolean = false, val depot: List = emptyList(), val manifest: List = emptyList(), + verify: Boolean = false, downloadManifestOnly: Boolean = false, -) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index 9d25aaf1..bcb28c2b 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -146,7 +146,7 @@ private void onConnected(ConnectedCallback callback) { accountName = pollResponse.getAccountName(); refreshToken = pollResponse.getRefreshToken(); - // Save out refresh token for automatic login on next sample run. + // Save our refresh token for automatic login on next sample run. Files.writeString(path, pollResponse.getRefreshToken()); } else { System.out.println("Existing refresh token found"); @@ -251,6 +251,7 @@ private void downloadApp() { /* pubfile */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, + /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false ); // TODO find actual pub item @@ -259,6 +260,7 @@ private void downloadApp() { /* ugcId */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, + /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false ); // TODO find actual ugc item @@ -277,6 +279,7 @@ private void downloadApp() { /* (Optional) lowViolence */ false, /* (Optional) depot */ List.of(), /* (Optional) manifest */ List.of(), + /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false ); @@ -287,7 +290,7 @@ private void downloadApp() { // After 'depotDownloader' is constructed, items added are downloaded in a First-In, First-Out queue on the fly. // Add a singular item to process. - depotDownloader.add(new AppItem(appId, true)); + depotDownloader.add(new AppItem(appId, true, "steamapps")); // You can add a List of items to be processed. // depotDownloader.add(List.of()); @@ -312,72 +315,93 @@ private void downloadApp() { // Depot Downloader Callbacks. @Override - public void onItemAdded(int appId) { - System.out.println("Depot Downloader: Item Added: " + appId); - System.out.println(" ---- "); + public void onItemAdded(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Item Added: " + item.getAppId() + "\n ---- "); } @Override - public void onDownloadStarted(int appId) { - System.out.println("Depot Downloader: Download started for item: " + appId); - System.out.println(" ---- "); + public void onDownloadStarted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download started for item: " + item.getAppId() + "\n ---- "); } @Override - public void onDownloadCompleted(int appId) { - System.out.println("Depot Downloader: Download completed for item: " + appId); - System.out.println(" ---- "); + public void onDownloadCompleted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download completed for item: " + item.getAppId() + "\n ---- "); } @Override - public void onDownloadFailed(int appId, @NotNull Throwable error) { - System.out.println("Depot Downloader: Download failed for item: " + appId); - System.err.println(error.getMessage()); - System.out.println(" ---- "); + public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { + System.out.println("Depot Downloader: Download failed for item: " + item.getAppId() + "\n ---- "); + if (!error.getMessage().isEmpty()) { + System.err.println(error.getMessage()); + } } @Override public void onOverallProgress(@NotNull OverallProgress progress) { - System.out.println("Depot Downloader: Overall Progress"); - System.out.println("currentItem: " + progress.getCurrentItem()); - System.out.println("totalItems: " + progress.getTotalItems()); - System.out.println("totalBytesDownloaded: " + progress.getTotalBytesDownloaded()); - System.out.println("totalBytesExpected: " + progress.getTotalBytesExpected()); - System.out.println("status: " + progress.getStatus()); - System.out.println("percentComplete: " + progress.getPercentComplete()); - System.out.println(" ---- "); + System.out.printf( + "Depot Downloader: Overall Progress\n" + + "currentItem: %d\n" + + "totalItems: %d\n" + + "totalBytesDownloaded: %d\n" + + "totalBytesExpected: %d\n" + + "status: %s\n" + + "percentComplete: %.2f\n ---- %n \n", + progress.getCurrentItem(), + progress.getTotalItems(), + progress.getTotalBytesDownloaded(), + progress.getTotalBytesExpected(), + progress.getStatus(), + progress.getPercentComplete() + ); } @Override public void onDepotProgress(@NotNull DepotProgress progress) { - System.out.println("Depot Downloader: Depot Progress"); - System.out.println("depotId: " + progress.getDepotId()); - System.out.println("filesCompleted: " + progress.getFilesCompleted()); - System.out.println("totalFiles: " + progress.getTotalFiles()); - System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); - System.out.println("totalBytes: " + progress.getTotalBytes()); - System.out.println("status: " + progress.getStatus()); - System.out.println("percentComplete: " + progress.getPercentComplete()); - System.out.println(" ---- "); + System.out.printf( + "Depot Downloader: Depot Progress\n" + + "depotId: %d\n" + + "filesCompleted: %d\n" + + "totalFiles: %d\n" + + "bytesDownloaded: %d\n" + + "totalBytes: %d\n" + + "status: %s\n" + + "percentComplete: %.2f\n ---- %n \n", + progress.getDepotId(), + progress.getFilesCompleted(), + progress.getTotalFiles(), + progress.getBytesDownloaded(), + progress.getTotalBytes(), + progress.getStatus(), + progress.getPercentComplete() + ); } @Override public void onFileProgress(@NotNull FileProgress progress) { - System.out.println("Depot Downloader: File Progress"); - System.out.println("depotId: " + progress.getDepotId()); - System.out.println("fileName: " + progress.getFileName()); - System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); - System.out.println("totalBytes: " + progress.getTotalBytes()); - System.out.println("chunksCompleted: " + progress.getChunksCompleted()); - System.out.println("totalChunks: " + progress.getTotalChunks()); - System.out.println("status: " + progress.getStatus()); - System.out.println("percentComplete: " + progress.getPercentComplete()); - System.out.println(" ---- "); + System.out.printf( + "Depot Downloader: File Progress\n" + + "depotId: %d\n" + + "fileName: %s\n" + + "bytesDownloaded: %d\n" + + "totalBytes: %d\n" + + "chunksCompleted: %d\n" + + "totalChunks: %d\n" + + "status: %s\n" + + "percentComplete: %.2f\n ---- %n \n", + progress.getDepotId(), + progress.getFileName(), + progress.getBytesDownloaded(), + progress.getTotalBytes(), + progress.getChunksCompleted(), + progress.getTotalChunks(), + progress.getStatus(), + progress.getPercentComplete() + ); } @Override public void onStatusUpdate(@NotNull String message) { - System.out.println("Depot Downloader: Status Message: " + message); - System.out.println(" ---- "); + System.out.println("Depot Downloader: Status Message: " + message + "\n ---- "); } } diff --git a/src/main/java/in/dragonbra/javasteam/types/FileData.kt b/src/main/java/in/dragonbra/javasteam/types/FileData.kt index 53a7c2f4..41ee3ead 100644 --- a/src/main/java/in/dragonbra/javasteam/types/FileData.kt +++ b/src/main/java/in/dragonbra/javasteam/types/FileData.kt @@ -17,7 +17,6 @@ import java.util.EnumSet * @param fileHash Gets SHA-1 hash of this file. * @param linkTarget Gets symlink target of this file. */ -@Suppress("ArrayInDataClass") data class FileData( var fileName: String = "", var fileNameHash: ByteArray = byteArrayOf(), @@ -48,4 +47,32 @@ data class FileData( fileHash = hash, linkTarget = linkTarget, ) + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as FileData + + if (totalSize != other.totalSize) return false + if (fileName != other.fileName) return false + if (!fileNameHash.contentEquals(other.fileNameHash)) return false + if (chunks != other.chunks) return false + if (flags != other.flags) return false + if (!fileHash.contentEquals(other.fileHash)) return false + if (linkTarget != other.linkTarget) return false + + return true + } + + override fun hashCode(): Int { + var result = totalSize.hashCode() + result = 31 * result + fileName.hashCode() + result = 31 * result + fileNameHash.contentHashCode() + result = 31 * result + chunks.hashCode() + result = 31 * result + flags.hashCode() + result = 31 * result + fileHash.contentHashCode() + result = 31 * result + (linkTarget?.hashCode() ?: 0) + return result + } } From cc5e3cc4ab49652e07dab1fc7e143a68e58ebf98 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 11:25:55 -0500 Subject: [PATCH 10/21] Fix getting sha file --- .../main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt index 9d2bc32f..a46951a0 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -99,7 +99,7 @@ object Util { if (FileSystem.SYSTEM.exists(filename)) { val expectedChecksum = try { - FileSystem.SYSTEM.read(filename / ".sha") { + FileSystem.SYSTEM.read("$filename.sha".toPath()) { readByteArray() } } catch (e: IOException) { From 87d58c171c7beec59c6a51649f17d8da582293c6 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 14:54:38 -0500 Subject: [PATCH 11/21] Simplify IDownloadListener, fix up DepotDownloader yielding. Update SampleDownloadApp --- .../depotdownloader/DepotDownloader.kt | 444 ++++++++---------- .../depotdownloader/IDownloadListener.kt | 50 +- .../depotdownloader/Steam3Session.kt | 7 +- .../depotdownloader/data/DownloadItems.kt | 2 +- .../depotdownloader/data/DownloadProgress.kt | 111 ----- .../_023_downloadapp/SampleDownloadApp.java | 128 ++--- 6 files changed, 243 insertions(+), 499 deletions(-) delete mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index f909e123..415fec72 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -5,13 +5,9 @@ import `in`.dragonbra.javasteam.depotdownloader.data.ChunkMatch import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadCounter import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo import `in`.dragonbra.javasteam.depotdownloader.data.DepotFilesData -import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem -import `in`.dragonbra.javasteam.depotdownloader.data.DownloadStatus -import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress import `in`.dragonbra.javasteam.depotdownloader.data.FileStreamData import `in`.dragonbra.javasteam.depotdownloader.data.GlobalDownloadCounter -import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress import `in`.dragonbra.javasteam.depotdownloader.data.PubFileItem import `in`.dragonbra.javasteam.depotdownloader.data.UgcItem import `in`.dragonbra.javasteam.enums.EAccountType @@ -38,7 +34,6 @@ import io.ktor.client.request.get import io.ktor.client.statement.bodyAsChannel import io.ktor.http.HttpHeaders import io.ktor.utils.io.core.readAvailable -import io.ktor.utils.io.core.remaining import io.ktor.utils.io.readRemaining import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope @@ -50,7 +45,6 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay import kotlinx.coroutines.ensureActive import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking @@ -125,14 +119,12 @@ class DepotDownloader @JvmOverloads constructor( val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" } - private val activeDownloads = AtomicInteger(0) + private val completionFuture = CompletableFuture() private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } private val httpClient: HttpClient by lazy { HttpClient(maxConnections = maxDownloads) } - private val lastFileProgressUpdate = ConcurrentHashMap() - private val listeners = CopyOnWriteArrayList() private val progressUpdateInterval = 500L // ms @@ -280,17 +272,16 @@ class DepotDownloader @JvmOverloads constructor( filesystem.sink(fileStagingPath).buffer().use { sink -> val buffer = Buffer() + val tempArray = ByteArray(DEFAULT_BUFFER_SIZE) + while (!channel.isClosedForRead) { val packet = channel.readRemaining(DEFAULT_BUFFER_SIZE.toLong()) if (!packet.exhausted()) { - // Read from Ktor packet into Okio buffer - val bytesRead = packet.remaining.toInt() - val tempArray = ByteArray(bytesRead) - packet.readAvailable(tempArray) - buffer.write(tempArray) - - // Write from buffer to sink - sink.writeAll(buffer) + val bytesRead = packet.readAvailable(tempArray, 0, tempArray.size) + if (bytesRead > 0) { + buffer.write(tempArray, 0, bytesRead) + sink.writeAll(buffer) + } } } } @@ -368,14 +359,11 @@ class DepotDownloader @JvmOverloads constructor( logger?.debug("Using app branch: $branch") depots?.children?.forEach { depotSection -> - @Suppress("VariableInitializerIsRedundant") - var id = INVALID_DEPOT_ID - if (depotSection.children.isEmpty()) { return@forEach } - id = depotSection.name?.toIntOrNull() ?: return@forEach + val id: Int = depotSection.name?.toIntOrNull() ?: return@forEach if (hasSpecificDepots && !depotIdsExpected.contains(id)) { return@forEach @@ -780,15 +768,13 @@ class DepotDownloader @JvmOverloads constructor( val depotsToDownload = ArrayList(depots.size) val allFileNamesAllDepots = hashSetOf() - var completedDepots = 0 - // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup depots.forEach { depot -> val depotFileData = processDepotManifestAndFiles(depot, downloadCounter) if (depotFileData != null) { depotsToDownload.add(depotFileData) - allFileNamesAllDepots.union(depotFileData.allFileNames) + allFileNamesAllDepots.addAll(depotFileData.allFileNames) } ensureActive() @@ -807,22 +793,6 @@ class DepotDownloader @JvmOverloads constructor( depotsToDownload.forEach { depotFileData -> downloadSteam3DepotFiles(downloadCounter, depotFileData, allFileNamesAllDepots) - - completedDepots++ - - val snapshot = synchronized(downloadCounter) { - OverallProgress( - currentItem = completedDepots, - totalItems = depotsToDownload.size, - totalBytesDownloaded = downloadCounter.totalBytesUncompressed, - totalBytesExpected = downloadCounter.completeDownloadSize, - status = DownloadStatus.DOWNLOADING - ) - } - - notifyListeners { listener -> - listener.onOverallProgress(progress = snapshot) - } } logger?.debug( @@ -993,7 +963,11 @@ class DepotDownloader @JvmOverloads constructor( val allFileNames = HashSet(filesAfterExclusions.size) // Pre-process - filesAfterExclusions.forEach { file -> + filesAfterExclusions.forEachIndexed { index, file -> + if (index % 50 == 0) { + ensureActive() // Check cancellation periodically + } + allFileNames.add(file.fileName) val fileFinalPath = depot.installDir / file.fileName @@ -1038,89 +1012,40 @@ class DepotDownloader @JvmOverloads constructor( val networkChunkQueue = Channel(Channel.UNLIMITED) try { - val filesCompleted = AtomicInteger(0) - val lastReportedProgress = AtomicInteger(0) coroutineScope { // First parallel loop - process files and enqueue chunks - files.map { file -> - async { - yield() // Does this matter if its before? - downloadSteam3DepotFile( - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = file, - networkChunkQueue = networkChunkQueue - ) - - val completed = filesCompleted.incrementAndGet() - if (completed % 10 == 0 || completed == files.size) { - val snapshot = synchronized(depotCounter) { - DepotProgress( - depotId = depot.depotId, - filesCompleted = completed, - totalFiles = files.size, - bytesDownloaded = depotCounter.sizeDownloaded, - totalBytes = depotCounter.completeDownloadSize, - status = DownloadStatus.PREPARING // Changed from DOWNLOADING - ) - } + files.chunked(50).forEach { batch -> + yield() - val lastReported = lastReportedProgress.get() - if (completed > lastReported && - lastReportedProgress.compareAndSet( - lastReported, - completed - ) - ) { - notifyListeners { listener -> - listener.onDepotProgress(snapshot) - } - } + batch.map { file -> + async { + downloadSteam3DepotFile( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = file, + networkChunkQueue = networkChunkQueue + ) } - } - }.awaitAll() + }.awaitAll() + } // Close the channel to signal no more items will be added networkChunkQueue.close() - // After all files allocated, send one update showing preparation complete - val progressReporter = launch { - while (true) { - delay(1000) - val snapshot = synchronized(depotCounter) { - DepotProgress( - depotId = depot.depotId, - filesCompleted = files.size, - totalFiles = files.size, - bytesDownloaded = depotCounter.sizeDownloaded, - totalBytes = depotCounter.completeDownloadSize, - status = DownloadStatus.DOWNLOADING + // Second parallel loop - process chunks from queue + List(maxDownloads) { + async { + for (item in networkChunkQueue) { + downloadSteam3DepotFileChunk( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = item.fileData, + fileStreamData = item.fileStreamData, + chunk = item.chunk ) } - notifyListeners { listener -> - listener.onDepotProgress(snapshot) - } } - } - - // Second parallel loop - process chunks from queue - try { - List(maxDownloads) { - async { - for (item in networkChunkQueue) { - downloadSteam3DepotFileChunk( - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = item.fileData, - fileStreamData = item.fileStreamData, - chunk = item.chunk - ) - } - } - }.awaitAll() - } finally { - progressReporter.cancel() - } + }.awaitAll() } } finally { if (!networkChunkQueue.isClosedForSend) { @@ -1158,6 +1083,16 @@ class DepotDownloader @JvmOverloads constructor( DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = depot.manifestId DepotConfigStore.save() + + // Notify depot completion + notifyListeners { listener -> + listener.onDepotCompleted( + depotId = depot.depotId, + compressedBytes = depotCounter.depotBytesCompressed, + uncompressedBytes = depotCounter.depotBytesUncompressed + ) + } + logger?.debug("Depot ${depot.depotId} - Downloaded ${depotCounter.depotBytesCompressed} bytes (${depotCounter.depotBytesUncompressed} bytes uncompressed)") } @@ -1218,7 +1153,7 @@ class DepotDownloader @JvmOverloads constructor( val matchingChunks = arrayListOf() file.chunks.forEach { chunk -> - ensureActive() + yield() val oldChunk = oldManifestFile.chunks.firstOrNull { c -> c.chunkID.contentEquals(chunk.chunkID) @@ -1237,7 +1172,7 @@ class DepotDownloader @JvmOverloads constructor( filesystem.openReadOnly(fileFinalPath).use { handle -> orderedChunks.forEach { match -> - ensureActive() + yield() // Read the chunk data into a byte array val length = match.oldChunk.uncompressedLength @@ -1266,28 +1201,24 @@ class DepotDownloader @JvmOverloads constructor( if (!hashMatches || neededChunks.isNotEmpty()) { filesystem.atomicMove(fileFinalPath, fileStagingPath) - try { - filesystem.openReadOnly(fileStagingPath).use { oldHandle -> - filesystem.openReadWrite(fileFinalPath).use { newHandle -> - try { - newHandle.resize(file.totalSize) - } catch (ex: IOException) { - throw DepotDownloaderException( - "Failed to resize file to expected size $fileFinalPath: ${ex.message}" - ) - } - - for (match in copyChunks) { - ensureActive() - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) - newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) - } + filesystem.openReadOnly(fileStagingPath).use { oldHandle -> + filesystem.openReadWrite(fileFinalPath).use { newHandle -> + try { + newHandle.resize(file.totalSize) + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to resize file to expected size $fileFinalPath: ${ex.message}" + ) + } + + for (match in copyChunks) { + ensureActive() + + val tmp = ByteArray(match.oldChunk.uncompressedLength) + oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) + newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) } } - } catch (e: Exception) { - logger?.error(e) } filesystem.delete(fileStagingPath) @@ -1390,149 +1321,137 @@ class DepotDownloader @JvmOverloads constructor( var written = 0 val chunkBuffer = ByteArray(chunk.uncompressedLength) - try { - do { - ensureActive() + do { + ensureActive() - var connection: Server? = null + var connection: Server? = null - try { - connection = cdnClientPool?.getConnection() - ?: throw IllegalStateException("ContentDownloader already closed") + try { + connection = cdnClientPool?.getConnection() + ?: throw IllegalStateException("ContentDownloader already closed") - var cdnToken: String? = null + var cdnToken: String? = null - val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] - if (authTokenCallbackPromise != null) { - try { - val result = authTokenCallbackPromise.await() - cdnToken = result.token - } catch (e: Exception) { - logger?.error("Failed to get CDN auth token: ${e.message}") - } + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") } + } - logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") - - written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( - depotId = depot.depotId, - chunk = chunk, - server = connection, - destination = chunkBuffer, - depotKey = depot.depotKey, - proxyServer = cdnClientPool!!.proxyServer, - cdnAuthToken = cdnToken, - ) + logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") - cdnClientPool!!.returnConnection(connection) + written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( + depotId = depot.depotId, + chunk = chunk, + server = connection, + destination = chunkBuffer, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) - break - } catch (e: CancellationException) { - logger?.error(e) - } catch (e: SteamKitWebRequestException) { - // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, - // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above - if (e.statusCode == 403 && - ( - !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host) || - steam3!!.cdnAuthTokens[depot.depotId to connection.host]?.isCompleted == false - ) - ) { - steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + cdnClientPool!!.returnConnection(connection) - cdnClientPool!!.returnConnection(connection) + break + } catch (e: CancellationException) { + logger?.error(e) + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, + // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above + if (e.statusCode == 403 && + ( + !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host) || + steam3!!.cdnAuthTokens[depot.depotId to connection.host]?.isCompleted == false + ) + ) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) - continue - } + cdnClientPool!!.returnConnection(connection) - cdnClientPool!!.returnBrokenConnection(connection) + continue + } - // Unauthorized || Forbidden - if (e.statusCode == 401 || e.statusCode == 403) { - logger?.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") - break - } + cdnClientPool!!.returnBrokenConnection(connection) - logger?.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") - } catch (e: Exception) { - cdnClientPool!!.returnBrokenConnection(connection) - logger?.error("Encountered unexpected error downloading chunk $chunkID", e) + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") + break } - } while (written == 0) - if (written == 0) { - logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") - cancel() + logger?.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered unexpected error downloading chunk $chunkID", e) } + } while (written == 0) - // Throw the cancellation exception if requested so that this task is marked failed - ensureActive() + if (written == 0) { + logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") + cancel() + } - try { - fileStreamData.fileLock.lock() + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() - if (fileStreamData.fileHandle == null) { - val fileFinalPath = depot.installDir / file.fileName - fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) - } + try { + fileStreamData.fileLock.lock() - fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) - } finally { - fileStreamData.fileLock.unlock() + if (fileStreamData.fileHandle == null) { + val fileFinalPath = depot.installDir / file.fileName + fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) } + + fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) } finally { + fileStreamData.fileLock.unlock() } val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() if (remainingChunks == 0) { fileStreamData.fileHandle?.close() - } - - var sizeDownloaded = 0L - synchronized(depotDownloadCounter) { - sizeDownloaded = depotDownloadCounter.sizeDownloaded + written.toLong() - depotDownloadCounter.sizeDownloaded = sizeDownloaded - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - val now = System.currentTimeMillis() - val fileKey = "${depot.depotId}:${file.fileName}" - val lastUpdate = lastFileProgressUpdate[fileKey] ?: 0L - - if (now - lastUpdate >= progressUpdateInterval || remainingChunks == 0) { - lastFileProgressUpdate[fileKey] = now + // File completed - notify with percentage + val sizeDownloaded = synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + depotDownloadCounter.sizeDownloaded + } - val totalChunks = file.chunks.size - val completedChunks = totalChunks - remainingChunks + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } - // Approximate bytes based on completion ratio - val approximateBytesDownloaded = (file.totalSize * completedChunks) / totalChunks + val fileFinalPath = depot.installDir / file.fileName + val depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) notifyListeners { listener -> - listener.onFileProgress( - progress = FileProgress( - depotId = depot.depotId, - fileName = file.fileName, - bytesDownloaded = approximateBytesDownloaded, - totalBytes = file.totalSize, - chunksCompleted = completedChunks, - totalChunks = totalChunks, - status = if (remainingChunks == 0) DownloadStatus.COMPLETED else DownloadStatus.DOWNLOADING - ) + listener.onFileCompleted( + depotId = depot.depotId, + fileName = fileFinalPath.toString(), + depotPercentComplete = depotPercentage ) } - } - if (remainingChunks == 0) { - val fileFinalPath = depot.installDir / file.fileName - val percentage = (sizeDownloaded / depotDownloadCounter.completeDownloadSize.toFloat()) * 100.0f - logger?.debug("%.2f%% %s".format(percentage, fileFinalPath)) + logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) + } else { + // Just update counters without notifying + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } } } @@ -1583,7 +1502,6 @@ class DepotDownloader @JvmOverloads constructor( runBlocking { try { processingChannel.send(item) - activeDownloads.incrementAndGet() notifyListeners { it.onItemAdded(item) } } catch (e: Exception) { logger?.error(e) @@ -1600,7 +1518,6 @@ class DepotDownloader @JvmOverloads constructor( try { items.forEach { item -> processingChannel.send(item) - activeDownloads.incrementAndGet() notifyListeners { it.onItemAdded(item) } } } catch (e: Exception) { @@ -1611,14 +1528,15 @@ class DepotDownloader @JvmOverloads constructor( } /** - * Get the current queue size of pending items to be downloaded. + * Signals that no more items will be added to the download queue. + * After calling this, the downloader will complete once all queued items finish. + * + * This is called automatically by [close], but you can call it explicitly + * if you want to wait for completion without closing the downloader. */ - fun queueSize(): Int = activeDownloads.get() - - /** - * Get a boolean value if there are items in queue to be downloaded. - */ - fun isProcessing(): Boolean = activeDownloads.get() > 0 + fun finishAdding() { + processingChannel.close() + } // endregion @@ -1649,7 +1567,7 @@ class DepotDownloader @JvmOverloads constructor( is PubFileItem -> { logger?.debug("Downloading PUB File for ${item.appId}") notifyListeners { it.onDownloadStarted(item) } - downloadPubFile(item.appId, item.pubfile) + downloadPubFile(item.appId, item.pubFile) } is UgcItem -> { @@ -1724,10 +1642,25 @@ class DepotDownloader @JvmOverloads constructor( } catch (e: Exception) { logger?.error("Error downloading item ${item.appId}: ${e.message}", e) notifyListeners { it.onDownloadFailed(item, e) } - } finally { - activeDownloads.decrementAndGet() } } + + completionFuture.complete(null) + } + + /** + * Returns a CompletableFuture that completes when all queued downloads finish. + * @return CompletableFuture that completes when all downloads finish + */ + fun getCompletion(): CompletableFuture = completionFuture + + /** + * Blocks the current thread until all queued downloads complete. + * Convenience method that calls `getCompletion().join()`. + * @throws CompletionException if any download fails + */ + fun awaitCompletion() { + completionFuture.join() } override fun close() { @@ -1737,7 +1670,6 @@ class DepotDownloader @JvmOverloads constructor( httpClient.close() - lastFileProgressUpdate.clear() listeners.clear() steam3?.close() diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt index 072ea85a..331f4a72 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -1,9 +1,6 @@ package `in`.dragonbra.javasteam.depotdownloader -import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem -import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress -import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress /** * Listener interface for receiving download progress and status events. @@ -17,64 +14,47 @@ import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress interface IDownloadListener { /** * Called when an item is added to the download queue. - * - * @param item The [DownloadItem] that was queued */ fun onItemAdded(item: DownloadItem) {} /** * Called when a download begins processing. - * - * @param item The [DownloadItem] being downloaded */ fun onDownloadStarted(item: DownloadItem) {} /** * Called when a download completes successfully. - * - * @param item The [DownloadItem] that finished downloading */ fun onDownloadCompleted(item: DownloadItem) {} /** * Called when a download fails with an error. - * - * @param item The [DownloadItem] that failed - * @param error The exception that caused the failure */ fun onDownloadFailed(item: DownloadItem, error: Throwable) {} /** - * Called periodically with overall download progress across all items. - * Reports progress for the entire download queue, including completed - * and remaining items. - * - * @param progress Overall download statistics + * Called during file preparation with informational messages. + * Examples: "Pre-allocating depots\441\file.txt", "Validating file.cab" */ - fun onOverallProgress(progress: OverallProgress) {} - - /** - * Called periodically with progress for a specific depot. - * Reports file allocation and download progress for an individual depot. - * - * @param progress Depot-specific download statistics - */ - fun onDepotProgress(progress: DepotProgress) {} + fun onStatusUpdate(message: String) {} /** - * Called periodically with progress for a specific file. - * Reports chunk-level download progress for individual files. + * Called when a file completes downloading. + * Use this for printing progress like "20.42% depots\441\maps\ctf_haarp.bsp" * - * @param progress File-specific download statistics + * @param depotId The depot being downloaded + * @param fileName Relative file path + * @param depotPercentComplete Overall depot completion percentage (0f to 1f) */ - fun onFileProgress(progress: FileProgress) {} + fun onFileCompleted(depotId: Int, fileName: String, depotPercentComplete: Float) {} /** - * Called with informational status messages during download operations. - * Used for logging or displaying current operations like manifest - * downloads, file validation, and allocation. + * Called when a depot finishes downloading. + * Use this for printing summary like "Depot 228990 - Downloaded X bytes (Y bytes uncompressed)" * - * @param message Human-readable status message + * @param depotId The depot that completed + * @param compressedBytes Bytes transferred (compressed) + * @param uncompressedBytes Actual data size (uncompressed) */ - fun onStatusUpdate(message: String) {} + fun onDepotCompleted(depotId: Int, compressedBytes: Long, uncompressedBytes: Long) {} } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 9a3ebb94..1afcf241 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -152,6 +152,8 @@ class Steam3Session( private val packageInfoMutex = Mutex() suspend fun requestPackageInfo(packageIds: List) { packageInfoMutex.withLock { + logger?.debug("requestPackageInfo() invoked with ${packageIds.size} packageIds") + // I have a silly race condition??? val packages = packageIds.filter { !packageInfo.containsKey(it) } @@ -173,10 +175,7 @@ class Steam3Session( val packageInfoMultiple = steamApps!!.picsGetProductInfo(emptyList(), packageRequests).await() - logger?.debug( - "requestPackageInfo(packageIds =${packageIds.size}) \n" + - "picsGetProductInfo result size: ${packageInfoMultiple.results.size} " - ) + logger?.debug("requestPackageInfo() picsGetProductInfo result size: ${packageInfoMultiple.results.size} ") packageInfoMultiple.results.forEach { pkgInfo -> pkgInfo.packages.forEach { pkgValue -> diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt index 1c6baa00..813e0b52 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -49,7 +49,7 @@ class UgcItem @JvmOverloads constructor( */ class PubFileItem @JvmOverloads constructor( appId: Int, - val pubfile: Long, + val pubFile: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, verify: Boolean = false, diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt deleted file mode 100644 index 2e058560..00000000 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt +++ /dev/null @@ -1,111 +0,0 @@ -package `in`.dragonbra.javasteam.depotdownloader.data - -/** - * Reports overall download progress across all queued items. - * Provides high-level statistics for the entire download session, tracking - * which item is currently processing and cumulative byte transfer. - * - * @property currentItem Number of items completed (1-based) - * @property totalItems Total number of items in the download session - * @property totalBytesDownloaded Cumulative uncompressed bytes downloaded across all depots - * @property totalBytesExpected Total uncompressed bytes expected for all items - * @property status Current download phase - * @property percentComplete Calculated completion percentage (0.0 to 100.0) - * - * @author Lossy - * @since Oct 1, 2025 - */ -data class OverallProgress( - val currentItem: Int, - val totalItems: Int, - val totalBytesDownloaded: Long, - val totalBytesExpected: Long, - val status: DownloadStatus, -) { - val percentComplete: Double - get() = if (totalBytesExpected > 0) { - (totalBytesDownloaded.toDouble() / totalBytesExpected) * 100.0 - } else { - 0.0 - } -} - -/** - * Reports download progress for a specific depot within an item. - * Tracks both file-level progress (allocation/validation) and byte-level - * download progress. During the [DownloadStatus.PREPARING] phase, tracks - * file allocation; during [DownloadStatus.DOWNLOADING], tracks actual transfers. - * - * @property depotId The Steam depot identifier - * @property filesCompleted Number of files fully allocated or downloaded - * @property totalFiles Total files to process in this depot (excludes directories) - * @property bytesDownloaded Uncompressed bytes successfully downloaded - * @property totalBytes Total uncompressed bytes expected for this depot - * @property status Current depot processing phase - * @property percentComplete Calculated completion percentage (0.0 to 100.0) - * - * @author Lossy - * @since Oct 1, 2025 - */ -data class DepotProgress( - val depotId: Int, - val filesCompleted: Int, - val totalFiles: Int, - val bytesDownloaded: Long, - val totalBytes: Long, - val status: DownloadStatus, -) { - val percentComplete: Double - get() = if (totalBytes > 0) { - (bytesDownloaded.toDouble() / totalBytes) * 100.0 - } else { - 0.0 - } -} - -/** - * Reports download progress for an individual file. - * Provides chunk-level granularity for tracking file downloads. Updates are - * throttled to every 500ms to avoid excessive callback overhead. - * - * @property depotId The Steam depot containing this file - * @property fileName Relative path of the file within the depot - * @property bytesDownloaded Approximate uncompressed bytes downloaded (based on chunk completion) - * @property totalBytes Total uncompressed file size - * @property chunksCompleted Number of chunks successfully downloaded and written - * @property totalChunks Total chunks comprising this file - * @property status Current file download status - * @property percentComplete Calculated completion percentage (0.0 to 100.0) - * - * @author Lossy - * @since Oct 1, 2025 - */ -data class FileProgress( - val depotId: Int, - val fileName: String, - val bytesDownloaded: Long, - val totalBytes: Long, - val chunksCompleted: Int, - val totalChunks: Int, - val status: DownloadStatus, -) { - val percentComplete: Double - get() = if (totalBytes > 0) { - (bytesDownloaded.toDouble() / totalBytes) * 100.0 - } else { - 0.0 - } -} - -/** - * Represents the current phase of a download operation. - * - * @property PREPARING File allocation and validation phase. Files are being pre-allocated on disk and existing content is being verified. - * @property DOWNLOADING Active chunk download phase. Content is being transferred from CDN. - * @property COMPLETED Download finished successfully. All files written and verified. - */ -enum class DownloadStatus { - PREPARING, - DOWNLOADING, - COMPLETED, -} diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index bcb28c2b..b6ec70b4 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -28,10 +28,8 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.Scanner; import java.util.concurrent.CancellationException; - /** * @author Oxters * @since 2024-11-07 @@ -245,27 +243,26 @@ private void downloadApp() { // Add this class as a listener of IDownloadListener depotDownloader.addListener(this); - // An app id is required at minimum for all item types. var pubItem = new PubFileItem( - /* appId */ 0, - /* pubfile */ 0, + /* (Required) appId */ 0, + /* (Required) pubFile */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false - ); // TODO find actual pub item + ); var ugcItem = new UgcItem( - /* appId */0, - /* ugcId */ 0, + /* (Required) appId */0, + /* (Required) ugcId */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false - ); // TODO find actual ugc item + ); var appItem = new AppItem( - /* appId */ 204360, + /* (Required) appId */ 1303350, /* (Optional) installToGameNameDirectory */ true, /* (Optional) installDirectory */ "steamapps", /* (Optional) branch */ "public", @@ -283,26 +280,25 @@ private void downloadApp() { /* (Optional) downloadManifestOnly */ false ); - var scanner = new Scanner(System.in); - System.out.print("Enter a game app id: "); - var appId = scanner.nextInt(); - - // After 'depotDownloader' is constructed, items added are downloaded in a First-In, First-Out queue on the fly. + // Items added are downloaded automatically in a FIFO (First-In, First-Out) queue. // Add a singular item to process. - depotDownloader.add(new AppItem(appId, true, "steamapps")); + depotDownloader.add(appItem); // You can add a List of items to be processed. - // depotDownloader.add(List.of()); - - // Stay here while content downloads. Note this sample is synchronous so we'll loop here. - while (depotDownloader.isProcessing()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - break; - } - } + // depotDownloader.add(List.of(a, b, c)); + + // Signal the downloader that no more items will be added. + // Once all items in queue are done, 'completion' will signal that everything had finished. + depotDownloader.finishAdding(); + + // Block until we're done downloading. + // Note: If you did not call `finishAdding()` before awaiting, depotDownloader will be expecting + // more items to be added to queue. It may look like a hang. You could call `close()` to finish too. + depotDownloader.awaitCompletion(); + + // Kotlin users can use: + // depotDownloader.getCompletion().await() // Remove this class as a listener of IDownloadListener depotDownloader.removeListener(this); @@ -316,92 +312,40 @@ private void downloadApp() { @Override public void onItemAdded(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Item Added: " + item.getAppId() + "\n ---- "); + System.out.println("Item " + item.getAppId() + " added to queue."); } @Override public void onDownloadStarted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download started for item: " + item.getAppId() + "\n ---- "); + System.out.println("Item " + item.getAppId() + " download started."); } @Override public void onDownloadCompleted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download completed for item: " + item.getAppId() + "\n ---- "); + System.out.println("Item " + item.getAppId() + " download completed."); } @Override public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { - System.out.println("Depot Downloader: Download failed for item: " + item.getAppId() + "\n ---- "); - if (!error.getMessage().isEmpty()) { - System.err.println(error.getMessage()); - } + System.out.println("Item " + item.getAppId() + " failed to download"); + System.err.println(error.getMessage()); } @Override - public void onOverallProgress(@NotNull OverallProgress progress) { - System.out.printf( - "Depot Downloader: Overall Progress\n" + - "currentItem: %d\n" + - "totalItems: %d\n" + - "totalBytesDownloaded: %d\n" + - "totalBytesExpected: %d\n" + - "status: %s\n" + - "percentComplete: %.2f\n ---- %n \n", - progress.getCurrentItem(), - progress.getTotalItems(), - progress.getTotalBytesDownloaded(), - progress.getTotalBytesExpected(), - progress.getStatus(), - progress.getPercentComplete() - ); - } - - @Override - public void onDepotProgress(@NotNull DepotProgress progress) { - System.out.printf( - "Depot Downloader: Depot Progress\n" + - "depotId: %d\n" + - "filesCompleted: %d\n" + - "totalFiles: %d\n" + - "bytesDownloaded: %d\n" + - "totalBytes: %d\n" + - "status: %s\n" + - "percentComplete: %.2f\n ---- %n \n", - progress.getDepotId(), - progress.getFilesCompleted(), - progress.getTotalFiles(), - progress.getBytesDownloaded(), - progress.getTotalBytes(), - progress.getStatus(), - progress.getPercentComplete() - ); + public void onStatusUpdate(@NotNull String message) { + System.out.println("Status: " + message); } @Override - public void onFileProgress(@NotNull FileProgress progress) { - System.out.printf( - "Depot Downloader: File Progress\n" + - "depotId: %d\n" + - "fileName: %s\n" + - "bytesDownloaded: %d\n" + - "totalBytes: %d\n" + - "chunksCompleted: %d\n" + - "totalChunks: %d\n" + - "status: %s\n" + - "percentComplete: %.2f\n ---- %n \n", - progress.getDepotId(), - progress.getFileName(), - progress.getBytesDownloaded(), - progress.getTotalBytes(), - progress.getChunksCompleted(), - progress.getTotalChunks(), - progress.getStatus(), - progress.getPercentComplete() - ); + public void onFileCompleted(int depotId, @NotNull String fileName, float depotPercentComplete) { + var complete = String.format("%.2f%%", depotPercentComplete * 100f); + System.out.println("Depot " + depotId + " with file " + fileName + " completed. " + complete); } @Override - public void onStatusUpdate(@NotNull String message) { - System.out.println("Depot Downloader: Status Message: " + message + "\n ---- "); + public void onDepotCompleted(int depotId, long compressedBytes, long uncompressedBytes) { + System.out.println("Depot " + depotId + " completed."); + System.out.println("\t" + compressedBytes + " compressed bytes"); + System.out.println("\t" + uncompressedBytes + " uncompressed bytes"); } } From 5e3f386b007799d31b7da4e04a01da6b0e26fc5f Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 15:03:54 -0500 Subject: [PATCH 12/21] Bump version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index aed549cc..5a11a37e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -20,7 +20,7 @@ plugins { allprojects { group = "in.dragonbra" - version = "1.7.1-SNAPSHOT" + version = "1.8.0-SNAPSHOT" } repositories { From b9a3c85c38ad7d18ae802e6edad1d7df5934069c Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 16:21:24 -0500 Subject: [PATCH 13/21] Add reified method for removeHandler. --- .../dragonbra/javasteam/steam/steamclient/SteamClient.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt index 1872ff77..780a8c43 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt @@ -122,6 +122,15 @@ class SteamClient @JvmOverloads constructor( handlers.remove(handler) } + /** + * Kotlin Helper: + * Removes a registered handler by name. + * @param T The handler name to remove. + */ + inline fun removeHandler() { + removeHandler(T::class.java) + } + /** * Removes a registered handler. * @param handler The handler name to remove. From eda5d0e8d601ea3acd646c4959bb4e1a8c3d6477 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Tue, 14 Oct 2025 16:40:04 -0500 Subject: [PATCH 14/21] Add reified method for addHandler. --- .../javasteam/steam/steamclient/SteamClient.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt index 780a8c43..950ed9d9 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt @@ -109,6 +109,16 @@ class SteamClient @JvmOverloads constructor( addHandlerCore(handler) } + /** + * Kotlin Helper: + * Adds a new handler to the internal list of message handlers. + * @param T The handler to add. + */ + inline fun addHandler() { + val handler = T::class.java.getDeclaredConstructor().newInstance() + addHandler(handler) + } + private fun addHandlerCore(handler: ClientMsgHandler) { handler.setup(this) handlers[handler.javaClass] = handler From b9d37897a78c2475629e9dd644fdbfdcd1f5e113 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 14 Nov 2025 10:47:34 -0600 Subject: [PATCH 15/21] Fix OOM on android when allocating a file. --- .../depotdownloader/DepotDownloader.kt | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 415fec72..39295db4 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -59,6 +59,7 @@ import okio.buffer import org.apache.commons.lang3.SystemUtils import java.io.Closeable import java.io.IOException +import java.io.RandomAccessFile import java.lang.IllegalStateException import java.time.Instant import java.time.temporal.ChronoUnit @@ -1130,8 +1131,9 @@ class DepotDownloader @JvmOverloads constructor( // create new file. need all chunks try { - filesystem.openReadWrite(fileFinalPath).use { handle -> - handle.resize(file.totalSize) + // okio resize can OOM for large files on android. + RandomAccessFile(fileFinalPath.toFile(), "rw").use { + it.setLength(file.totalSize) } } catch (e: IOException) { throw DepotDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") @@ -1201,16 +1203,19 @@ class DepotDownloader @JvmOverloads constructor( if (!hashMatches || neededChunks.isNotEmpty()) { filesystem.atomicMove(fileFinalPath, fileStagingPath) + try { + RandomAccessFile(fileFinalPath.toFile(), "rw").use { raf -> + raf.setLength(file.totalSize) + } + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to resize file to expected size $fileFinalPath: ${ex.message}" + ) + } + filesystem.openReadOnly(fileStagingPath).use { oldHandle -> filesystem.openReadWrite(fileFinalPath).use { newHandle -> - try { - newHandle.resize(file.totalSize) - } catch (ex: IOException) { - throw DepotDownloaderException( - "Failed to resize file to expected size $fileFinalPath: ${ex.message}" - ) - } - + // okio resize can OOM for large files on android. for (match in copyChunks) { ensureActive() @@ -1226,18 +1231,21 @@ class DepotDownloader @JvmOverloads constructor( } } else { // No old manifest or file not in old manifest. We must validate. - filesystem.openReadWrite(fileFinalPath).use { handle -> - val fileSize = filesystem.metadata(fileFinalPath).size ?: 0L - if (fileSize.toULong() != file.totalSize.toULong()) { - try { - handle.resize(file.totalSize) - } catch (ex: IOException) { - throw DepotDownloaderException( - "Failed to allocate file $fileFinalPath: ${ex.message}" - ) + val fileSize = filesystem.metadata(fileFinalPath).size ?: 0L + if (fileSize.toULong() != file.totalSize.toULong()) { + try { + // okio resize can OOM for large files on android. + RandomAccessFile(fileFinalPath.toFile(), "rw").use { raf -> + raf.setLength(file.totalSize) } + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to allocate file $fileFinalPath: ${ex.message}" + ) } + } + filesystem.openReadWrite(fileFinalPath).use { handle -> logger?.debug("Validating $fileFinalPath") notifyListeners { it.onStatusUpdate("Validating: ${file.fileName}") } From 72cd19da6c3cec78123576030f020169c399c24e Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 14 Nov 2025 11:16:24 -0600 Subject: [PATCH 16/21] Allow getting more servers in getServersForSteamPipe --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 39295db4..e98cd1c2 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -763,7 +763,8 @@ class DepotDownloader @JvmOverloads constructor( } private suspend fun downloadSteam3(depots: List): Unit = coroutineScope { - cdnClientPool?.updateServerList() + val maxNumServers = maxDownloads.coerceIn(20, 64) // Hard clamp at 64. Not sure how high we can go. + cdnClientPool?.updateServerList(maxNumServers) val downloadCounter = GlobalDownloadCounter() val depotsToDownload = ArrayList(depots.size) From 2888c90c0eec97688786694a18993df4459b82af Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 14 Nov 2025 15:54:28 -0600 Subject: [PATCH 17/21] Add basic support for downloading Workshop collections --- .../depotdownloader/DepotDownloader.kt | 58 +++++++++++++++---- .../depotdownloader/Steam3Session.kt | 1 + 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index e98cd1c2..78b91640 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -13,6 +13,7 @@ import `in`.dragonbra.javasteam.depotdownloader.data.UgcItem import `in`.dragonbra.javasteam.enums.EAccountType import `in`.dragonbra.javasteam.enums.EAppInfoSection import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.enums.EWorkshopFileType import `in`.dragonbra.javasteam.steam.cdn.ClientLancache import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.steamapps.License @@ -118,6 +119,15 @@ class DepotDownloader @JvmOverloads constructor( const val DEFAULT_DOWNLOAD_DIR: String = "depots" val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" + + private val SupportedWorkshopFileTypes: Set = setOf( + EWorkshopFileType.Community, + EWorkshopFileType.Art, + EWorkshopFileType.Screenshot, + EWorkshopFileType.Merch, + EWorkshopFileType.IntegratedGuide, + EWorkshopFileType.ControllerBinding, + ) } private val completionFuture = CompletableFuture() @@ -189,27 +199,55 @@ class DepotDownloader @JvmOverloads constructor( // region [REGION] Downloading Operations + private suspend fun processPublishedFile( + appId: Int, + publishedFileId: Long, + fileUrls: MutableList>, + contentFileIds: MutableList, + ) { + val details = steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) + val fileType = EWorkshopFileType.from(details!!.fileType) + + if (fileType == EWorkshopFileType.Collection) { + details.childrenList.forEach { child -> + processPublishedFile(appId, child.publishedfileid, fileUrls, contentFileIds) + } + } else if (SupportedWorkshopFileTypes.contains(fileType)) { + if (details.fileUrl.isNotEmpty()) { + fileUrls.add(Pair(details.filename, details.fileUrl)) + } else if (details.hcontentFile > 0) { + contentFileIds.add(details.hcontentFile) + } else { + logger?.error("Unable to locate manifest ID for published file $publishedFileId") + } + } else { + logger?.error("Published file $publishedFileId has unsupported file type $fileType. Skipping file") + } + } + @Throws(IllegalStateException::class) private suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { - val details = requireNotNull( - steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) - ) { "Pub File Null" } + val fileUrls = mutableListOf>() + val contentFileIds = mutableListOf() + + processPublishedFile(appId, publishedFileId, fileUrls, contentFileIds) - if (!details.fileUrl.isNullOrBlank()) { - downloadWebFile(appId, details.filename, details.fileUrl) - } else if (details.hcontentFile > 0) { + fileUrls.forEach { item -> + downloadWebFile(appId, item.first, item.second) + } + + if (contentFileIds.isNotEmpty()) { + val depotManifestIds = contentFileIds.map { id -> appId to id } downloadApp( appId = appId, - depotManifestIds = listOf(appId to details.hcontentFile), + depotManifestIds = depotManifestIds, branch = DEFAULT_BRANCH, os = null, arch = null, language = null, lv = false, - isUgc = true, + isUgc = true ) - } else { - logger?.error("Unable to locate manifest ID for published file $publishedFileId") } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 1afcf241..c78ac43b 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -319,6 +319,7 @@ class Steam3Session( val pubFileRequest = SteammessagesPublishedfileSteamclient.CPublishedFile_GetDetails_Request.newBuilder().apply { this.appid = appId + this.includechildren = true this.addPublishedfileids(pubFile.toLong()) }.build() From f6c49d3a95b955356d3ad824573e0487f537c68a Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Tue, 18 Nov 2025 13:19:20 -0600 Subject: [PATCH 18/21] Add some more logs --- .../dragonbra/javasteam/depotdownloader/CDNClientPool.kt | 4 ++++ .../javasteam/depotdownloader/DepotConfigStore.kt | 9 +++++++++ .../javasteam/depotdownloader/DepotDownloader.kt | 9 +++++---- .../dragonbra/javasteam/depotdownloader/Steam3Session.kt | 8 ++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index bd24d98d..ce7ec2d5 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -55,6 +55,8 @@ class CDNClientPool( } override fun close() { + logger?.debug("Closing...") + servers.set(emptyList()) cdnClient = null @@ -107,6 +109,7 @@ class CDNClientPool( fun returnConnection(server: Server?) { if (server == null) { + logger?.error("null server returned to cdn pool.") return } @@ -117,6 +120,7 @@ class CDNClientPool( fun returnBrokenConnection(server: Server?) { if (server == null) { + logger?.error("null broken server returned to pool") return } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index 0bfebe14..327692d9 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -1,9 +1,12 @@ package `in`.dragonbra.javasteam.depotdownloader import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json import okio.FileSystem +import okio.IOException import okio.Path +import kotlin.IllegalArgumentException /** * Singleton storage for tracking installed depot manifests. @@ -27,6 +30,11 @@ data class DepotConfigStore( private val json = Json { prettyPrint = true } + @Throws( + IOException::class, + SerializationException::class, + IllegalArgumentException::class, + ) fun loadFromFile(path: Path) { instance = if (FileSystem.SYSTEM.exists(path)) { FileSystem.SYSTEM.read(path) { @@ -39,6 +47,7 @@ data class DepotConfigStore( filePath = path } + @Throws(IllegalArgumentException::class) fun save() { val currentInstance = requireNotNull(instance) { "Saved config before loading" } val currentPath = requireNotNull(filePath) { "File path not set" } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 78b91640..f507458d 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -288,7 +288,7 @@ class DepotDownloader @JvmOverloads constructor( val (success, installDir) = createDirectories(appId, 0, appId) if (!success) { - logger?.debug("Error: Unable to create install directories!") + logger?.error("Error: Unable to create install directories!") return } @@ -337,12 +337,13 @@ class DepotDownloader @JvmOverloads constructor( filesystem.atomicMove(fileStagingPath, fileFinalPath) logger?.debug("File '$fileStagingPath' moved to final location: $fileFinalPath") } catch (e: IOException) { + logger?.error(e) throw e } } // L4D2 (app) supports LV - @Throws(IllegalStateException::class) + @Throws(IllegalStateException::class, DepotDownloaderException::class) private suspend fun downloadApp( appId: Int, depotManifestIds: List>, @@ -674,7 +675,7 @@ class DepotDownloader @JvmOverloads constructor( return depotChild["depotfromapp"].asInteger() } - @Throws(IllegalStateException::class) + @Throws(IllegalStateException::class, IOException::class) private fun createDirectories(depotId: Int, depotVersion: Int, appId: Int = 0): DirectoryResult { var installDir: Path? try { @@ -1593,7 +1594,7 @@ class DepotDownloader @JvmOverloads constructor( } if (ClientLancache.useLanCacheServer) { - logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") + logger?.debug("Detected Lan-Cache server! Downloads will be directed through the LanCache.") if (maxDownloads == 8) { maxDownloads = 25 } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index c78ac43b..de86be16 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -107,6 +107,7 @@ class Steam3Session( suspend fun requestAppInfo(appId: Int, bForce: Boolean = false) { if ((appInfo.containsKey(appId) && !bForce) || isAborted) { + logger?.debug("requestAppInfo already has $appId or is aborting") return } @@ -204,6 +205,7 @@ class Steam3Session( suspend fun requestDepotKey(depotId: Int, appId: Int = 0) { if (depotKeys.containsKey(depotId) || isAborted) { + logger?.debug("requestDepotKey already has $depotId or is aborting.") return } @@ -215,6 +217,7 @@ class Steam3Session( ) if (depotKey.result != EResult.OK) { + logger?.error("requestDepotKey result was ${depotKey.result}") return } @@ -228,6 +231,7 @@ class Steam3Session( branch: String, ): ULong = withContext(Dispatchers.IO) { if (isAborted) { + logger?.debug("getDepotManifestRequestCode aborting.") return@withContext 0UL } @@ -262,12 +266,14 @@ class Steam3Session( val cdnKey = depotId to server.host!! if (cdnAuthTokens.containsKey(cdnKey)) { + logger?.debug("requestCDNAuthToken already has $cdnKey") return@withContext } val completion = CompletableDeferred() if (isAborted || cdnAuthTokens.putIfAbsent(cdnKey, completion) != null) { + logger?.debug("requestCDNAuthToken is aborting or unable to map $cdnKey") return@withContext } @@ -278,6 +284,7 @@ class Steam3Session( logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") if (cdnAuth.result != EResult.OK) { + logger?.error("requestCDNAuthToken result was ${cdnAuth.result}") return@withContext } @@ -342,6 +349,7 @@ class Steam3Session( if (callback.result == EResult.OK) { return callback } else if (callback.result == EResult.FileNotFound) { + logger?.error("getUGCDetails got FileNotFound for ${ugcHandle.value}") return null } From 49a1ea4a78cb4bf605c23a1575deb5c8c99204f9 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Tue, 18 Nov 2025 15:39:45 -0600 Subject: [PATCH 19/21] Actually populate packageTokens. --- .../javasteam/depotdownloader/DepotDownloader.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index f507458d..20428057 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -187,10 +187,15 @@ class DepotDownloader @JvmOverloads constructor( logger = LogManager.getLogger(DepotDownloader::class.java) } - logger?.debug("DepotDownloader launched with ${licenses.size} for account") - steam3 = Steam3Session(steamClient, debug) + logger?.debug("DepotDownloader launched with ${licenses.size} for account") + licenses.forEach { license -> + if (license.accessToken.toULong() > 0UL) { + steam3!!.packageTokens[license.packageID] = license.accessToken + } + } + // Launch the processing loop scope.launch { processItems() From c402bf901da8b368c90eb11862761e07e1488791 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 19 Nov 2025 00:28:53 -0600 Subject: [PATCH 20/21] use okhttp timeouts, optimize a performance chokepoint in manifest serialization, and update sample. --- .../_023_downloadapp/SampleDownloadApp.java | 18 ++++- .../dragonbra/javasteam/steam/cdn/Client.kt | 74 ++++++++----------- .../javasteam/types/DepotManifest.kt | 33 ++++----- 3 files changed, 63 insertions(+), 62 deletions(-) diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index b6ec70b4..1305d187 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -18,8 +18,10 @@ import in.dragonbra.javasteam.steam.steamclient.callbackmgr.CallbackManager; import in.dragonbra.javasteam.steam.steamclient.callbacks.ConnectedCallback; import in.dragonbra.javasteam.steam.steamclient.callbacks.DisconnectedCallback; +import in.dragonbra.javasteam.steam.steamclient.configuration.SteamConfiguration; import in.dragonbra.javasteam.util.log.DefaultLogListener; import in.dragonbra.javasteam.util.log.LogManager; +import okhttp3.OkHttpClient; import org.jetbrains.annotations.NotNull; import java.io.Closeable; @@ -29,6 +31,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CancellationException; +import java.util.concurrent.TimeUnit; /** * @author Oxters @@ -84,7 +87,18 @@ public void run() { // Most everything has been described in earlier samples. // Anything pertaining to this sample will be commented. - steamClient = new SteamClient(); + // Depot chunks are downloaded using OKHttp, it's best to set some timeouts. + var config = SteamConfiguration.create(builder -> { + builder.withHttpClient( + new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) // Time to establish connection + .readTimeout(60, TimeUnit.SECONDS) // Max inactivity between reads + .writeTimeout(30, TimeUnit.SECONDS) // Time for writes + .build() + ); + }); + + steamClient = new SteamClient(config); manager = new CallbackManager(steamClient); @@ -238,7 +252,7 @@ private void onLoggedOff(LoggedOffCallback callback) { private void downloadApp() { // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. // You will need to subscribe to LicenseListCallback to obtain your app licenses. - try (var depotDownloader = new DepotDownloader(steamClient, licenseList, false)) { + try (var depotDownloader = new DepotDownloader(steamClient, licenseList, true)) { // Add this class as a listener of IDownloadListener depotDownloader.addListener(this); diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index f77917e4..c34da41c 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -14,7 +14,6 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout import okhttp3.HttpUrl import okhttp3.OkHttpClient import okhttp3.Request @@ -39,16 +38,6 @@ class Client(steamClient: SteamClient) : Closeable { private val logger: Logger = LogManager.getLogger(Client::class.java) - /** - * Default timeout to use when making requests - */ - var requestTimeout = 10_000L - - /** - * Default timeout to use when reading the response body - */ - var responseBodyTimeout = 60_000L - private fun buildCommand( server: Server, command: String, @@ -144,9 +133,7 @@ class Client(steamClient: SteamClient) : Closeable { logger.debug("Request URL is: $request") try { - val response = withTimeout(requestTimeout) { - httpClient.newCall(request).executeAsync() - } + val response = httpClient.newCall(request).executeAsync() if (!response.isSuccessful) { throw SteamKitWebRequestException( @@ -155,30 +142,28 @@ class Client(steamClient: SteamClient) : Closeable { ) } - return@withContext withTimeout(responseBodyTimeout) { - response.use { resp -> - val responseBody = resp.body?.bytes() - ?: throw SteamKitWebRequestException("Response body is null") + return@withContext response.use { resp -> + val responseBody = resp.body?.bytes() + ?: throw SteamKitWebRequestException("Response body is null") - if (responseBody.isEmpty()) { - throw SteamKitWebRequestException("Response is empty") - } - - // Decompress the zipped manifest data - ZipInputStream(ByteArrayInputStream(responseBody)).use { zipInputStream -> - zipInputStream.nextEntry - ?: throw SteamKitWebRequestException("Expected the zip to contain at least one file") + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") + } - val manifestData = zipInputStream.readBytes() + // Decompress the zipped manifest data + ZipInputStream(ByteArrayInputStream(responseBody)).use { zipInputStream -> + zipInputStream.nextEntry + ?: throw SteamKitWebRequestException("Expected the zip to contain at least one file") - val depotManifest = DepotManifest.deserialize(ByteArrayInputStream(manifestData)) + val manifestData = zipInputStream.readBytes() - if (depotKey != null) { - depotManifest.decryptFilenames(depotKey) - } + val depotManifest = DepotManifest.deserialize(ByteArrayInputStream(manifestData)) - depotManifest + if (depotKey != null) { + depotManifest.decryptFilenames(depotKey) } + + depotManifest } } } catch (e: Exception) { @@ -240,9 +225,7 @@ class Client(steamClient: SteamClient) : Closeable { } try { - val response = withTimeout(requestTimeout) { - httpClient.newCall(request).executeAsync() - } + val response = httpClient.newCall(request).executeAsync() response.use { resp -> if (!resp.isSuccessful) { @@ -252,16 +235,23 @@ class Client(steamClient: SteamClient) : Closeable { ) } - val contentLength = resp.body.contentLength().toInt() + var contentLength = chunk.compressedLength - if (contentLength == 0) { - chunk.compressedLength - } + if (resp.body.contentLength().toInt() > 0) { + contentLength = resp.body.contentLength().toInt() - // Validate content length - if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { + // assert that lengths match only if the chunk has a length assigned. + if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { + throw SteamKitWebRequestException( + "Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})" + ) + } + } else if (contentLength > 0) { + logger.debug("Response does not have Content-Length, falling back to chunk.CompressedLength.") + } else { throw SteamKitWebRequestException( - "Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})" + "Response does not have Content-Length and chunk.CompressedLength is not set.", + response ) } diff --git a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt index ec30be2d..8069c9d6 100644 --- a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt @@ -23,7 +23,6 @@ import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec import kotlin.NoSuchElementException import kotlin.collections.ArrayList -import kotlin.collections.HashSet /** * Represents a Steam3 depot manifest. @@ -400,25 +399,23 @@ class DepotManifest { * @param output The stream to which the serialized depot manifest will be written. */ fun serialize(output: OutputStream) { - val payload = ContentManifestPayload.newBuilder() - val uniqueChunks = object : HashSet() { - // This acts like "ChunkIdComparer" - private val items = mutableListOf() - - override fun add(element: ByteArray): Boolean { - if (contains(element)) return false - items.add(element) - return true + // Basically like ChunkIdComparer from DepotDownloader + class ByteArrayKey(private val byteArray: ByteArray) { + override fun equals(other: Any?): Boolean = other is ByteArrayKey && byteArray.contentEquals(other.byteArray) + + override fun hashCode(): Int = if (byteArray.size >= 4) { + ((byteArray[0].toInt() and 0xFF)) or + ((byteArray[1].toInt() and 0xFF) shl 8) or + ((byteArray[2].toInt() and 0xFF) shl 16) or + ((byteArray[3].toInt() and 0xFF) shl 24) + } else { + byteArray.contentHashCode() } - - override fun contains(element: ByteArray): Boolean = items.any { it.contentEquals(element) } - - override fun iterator(): MutableIterator = items.iterator() - - override val size: Int - get() = items.size } + val payload = ContentManifestPayload.newBuilder() + val uniqueChunks = hashSetOf() + files.forEach { file -> val protofile = ContentManifestPayload.FileMapping.newBuilder().apply { this.size = file.totalSize @@ -457,7 +454,7 @@ class DepotManifest { }.build() protofile.addChunks(protochunk) - uniqueChunks.add(chunk.chunkID!!) + uniqueChunks.add(ByteArrayKey(chunk.chunkID!!)) } payload.addMappings(protofile.build()) From 1eb0acbd8296b8e3beefe40e1413248f6e33eff6 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Thu, 20 Nov 2025 12:20:20 -0600 Subject: [PATCH 21/21] Temp work around. Don't cancel on a depot that has no manifest request code. This may have unforseen issues, but stops some games from failing because depot 1523211 doesnt have a code. --- .../dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 20428057..1bb4ccc3 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -928,7 +928,10 @@ class DepotDownloader @JvmOverloads constructor( // If we could not get the manifest code, this is a fatal error if (manifestRequestCode == 0UL) { - cancel("manifestRequestCode is 0UL") + // TODO this should be a fatal error and bail out. But I guess we can continue. + logger?.error("Manifest request code is 0. Skipping depot ${depot.depotId}") + return@withContext null + // cancel("manifestRequestCode is 0UL") } }