diff --git a/.github/workflows/javasteam-build-push.yml b/.github/workflows/javasteam-build-push.yml index 70c27cb9..cbc6ad95 100644 --- a/.github/workflows/javasteam-build-push.yml +++ b/.github/workflows/javasteam-build-push.yml @@ -51,3 +51,7 @@ jobs: with: name: javasteam-tf path: javasteam-tf/build/libs + - uses: actions/upload-artifact@v4 + with: + name: javasteam-depotdownloader + path: javasteam-depotdownloader/build/libs diff --git a/.gitignore b/.gitignore index 2f54edb3..781bc548 100644 --- a/.gitignore +++ b/.gitignore @@ -75,7 +75,9 @@ loginkey.txt sentry.bin server_list.bin /steamapps/ +/depots/ /userfiles/ +refreshtoken.txt # Kotlin 2.0 /.kotlin/sessions/ diff --git a/build.gradle.kts b/build.gradle.kts index 753cfba6..5a11a37e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -20,7 +20,7 @@ plugins { allprojects { group = "in.dragonbra" - version = "1.7.1-SNAPSHOT" + version = "1.8.0-SNAPSHOT" } repositories { @@ -133,10 +133,10 @@ tasks.withType { dependencies { implementation(libs.bundles.ktor) + implementation(libs.bundles.okHttp) implementation(libs.commons.lang3) implementation(libs.kotlin.coroutines) implementation(libs.kotlin.stdib) - implementation(libs.okHttp) implementation(libs.protobuf.java) compileOnly(libs.xz) compileOnly(libs.zstd) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d42f0aef..5079c202 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -21,6 +21,10 @@ publishPlugin = "2.0.0" # https://mvnrepository.com/artifact/io.github.gradle-ne xz = "1.10" # https://mvnrepository.com/artifact/org.tukaani/xz zstd = "1.5.7-4" # https://search.maven.org/artifact/com.github.luben/zstd-jni +# Depot Downloader +kotlin-serialization-json = "1.9.0" # https://mvnrepository.com/artifact/org.jetbrains.kotlinx/kotlinx-serialization-json-jvm +okio = "3.16.0" # https://mvnrepository.com/artifact/com.squareup.okio/okio + # Testing Lib versions commons-io = "2.20.0" # https://mvnrepository.com/artifact/commons-io/commons-io commonsCodec = "1.19.0" # https://mvnrepository.com/artifact/commons-codec/commons-codec @@ -41,11 +45,16 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-websocket = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" } okHttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okHttp" } +okHttp-coroutines = { module = "com.squareup.okhttp3:okhttp-coroutines", version.ref = "okHttp" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } protobuf-protoc = { module = "com.google.protobuf:protoc", version.ref = "protobuf" } xz = { module = "org.tukaani:xz", version.ref = "xz" } zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } +# Depot Downloader +kotlin-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlin-serialization-json"} +okio = { module = "com.squareup.okio:okio", version.ref = "okio"} + # Tests test-commons-codec = { module = "commons-codec:commons-codec", version.ref = "commonsCodec" } test-commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" } @@ -65,6 +74,7 @@ qrCode = { module = "pro.leaco.qrcode:console-qrcode", version.ref = "qrCode" } kotlin-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } kotlin-kotlinter = { id = "org.jmailen.kotlinter", version.ref = "kotlinter" } +kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } maven-publish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "publishPlugin" } protobuf-gradle = { id = "com.google.protobuf", version.ref = "protobuf-gradle" } @@ -86,3 +96,8 @@ ktor = [ "ktor-client-cio", "ktor-client-websocket", ] + +okHttp = [ + "okHttp", + "okHttp-coroutines", +] diff --git a/javasteam-depotdownloader/.gitignore b/javasteam-depotdownloader/.gitignore new file mode 100644 index 00000000..d1638636 --- /dev/null +++ b/javasteam-depotdownloader/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/javasteam-depotdownloader/build.gradle.kts b/javasteam-depotdownloader/build.gradle.kts new file mode 100644 index 00000000..e22ddc43 --- /dev/null +++ b/javasteam-depotdownloader/build.gradle.kts @@ -0,0 +1,116 @@ +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jmailen.gradle.kotlinter.tasks.FormatTask +import org.jmailen.gradle.kotlinter.tasks.LintTask + +plugins { + alias(libs.plugins.kotlin.dokka) + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.kotlin.kotlinter) + alias(libs.plugins.kotlin.serialization) + alias(libs.plugins.protobuf.gradle) + id("maven-publish") + id("signing") +} + +repositories { + mavenCentral() +} + +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget(libs.versions.java.get())) + } +} + +/* Protobufs */ +protobuf.protoc { + artifact = libs.protobuf.protoc.get().toString() +} + +java { + sourceCompatibility = JavaVersion.toVersion(libs.versions.java.get()) + targetCompatibility = JavaVersion.toVersion(libs.versions.java.get()) + withSourcesJar() +} + +/* Java-Kotlin Docs */ +dokka { + moduleName.set("JavaSteam") + dokkaSourceSets.main { + suppressGeneratedFiles.set(false) // Allow generated files to be documented. + perPackageOption { + // Deny most of the generated files. + matchingRegex.set("in.dragonbra.javasteam.(protobufs|enums|generated).*") + suppress.set(true) + } + } +} + +// Make sure Maven Publishing gets javadoc +val javadocJar by tasks.registering(Jar::class) { + dependsOn(tasks.dokkaGenerate) + archiveClassifier.set("javadoc") + from(layout.buildDirectory.dir("dokka/html")) +} +artifacts { + archives(javadocJar) +} + +/* Kotlinter */ +tasks.withType { + this.source = this.source.minus(fileTree("build/generated")).asFileTree +} +tasks.withType { + this.source = this.source.minus(fileTree("build/generated")).asFileTree +} + +dependencies { + implementation(rootProject) + + implementation(libs.bundles.ktor) + implementation(libs.commons.lang3) + implementation(libs.kotlin.coroutines) + implementation(libs.kotlin.serialization.json) + implementation(libs.kotlin.stdib) + implementation(libs.okio) + implementation(libs.protobuf.java) +} + +/* Artifact publishing */ +publishing { + publications { + create("mavenJava") { + from(components["java"]) + pom { + name = "JavaSteam-depotdownloader" + packaging = "jar" + description = "Depot Downloader for JavaSteam." + url = "https://github.com/Longi94/JavaSteam" + inceptionYear = "2025" + scm { + connection = "scm:git:git://github.com/Longi94/JavaSteam.git" + developerConnection = "scm:git:ssh://github.com:Longi94/JavaSteam.git" + url = "https://github.com/Longi94/JavaSteam/tree/master" + } + licenses { + license { + name = "MIT License" + url = "https://www.opensource.org/licenses/mit-license.php" + } + } + developers { + developer { + id = "Longi" + name = "Long Tran" + email = "lngtrn94@gmail.com" + } + } + } + } + } +} + +signing { + sign(publishing.publications["mavenJava"]) +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt new file mode 100644 index 00000000..ce7ec2d5 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -0,0 +1,138 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.steam.cdn.Client +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference + +/** + * Manages a pool of CDN server connections for efficient content downloading. + * This class maintains a list of available CDN servers, automatically selects appropriate + * servers based on load and app compatibility, and handles connection rotation when + * servers fail or become unavailable. + * + * @param steamSession The Steam3 session for server communication + * @param appId The application ID to download - used to filter compatible CDN servers + * @param scope The coroutine scope for async operations + * @param debug If true, enables debug logging + * + * @author Oxters + * @author Lossy + * @since Nov 7, 2024 + */ +class CDNClientPool( + private val steamSession: Steam3Session, + private val appId: Int, + private val scope: CoroutineScope, + debug: Boolean = false, +) : AutoCloseable { + + private var logger: Logger? = null + + private val servers = AtomicReference>(emptyList()) + + private var nextServer: AtomicInteger = AtomicInteger(0) + + private val mutex: Mutex = Mutex() + + var cdnClient: Client? = null + private set + + var proxyServer: Server? = null + private set + + init { + cdnClient = Client(steamSession.steamClient) + + if (debug) { + logger = LogManager.getLogger(CDNClientPool::class.java) + } + } + + override fun close() { + logger?.debug("Closing...") + + servers.set(emptyList()) + + cdnClient = null + proxyServer = null + + logger = null + } + + @Throws(Exception::class) + suspend fun updateServerList(maxNumServers: Int? = null) = mutex.withLock { + val serversForSteamPipe = steamSession.steamContent!!.getServersForSteamPipe( + cellId = steamSession.steamClient.cellID ?: 0, + maxNumServers = maxNumServers, + parentScope = scope + ).await() + + proxyServer = serversForSteamPipe.firstOrNull { it.useAsProxy } + + val weightedCdnServers = serversForSteamPipe + .filter { server -> + val isEligibleForApp = server.allowedAppIds.isEmpty() || server.allowedAppIds.contains(appId) + isEligibleForApp && (server.type == "SteamCache" || server.type == "CDN") + } + .sortedBy { it.weightedLoad } + + // ContentServerPenalty removed for now. + + servers.set(weightedCdnServers) + + nextServer.set(0) + + // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } + logger?.debug("Found ${weightedCdnServers.size} Servers") + + if (weightedCdnServers.isEmpty()) { + throw Exception("Failed to retrieve any download servers.") + } + } + + fun getConnection(): Server { + val servers = servers.get() + + val index = nextServer.getAndIncrement() + val server = servers[index % servers.size] + + logger?.debug("Getting connection $server") + + return server + } + + fun returnConnection(server: Server?) { + if (server == null) { + logger?.error("null server returned to cdn pool.") + return + } + + logger?.debug("Returning connection: $server") + + // (SK) nothing to do, maybe remove from ContentServerPenalty? + } + + fun returnBrokenConnection(server: Server?) { + if (server == null) { + logger?.error("null broken server returned to pool") + return + } + + logger?.debug("Returning broken connection: $server") + + val servers = servers.get() + val currentIndex = nextServer.get() + + if (servers.isNotEmpty() && servers[currentIndex % servers.size] == server) { + nextServer.incrementAndGet() + + // TODO: (SK) Add server to ContentServerPenalty + } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt new file mode 100644 index 00000000..327692d9 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -0,0 +1,65 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json +import okio.FileSystem +import okio.IOException +import okio.Path +import kotlin.IllegalArgumentException + +/** + * Singleton storage for tracking installed depot manifests. + * Persists manifest IDs to disk to enable incremental updates and avoid + * re-downloading unchanged content. The configuration is serialized as JSON + * and must be loaded via [loadFromFile] before use. + * + * @property installedManifestIDs Map of depot IDs to their currently installed manifest IDs + * + * @author Lossy + * @since Oct 1, 2025 + */ +@Serializable +data class DepotConfigStore( + val installedManifestIDs: MutableMap = mutableMapOf(), +) { + companion object { + private var instance: DepotConfigStore? = null + + private var filePath: Path? = null + + private val json = Json { prettyPrint = true } + + @Throws( + IOException::class, + SerializationException::class, + IllegalArgumentException::class, + ) + fun loadFromFile(path: Path) { + instance = if (FileSystem.SYSTEM.exists(path)) { + FileSystem.SYSTEM.read(path) { + json.decodeFromString(readUtf8()) + } + } else { + DepotConfigStore() + } + + filePath = path + } + + @Throws(IllegalArgumentException::class) + fun save() { + val currentInstance = requireNotNull(instance) { "Saved config before loading" } + val currentPath = requireNotNull(filePath) { "File path not set" } + + currentPath.parent?.let { FileSystem.SYSTEM.createDirectories(it) } + + FileSystem.SYSTEM.write(currentPath) { + writeUtf8(json.encodeToString(currentInstance)) + } + } + + @Throws(IllegalArgumentException::class) + fun getInstance(): DepotConfigStore = requireNotNull(instance) { "Config not loaded" } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt new file mode 100644 index 00000000..1bb4ccc3 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -0,0 +1,1739 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.AppItem +import `in`.dragonbra.javasteam.depotdownloader.data.ChunkMatch +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadCounter +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo +import `in`.dragonbra.javasteam.depotdownloader.data.DepotFilesData +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem +import `in`.dragonbra.javasteam.depotdownloader.data.FileStreamData +import `in`.dragonbra.javasteam.depotdownloader.data.GlobalDownloadCounter +import `in`.dragonbra.javasteam.depotdownloader.data.PubFileItem +import `in`.dragonbra.javasteam.depotdownloader.data.UgcItem +import `in`.dragonbra.javasteam.enums.EAccountType +import `in`.dragonbra.javasteam.enums.EAppInfoSection +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.enums.EWorkshopFileType +import `in`.dragonbra.javasteam.steam.cdn.ClientLancache +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamapps.License +import `in`.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.callback.UGCDetailsCallback +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.types.FileData +import `in`.dragonbra.javasteam.types.KeyValue +import `in`.dragonbra.javasteam.types.PublishedFileID +import `in`.dragonbra.javasteam.types.UGCHandle +import `in`.dragonbra.javasteam.util.Adler32 +import `in`.dragonbra.javasteam.util.SteamKitWebRequestException +import `in`.dragonbra.javasteam.util.Strings +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsChannel +import io.ktor.http.HttpHeaders +import io.ktor.utils.io.core.readAvailable +import io.ktor.utils.io.readRemaining +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.withContext +import kotlinx.coroutines.yield +import okio.Buffer +import okio.FileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer +import org.apache.commons.lang3.SystemUtils +import java.io.Closeable +import java.io.IOException +import java.io.RandomAccessFile +import java.lang.IllegalStateException +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.* +import java.util.concurrent.atomic.AtomicInteger +import kotlin.collections.mutableListOf +import kotlin.collections.set +import kotlin.text.toLongOrNull + +/** + * Downloads games, workshop items, and other Steam content via depot manifests. + * + * This class provides a queue-based FIFO download system that processes items sequentially. + * Items are added via [add] or [addAll] and processed automatically in order. The processing + * loop starts immediately upon construction and waits for items to be queued. + * + * ## Download Process + * 1. Validates account access and obtains depot keys + * 2. Downloads and caches depot manifests + * 3. Allocates disk space for files + * 4. Downloads chunks concurrently (configured by [maxDownloads]) + * 5. Verifies checksums and moves files to final location + * + * ## Thread Safety + * Methods [add], [addAll], [addListener], and [removeListener] are thread-safe. + * Multiple concurrent calls are supported. + * + * @param steamClient Connected Steam client instance with valid login session + * @param licenses User's license list from [LicenseListCallback]. Required to determine which depots the account has access to. + * @param debug Enables detailed logging of all operations via [LogManager] + * @param useLanCache Attempts to detect and use local Steam cache servers (e.g., LANCache) for faster downloads on local networks + * @param maxDownloads Number of concurrent chunk downloads. Automatically increased to 25 when a LAN cache is detected. Default: 8 + * @param androidEmulation Forces "Windows" as the default OS filter. Used when running Android games in PC emulators that expect Windows builds. + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +@Suppress("unused") +class DepotDownloader @JvmOverloads constructor( + private val steamClient: SteamClient, + private val licenses: List, + private val debug: Boolean = false, + private val useLanCache: Boolean = false, + private var maxDownloads: Int = 8, + private val androidEmulation: Boolean = false, +) : Closeable { + + companion object { + const val INVALID_APP_ID: Int = Int.MAX_VALUE + const val INVALID_DEPOT_ID: Int = Int.MAX_VALUE + const val INVALID_MANIFEST_ID: Long = Long.MAX_VALUE + + const val CONFIG_DIR: String = ".DepotDownloader" + const val DEFAULT_BRANCH: String = "public" + const val DEFAULT_DOWNLOAD_DIR: String = "depots" + + val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" + + private val SupportedWorkshopFileTypes: Set = setOf( + EWorkshopFileType.Community, + EWorkshopFileType.Art, + EWorkshopFileType.Screenshot, + EWorkshopFileType.Merch, + EWorkshopFileType.IntegratedGuide, + EWorkshopFileType.ControllerBinding, + ) + } + + private val completionFuture = CompletableFuture() + + private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } + + private val httpClient: HttpClient by lazy { HttpClient(maxConnections = maxDownloads) } + + private val listeners = CopyOnWriteArrayList() + + private val progressUpdateInterval = 500L // ms + + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + + private var cdnClientPool: CDNClientPool? = null + + private var config: Config = Config(androidEmulation = androidEmulation) + + private var logger: Logger? = null + + private var processingChannel = Channel(Channel.UNLIMITED) + + private var steam3: Steam3Session? = null + + // region [REGION] Private data classes. + + private data class NetworkChunkItem( + val fileStreamData: FileStreamData, + val fileData: FileData, + val chunk: ChunkData, + val totalChunksForFile: Int, + ) + + private data class DirectoryResult(val success: Boolean, val installDir: Path?) + + private data class Config( + val installPath: Path? = null, + val betaPassword: String? = null, + val downloadAllPlatforms: Boolean = false, + val downloadAllArchs: Boolean = false, + val downloadAllLanguages: Boolean = false, + val androidEmulation: Boolean = false, + val downloadManifestOnly: Boolean = false, + val installToGameNameDirectory: Boolean = false, + + // Not used yet in code + val usingFileList: Boolean = false, + var filesToDownloadRegex: List = emptyList(), + var filesToDownload: HashSet = hashSetOf(), + var verifyAll: Boolean = false, + ) + + // endregion + + init { + if (debug) { + logger = LogManager.getLogger(DepotDownloader::class.java) + } + + steam3 = Steam3Session(steamClient, debug) + + logger?.debug("DepotDownloader launched with ${licenses.size} for account") + licenses.forEach { license -> + if (license.accessToken.toULong() > 0UL) { + steam3!!.packageTokens[license.packageID] = license.accessToken + } + } + + // Launch the processing loop + scope.launch { + processItems() + } + } + + // region [REGION] Downloading Operations + + private suspend fun processPublishedFile( + appId: Int, + publishedFileId: Long, + fileUrls: MutableList>, + contentFileIds: MutableList, + ) { + val details = steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) + val fileType = EWorkshopFileType.from(details!!.fileType) + + if (fileType == EWorkshopFileType.Collection) { + details.childrenList.forEach { child -> + processPublishedFile(appId, child.publishedfileid, fileUrls, contentFileIds) + } + } else if (SupportedWorkshopFileTypes.contains(fileType)) { + if (details.fileUrl.isNotEmpty()) { + fileUrls.add(Pair(details.filename, details.fileUrl)) + } else if (details.hcontentFile > 0) { + contentFileIds.add(details.hcontentFile) + } else { + logger?.error("Unable to locate manifest ID for published file $publishedFileId") + } + } else { + logger?.error("Published file $publishedFileId has unsupported file type $fileType. Skipping file") + } + } + + @Throws(IllegalStateException::class) + private suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { + val fileUrls = mutableListOf>() + val contentFileIds = mutableListOf() + + processPublishedFile(appId, publishedFileId, fileUrls, contentFileIds) + + fileUrls.forEach { item -> + downloadWebFile(appId, item.first, item.second) + } + + if (contentFileIds.isNotEmpty()) { + val depotManifestIds = contentFileIds.map { id -> appId to id } + downloadApp( + appId = appId, + depotManifestIds = depotManifestIds, + branch = DEFAULT_BRANCH, + os = null, + arch = null, + language = null, + lv = false, + isUgc = true + ) + } + } + + private suspend fun downloadUGC( + appId: Int, + ugcId: Long, + ) { + var details: UGCDetailsCallback? = null + + val steamUser = requireNotNull(steam3!!.steamUser) + val steamId = requireNotNull(steamUser.steamID) + + if (steamId.accountType != EAccountType.AnonUser) { + val ugcHandle = UGCHandle(ugcId) + details = steam3!!.getUGCDetails(ugcHandle) + } else { + logger?.error("Unable to query UGC details for $ugcId from an anonymous account") + } + + if (!details?.url.isNullOrBlank()) { + downloadWebFile(appId = appId, fileName = details.fileName, url = details.url) + } else { + downloadApp( + appId = appId, + depotManifestIds = listOf(appId to ugcId), + branch = DEFAULT_BRANCH, + os = null, + arch = null, + language = null, + lv = false, + isUgc = true, + ) + } + } + + @Throws(IllegalStateException::class, IOException::class) + private suspend fun downloadWebFile(appId: Int, fileName: String, url: String) { + val (success, installDir) = createDirectories(appId, 0, appId) + + if (!success) { + logger?.error("Error: Unable to create install directories!") + return + } + + val stagingDir = installDir!! / "staging" + val fileStagingPath = stagingDir / fileName + val fileFinalPath = installDir / fileName + + filesystem.createDirectories(fileFinalPath.parent!!) + filesystem.createDirectories(fileStagingPath.parent!!) + + httpClient.getClient().use { client -> + logger?.debug("Starting download of $fileName...") + + val response = client.get(url) + val channel = response.bodyAsChannel() + + val totalBytes = response.headers[HttpHeaders.ContentLength]?.toLongOrNull() + + logger?.debug("File size: ${totalBytes?.let { Util.formatBytes(it) } ?: "Unknown"}") + + filesystem.sink(fileStagingPath).buffer().use { sink -> + val buffer = Buffer() + val tempArray = ByteArray(DEFAULT_BUFFER_SIZE) + + while (!channel.isClosedForRead) { + val packet = channel.readRemaining(DEFAULT_BUFFER_SIZE.toLong()) + if (!packet.exhausted()) { + val bytesRead = packet.readAvailable(tempArray, 0, tempArray.size) + if (bytesRead > 0) { + buffer.write(tempArray, 0, bytesRead) + sink.writeAll(buffer) + } + } + } + } + + logger?.debug("Download completed.") + } + + if (filesystem.exists(fileFinalPath)) { + logger?.debug("Deleting $fileFinalPath") + filesystem.delete(fileFinalPath) + } + + try { + filesystem.atomicMove(fileStagingPath, fileFinalPath) + logger?.debug("File '$fileStagingPath' moved to final location: $fileFinalPath") + } catch (e: IOException) { + logger?.error(e) + throw e + } + } + + // L4D2 (app) supports LV + @Throws(IllegalStateException::class, DepotDownloaderException::class) + private suspend fun downloadApp( + appId: Int, + depotManifestIds: List>, + branch: String, + os: String?, + arch: String?, + language: String?, + lv: Boolean, + isUgc: Boolean, + ) { + var depotManifestIds = depotManifestIds.toMutableList() + + val steamUser = requireNotNull(steam3!!.steamUser) + cdnClientPool = CDNClientPool(steam3!!, appId, scope, debug) + + // Load our configuration data containing the depots currently installed + var configPath = config.installPath + if (configPath == null) { + configPath = DEFAULT_DOWNLOAD_DIR.toPath() + } + + filesystem.createDirectories(configPath) + DepotConfigStore.loadFromFile(configPath / CONFIG_DIR / "depot.config") + + steam3!!.requestAppInfo(appId) + + if (!accountHasAccess(appId, appId)) { + if (steamUser.steamID!!.accountType != EAccountType.AnonUser && steam3!!.requestFreeAppLicense(appId)) { + logger?.debug("Obtained FreeOnDemand license for app $appId") + + // Fetch app info again in case we didn't get it fully without a license. + steam3!!.requestAppInfo(appId, true) + } else { + val contentName = getAppName(appId) + throw DepotDownloaderException("App $appId ($contentName) is not available from this account.") + } + } + + val hasSpecificDepots = depotManifestIds.isNotEmpty() + val depotIdsFound = mutableListOf() + val depotIdsExpected = depotManifestIds.map { x -> x.first }.toMutableList() + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) + + if (isUgc) { + val workshopDepot = depots!!["workshopdepot"].asInteger() + if (workshopDepot != 0 && !depotIdsExpected.contains(workshopDepot)) { + depotIdsExpected.add(workshopDepot) + depotManifestIds = depotManifestIds.map { pair -> workshopDepot to pair.second }.toMutableList() + } + + depotIdsFound.addAll(depotIdsExpected) + } else { + logger?.debug("Using app branch: $branch") + + depots?.children?.forEach { depotSection -> + if (depotSection.children.isEmpty()) { + return@forEach + } + + val id: Int = depotSection.name?.toIntOrNull() ?: return@forEach + + if (hasSpecificDepots && !depotIdsExpected.contains(id)) { + return@forEach + } + + if (!hasSpecificDepots) { + val depotConfig = depotSection["config"] + if (depotConfig != KeyValue.INVALID) { + if (!config.downloadAllPlatforms && + depotConfig["oslist"] != KeyValue.INVALID && + !depotConfig["oslist"].value.isNullOrBlank() + ) { + val osList = depotConfig["oslist"].value!!.split(",") + if (osList.indexOf(os ?: Util.getSteamOS(config.androidEmulation)) == -1) { + return@forEach + } + } + + if (!config.downloadAllArchs && + depotConfig["osarch"] != KeyValue.INVALID && + !depotConfig["osarch"].value.isNullOrBlank() + ) { + val depotArch = depotConfig["osarch"].value + if (depotArch != (arch ?: Util.getSteamArch())) { + return@forEach + } + } + + if (!config.downloadAllLanguages && + depotConfig["language"] != KeyValue.INVALID && + !depotConfig["language"].value.isNullOrBlank() + ) { + val depotLang = depotConfig["language"].value + if (depotLang != (language ?: "english")) { + return@forEach + } + } + + if (!lv && + depotConfig["lowviolence"] != KeyValue.INVALID && + depotConfig["lowviolence"].asBoolean() + ) { + return@forEach + } + } + } + + depotIdsFound.add(id) + + if (!hasSpecificDepots) { + depotManifestIds.add(id to INVALID_MANIFEST_ID) + } + } + + if (depotManifestIds.isEmpty() && !hasSpecificDepots) { + throw DepotDownloaderException("Couldn't find any depots to download for app $appId") + } + + if (depotIdsFound.size < depotIdsExpected.size) { + val remainingDepotIds = depotIdsExpected.subtract(depotIdsFound.toSet()) + throw DepotDownloaderException("Depot ${remainingDepotIds.joinToString(", ")} not listed for app $appId") + } + } + + val infos = mutableListOf() + + depotManifestIds.forEach { (depotId, manifestId) -> + val info = getDepotInfo(depotId, appId, manifestId, branch) + if (info != null) { + infos.add(info) + } + } + + downloadSteam3(infos) + } + + @Throws(IllegalStateException::class) + private suspend fun getDepotInfo( + depotId: Int, + appId: Int, + manifestId: Long, + branch: String, + ): DepotDownloadInfo? { + var manifestId = manifestId + var branch = branch + + if (appId != INVALID_APP_ID) { + steam3!!.requestAppInfo(appId) + } + + if (!accountHasAccess(appId, depotId)) { + logger?.error("Depot $depotId is not available from this account.") + return null + } + + if (manifestId == INVALID_MANIFEST_ID) { + manifestId = getSteam3DepotManifest(depotId, appId, branch) + + if (manifestId == INVALID_MANIFEST_ID && !branch.equals(DEFAULT_BRANCH, true)) { + logger?.error("Warning: Depot $depotId does not have branch named \"$branch\". Trying $DEFAULT_BRANCH branch.") + branch = DEFAULT_BRANCH + manifestId = getSteam3DepotManifest(depotId, appId, branch) + } + + if (manifestId == INVALID_MANIFEST_ID) { + logger?.error("Depot $depotId missing public subsection or manifest section.") + return null + } + } + + steam3!!.requestDepotKey(depotId, appId) + + val depotKey = steam3!!.depotKeys[depotId] + if (depotKey == null) { + logger?.error("No valid depot key for $depotId, unable to download.") + return null + } + + val uVersion = getSteam3AppBuildNumber(appId, branch) + + val (success, installDir) = createDirectories(depotId, uVersion, appId) + if (!success) { + logger?.error("Error: Unable to create install directories!") + return null + } + + // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id, unless the app is freetodownload + var containingAppId = appId + val proxyAppId = getSteam3DepotProxyAppId(depotId, appId) + if (proxyAppId != INVALID_APP_ID) { + val common = getSteam3AppSection(appId, EAppInfoSection.Common) + if (common == null || !common["FreeToDownload"].asBoolean()) { + containingAppId = proxyAppId + } + } + + return DepotDownloadInfo( + depotId = depotId, + appId = containingAppId, + manifestId = manifestId, + branch = branch, + installDir = installDir!!, + depotKey = depotKey + ) + } + + private suspend fun getSteam3DepotManifest( + depotId: Int, + appId: Int, + branch: String, + ): Long { + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) + var depotChild = depots?.get(depotId.toString()) ?: KeyValue.INVALID + + if (depotChild == KeyValue.INVALID) { + return INVALID_MANIFEST_ID + } + + // Shared depots can either provide manifests, or leave you relying on their parent app. + // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). + // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. + if (depotChild["manifests"] == KeyValue.INVALID && depotChild["depotfromapp"] != KeyValue.INVALID) { + val otherAppId = depotChild["depotfromapp"].asInteger() + if (otherAppId == appId) { + // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. + logger?.error("App $appId, Depot $depotId has depotfromapp of $otherAppId!") + return INVALID_MANIFEST_ID + } + + steam3!!.requestAppInfo(otherAppId) + + return getSteam3DepotManifest(depotId, otherAppId, branch) + } + + var manifests = depotChild["manifests"] + + if (manifests.children.isEmpty()) { + return INVALID_MANIFEST_ID + } + + var node = manifests[branch]["gid"] + + // Non passworded branch, found the manifest + if (node.value != null) { + return node.value!!.toLong() + } + + // If we requested public branch, and it had no manifest, nothing to do + if (branch.equals(DEFAULT_BRANCH, true)) { + return INVALID_MANIFEST_ID + } + + // Either the branch just doesn't exist, or it has a password + if (config.betaPassword.isNullOrBlank()) { + logger?.error("Branch $branch for depot $depotId was not found, either it does not exist or it has a password.") + return INVALID_MANIFEST_ID + } + + if (!steam3!!.appBetaPasswords.containsKey(branch)) { + // Submit the password to Steam now to get encryption keys + steam3!!.checkAppBetaPassword(appId, config.betaPassword!!) + + if (!steam3!!.appBetaPasswords.containsKey(branch)) { + logger?.error("Error: Password was invalid for branch $branch (or the branch does not exist)") + return INVALID_MANIFEST_ID + } + } + + // Got the password, request private depot section + // TODO: (SK) We're probably repeating this request for every depot? + val privateDepotSection = steam3!!.getPrivateBetaDepotSection(appId, branch) + + // Now repeat the same code to get the manifest gid from depot section + depotChild = privateDepotSection[depotId.toString()] + + if (depotChild == KeyValue.INVALID) { + return INVALID_MANIFEST_ID + } + + manifests = depotChild["manifests"] + + if (manifests.children.isEmpty()) { + return INVALID_MANIFEST_ID + } + + node = manifests[branch]["gid"] + + if (node.value == null) { + return INVALID_MANIFEST_ID + } + + return node.value!!.toLong() + } + + private fun getSteam3AppBuildNumber(appId: Int, branch: String): Int { + if (appId == INVALID_APP_ID) { + return 0 + } + + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) ?: KeyValue.INVALID + val branches = depots["branches"] + val node = branches[branch] + + if (node == KeyValue.INVALID) { + return 0 + } + + val buildId = node["buildid"] + + if (buildId == KeyValue.INVALID) { + return 0 + } + + return buildId.value!!.toInt() + } + + private fun getSteam3DepotProxyAppId(depotId: Int, appId: Int): Int { + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) ?: KeyValue.INVALID + val depotChild = depots[depotId.toString()] + + if (depotChild == KeyValue.INVALID) { + return INVALID_APP_ID + } + + if (depotChild["depotfromapp"] == KeyValue.INVALID) { + return INVALID_APP_ID + } + + return depotChild["depotfromapp"].asInteger() + } + + @Throws(IllegalStateException::class, IOException::class) + private fun createDirectories(depotId: Int, depotVersion: Int, appId: Int = 0): DirectoryResult { + var installDir: Path? + try { + if (config.installPath?.toString().isNullOrBlank()) { + // Android Check + if (SystemUtils.IS_OS_ANDROID) { + // This should propagate up to the caller. + throw IllegalStateException("Android must have an installation directory set.") + } + + filesystem.createDirectories(DEFAULT_DOWNLOAD_DIR.toPath()) + + if (config.installToGameNameDirectory) { + val gameName = getAppName(appId) + + if (gameName.isBlank()) { + throw IOException("Game name is blank, cannot create directory") + } + + installDir = DEFAULT_DOWNLOAD_DIR.toPath() / gameName + + filesystem.createDirectories(installDir) + } else { + val depotPath = DEFAULT_DOWNLOAD_DIR.toPath() / depotId.toString() + filesystem.createDirectories(depotPath) + + installDir = depotPath / depotVersion.toString() + filesystem.createDirectories(installDir) + } + + filesystem.createDirectories(installDir / CONFIG_DIR) + filesystem.createDirectories(installDir / STAGING_DIR) + } else { + filesystem.createDirectories(config.installPath!!) + + if (config.installToGameNameDirectory) { + val gameName = getAppName(appId) + + if (gameName.isBlank()) { + throw IOException("Game name is blank, cannot create directory") + } + + installDir = config.installPath!! / gameName + + filesystem.createDirectories(installDir) + } else { + installDir = config.installPath!! + } + + filesystem.createDirectories(installDir / CONFIG_DIR) + filesystem.createDirectories(installDir / STAGING_DIR) + } + } catch (e: IOException) { + logger?.error(e) + return DirectoryResult(false, null) + } + + return DirectoryResult(true, installDir) + } + + private fun getAppName(appId: Int): String { + val info = getSteam3AppSection(appId, EAppInfoSection.Common) ?: KeyValue.INVALID + return info["name"].asString() ?: "" + } + + private fun getSteam3AppSection(appId: Int, section: EAppInfoSection): KeyValue? { + if (steam3 == null) { + return null + } + + if (steam3!!.appInfo.isEmpty()) { + return null + } + + val app = steam3!!.appInfo[appId]?.value ?: return null + + val appInfo = app.keyValues + val sectionKey = when (section) { + EAppInfoSection.Common -> "common" + EAppInfoSection.Extended -> "extended" + EAppInfoSection.Config -> "config" + EAppInfoSection.Depots -> "depots" + else -> throw DepotDownloaderException("${section.name} not implemented") + } + + val sectionKV = appInfo.children.firstOrNull { c -> c.name == sectionKey } + return sectionKV + } + + private suspend fun accountHasAccess(appId: Int, depotId: Int): Boolean { + val steamUser = requireNotNull(steam3!!.steamUser) + val steamID = requireNotNull(steamUser.steamID) + + if (licenses.isEmpty() && steamID.accountType != EAccountType.AnonUser) { + return false + } + + val licenseQuery = arrayListOf() + if (steamID.accountType == EAccountType.AnonUser) { + licenseQuery.add(17906) + } else { + licenseQuery.addAll(licenses.map { it.packageID }.distinct()) + } + + steam3!!.requestPackageInfo(licenseQuery) + + licenseQuery.forEach { license -> + steam3!!.packageInfo[license]?.value?.let { pkg -> + val appIds = pkg.keyValues["appids"].children.map { it.asInteger() } + val depotIds = pkg.keyValues["depotids"].children.map { it.asInteger() } + if (depotId in appIds) { + return true + } + if (depotId in depotIds) { + return true + } + } + } + + // Check if this app is free to download without a license + val info = getSteam3AppSection(appId, EAppInfoSection.Common) + + return info != null && info["FreeToDownload"].asBoolean() + } + + private suspend fun downloadSteam3(depots: List): Unit = coroutineScope { + val maxNumServers = maxDownloads.coerceIn(20, 64) // Hard clamp at 64. Not sure how high we can go. + cdnClientPool?.updateServerList(maxNumServers) + + val downloadCounter = GlobalDownloadCounter() + val depotsToDownload = ArrayList(depots.size) + val allFileNamesAllDepots = hashSetOf() + + // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup + depots.forEach { depot -> + val depotFileData = processDepotManifestAndFiles(depot, downloadCounter) + + if (depotFileData != null) { + depotsToDownload.add(depotFileData) + allFileNamesAllDepots.addAll(depotFileData.allFileNames) + } + + ensureActive() + } + + // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path + // This is in last-depot-wins order, from Steam or the list of depots supplied by the user + if (config.installPath != null && depotsToDownload.isNotEmpty()) { + val claimedFileNames = mutableSetOf() + for (i in depotsToDownload.indices.reversed()) { + // For each depot, remove all files from the list that have been claimed by a later depot + depotsToDownload[i].filteredFiles.removeAll { file -> file.fileName in claimedFileNames } + claimedFileNames.addAll(depotsToDownload[i].allFileNames) + } + } + + depotsToDownload.forEach { depotFileData -> + downloadSteam3DepotFiles(downloadCounter, depotFileData, allFileNamesAllDepots) + } + + logger?.debug( + "Total downloaded: ${downloadCounter.totalBytesCompressed} bytes " + + "(${downloadCounter.totalBytesUncompressed} bytes uncompressed) from ${depots.size} depots" + ) + } + + private suspend fun processDepotManifestAndFiles( + depot: DepotDownloadInfo, + downloadCounter: GlobalDownloadCounter, + ): DepotFilesData? = withContext(Dispatchers.IO) { + val depotCounter = DepotDownloadCounter() + + logger?.debug("Processing depot ${depot.depotId}") + + var oldManifest: DepotManifest? = null + + @Suppress("VariableInitializerIsRedundant") + var newManifest: DepotManifest? = null + + val configDir = depot.installDir / CONFIG_DIR + + @Suppress("VariableInitializerIsRedundant") + var lastManifestId = INVALID_MANIFEST_ID + lastManifestId = DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] ?: INVALID_MANIFEST_ID + + // In case we have an early exit, this will force equiv of verifyall next run. + DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = INVALID_MANIFEST_ID + DepotConfigStore.save() + + if (lastManifestId != INVALID_MANIFEST_ID) { + // We only have to show this warning if the old manifest ID was different + val badHashWarning = lastManifestId != depot.manifestId + oldManifest = Util.loadManifestFromFile(configDir, depot.depotId, lastManifestId, badHashWarning) + } + + if (lastManifestId == depot.manifestId && oldManifest != null) { + newManifest = oldManifest + logger?.debug("Already have manifest ${depot.manifestId} for depot ${depot.depotId}.") + } else { + newManifest = Util.loadManifestFromFile(configDir, depot.depotId, depot.manifestId, true) + + if (newManifest != null) { + logger?.debug("Already have manifest ${depot.manifestId} for depot ${depot.depotId}.") + } else { + logger?.debug("Downloading depot ${depot.depotId} manifest") + notifyListeners { it.onStatusUpdate("Downloading manifest for depot ${depot.depotId}") } + + var manifestRequestCode: ULong = 0U + var manifestRequestCodeExpiration = Instant.MIN + + do { + ensureActive() + + var connection: Server? = null + + try { + connection = cdnClientPool!!.getConnection() + + var cdnToken: String? = null + + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") + } + } + + val now = Instant.now() + + // In order to download this manifest, we need the current manifest request code + // The manifest request code is only valid for a specific period in time + if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { + manifestRequestCode = steam3!!.getDepotManifestRequestCode( + depotId = depot.depotId, + appId = depot.appId, + manifestId = depot.manifestId, + branch = depot.branch, + ) + + // This code will hopefully be valid for one period following the issuing period + manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) + + // If we could not get the manifest code, this is a fatal error + if (manifestRequestCode == 0UL) { + // TODO this should be a fatal error and bail out. But I guess we can continue. + logger?.error("Manifest request code is 0. Skipping depot ${depot.depotId}") + return@withContext null + // cancel("manifestRequestCode is 0UL") + } + } + + logger?.debug("Downloading manifest ${depot.manifestId} from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") + + newManifest = cdnClientPool!!.cdnClient!!.downloadManifest( + depotId = depot.depotId, + manifestId = depot.manifestId, + manifestRequestCode = manifestRequestCode, + server = connection, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) + + cdnClientPool!!.returnConnection(connection) + } catch (e: CancellationException) { + // logger?.error("Connection timeout downloading depot manifest ${depot.depotId} ${depot.manifestId}. Retrying.") + logger?.error(e) + break + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet + if (e.statusCode == 403 && !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host)) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + + cdnClientPool!!.returnConnection(connection) + + continue + } + + cdnClientPool!!.returnBrokenConnection(connection) + + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${depot.depotId} for depot manifest ${depot.manifestId} ${e.statusCode}. Aborting.") + break + } + + // NotFound + if (e.statusCode == 404) { + logger?.error("Encountered 404 for depot manifest ${depot.depotId} ${depot.manifestId}. Aborting.") + break + } + + logger?.error("Encountered error downloading depot manifest ${depot.depotId} ${depot.manifestId}: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered error downloading manifest for depot ${depot.depotId} ${depot.manifestId}: ${e.message}") + } + } while (newManifest == null) + + if (newManifest == null) { + logger?.error("\nUnable to download manifest ${depot.manifestId} for depot ${depot.depotId}") + cancel() + } + + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() + + Util.saveManifestToFile(configDir, newManifest!!) + } + } + + logger?.debug("Manifest ${depot.manifestId} (${newManifest.creationTime})") + + if (config.downloadManifestOnly) { + Util.dumpManifestToTextFile(depot, newManifest) + return@withContext null + } + + val stagingDir = depot.installDir / STAGING_DIR + + val filesAfterExclusions = coroutineScope { + newManifest.files.filter { file -> + async { testIsFileIncluded(file.fileName) }.await() + } + } + val allFileNames = HashSet(filesAfterExclusions.size) + + // Pre-process + filesAfterExclusions.forEachIndexed { index, file -> + if (index % 50 == 0) { + ensureActive() // Check cancellation periodically + } + + allFileNames.add(file.fileName) + + val fileFinalPath = depot.installDir / file.fileName + val fileStagingPath = stagingDir / file.fileName + + if (file.flags.contains(EDepotFileFlag.Directory)) { + filesystem.createDirectories(fileFinalPath) + filesystem.createDirectories(fileStagingPath) + } else { + // Some manifests don't explicitly include all necessary directories + filesystem.createDirectories(fileFinalPath.parent!!) + filesystem.createDirectories(fileStagingPath.parent!!) + + downloadCounter.completeDownloadSize += file.totalSize + depotCounter.completeDownloadSize += file.totalSize + } + } + + return@withContext DepotFilesData( + depotDownloadInfo = depot, + depotCounter = depotCounter, + stagingDir = stagingDir, + manifest = newManifest, + previousManifest = oldManifest, + filteredFiles = filesAfterExclusions.toMutableList(), + allFileNames = allFileNames, + ) + } + + @OptIn(DelicateCoroutinesApi::class) + private suspend fun downloadSteam3DepotFiles( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + allFileNamesAllDepots: HashSet, + ) = withContext(Dispatchers.IO) { + val depot = depotFilesData.depotDownloadInfo + val depotCounter = depotFilesData.depotCounter + + logger?.debug("Downloading depot ${depot.depotId}") + + val files = depotFilesData.filteredFiles.filter { !it.flags.contains(EDepotFileFlag.Directory) } + val networkChunkQueue = Channel(Channel.UNLIMITED) + + try { + coroutineScope { + // First parallel loop - process files and enqueue chunks + files.chunked(50).forEach { batch -> + yield() + + batch.map { file -> + async { + downloadSteam3DepotFile( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = file, + networkChunkQueue = networkChunkQueue + ) + } + }.awaitAll() + } + + // Close the channel to signal no more items will be added + networkChunkQueue.close() + + // Second parallel loop - process chunks from queue + List(maxDownloads) { + async { + for (item in networkChunkQueue) { + downloadSteam3DepotFileChunk( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = item.fileData, + fileStreamData = item.fileStreamData, + chunk = item.chunk + ) + } + } + }.awaitAll() + } + } finally { + if (!networkChunkQueue.isClosedForSend) { + networkChunkQueue.close() + } + } + + // Check for deleted files if updating the depot. + if (depotFilesData.previousManifest != null) { + val previousFilteredFiles = depotFilesData.previousManifest.files + .filter { testIsFileIncluded(it.fileName) } + .map { it.fileName } + .toHashSet() + + // Check if we are writing to a single output directory. If not, each depot folder is managed independently + if (config.installPath == null) { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names + previousFilteredFiles.removeAll(depotFilesData.allFileNames) + } else { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded + previousFilteredFiles.removeAll(allFileNamesAllDepots) + } + + previousFilteredFiles.forEach { existingFileName -> + val fileFinalPath = depot.installDir / existingFileName + + if (!filesystem.exists(fileFinalPath)) { + return@forEach + } + + filesystem.delete(fileFinalPath) + logger?.debug("Deleted $fileFinalPath") + } + } + + DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = depot.manifestId + DepotConfigStore.save() + + // Notify depot completion + notifyListeners { listener -> + listener.onDepotCompleted( + depotId = depot.depotId, + compressedBytes = depotCounter.depotBytesCompressed, + uncompressedBytes = depotCounter.depotBytesUncompressed + ) + } + + logger?.debug("Depot ${depot.depotId} - Downloaded ${depotCounter.depotBytesCompressed} bytes (${depotCounter.depotBytesUncompressed} bytes uncompressed)") + } + + private suspend fun downloadSteam3DepotFile( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + file: FileData, + networkChunkQueue: Channel, + ) = withContext(Dispatchers.IO) { + ensureActive() + + val depot = depotFilesData.depotDownloadInfo + val stagingDir = depotFilesData.stagingDir + val depotDownloadCounter = depotFilesData.depotCounter + val oldProtoManifest = depotFilesData.previousManifest + + var oldManifestFile: FileData? = null + if (oldProtoManifest != null) { + oldManifestFile = oldProtoManifest.files.singleOrNull { it.fileName == file.fileName } + } + + val fileFinalPath = depot.installDir / file.fileName + val fileStagingPath = stagingDir / file.fileName + + // This may still exist if the previous run exited before cleanup + if (filesystem.exists(fileStagingPath)) { + filesystem.delete(fileStagingPath) + } + + var neededChunks: MutableList? = null + val fileDidExist = filesystem.exists(fileFinalPath) + if (!fileDidExist) { + logger?.debug("Pre-allocating: $fileFinalPath") + notifyListeners { it.onStatusUpdate("Allocating file: ${file.fileName}") } + + // create new file. need all chunks + try { + // okio resize can OOM for large files on android. + RandomAccessFile(fileFinalPath.toFile(), "rw").use { + it.setLength(file.totalSize) + } + } catch (e: IOException) { + throw DepotDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") + } + + neededChunks = ArrayList(file.chunks) + } else { + // open existing + if (oldManifestFile != null) { + neededChunks = arrayListOf() + + val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) + if (config.verifyAll || !hashMatches) { + // we have a version of this file, but it doesn't fully match what we want + if (config.verifyAll) { + logger?.debug("Validating: $fileFinalPath") + } + + val matchingChunks = arrayListOf() + + file.chunks.forEach { chunk -> + yield() + + val oldChunk = oldManifestFile.chunks.firstOrNull { c -> + c.chunkID.contentEquals(chunk.chunkID) + } + if (oldChunk != null) { + val chunkMatch = ChunkMatch(oldChunk, chunk) + matchingChunks.add(chunkMatch) + } else { + neededChunks.add(chunk) + } + } + + val orderedChunks = matchingChunks.sortedBy { x -> x.oldChunk.offset } + + val copyChunks = arrayListOf() + + filesystem.openReadOnly(fileFinalPath).use { handle -> + orderedChunks.forEach { match -> + yield() + + // Read the chunk data into a byte array + val length = match.oldChunk.uncompressedLength + val buffer = ByteArray(length) + handle.read(match.oldChunk.offset, buffer, 0, length) + + // Calculate Adler32 checksum + val adler = Adler32.calculate(buffer) + + // Convert checksum to byte array for comparison + val checksumBytes = Buffer().apply { + writeIntLe(match.oldChunk.checksum) + }.readByteArray() + val calculatedChecksumBytes = Buffer().apply { + writeIntLe(adler) + }.readByteArray() + + if (!calculatedChecksumBytes.contentEquals(checksumBytes)) { + neededChunks.add(match.newChunk) + } else { + copyChunks.add(match) + } + } + } + + if (!hashMatches || neededChunks.isNotEmpty()) { + filesystem.atomicMove(fileFinalPath, fileStagingPath) + + try { + RandomAccessFile(fileFinalPath.toFile(), "rw").use { raf -> + raf.setLength(file.totalSize) + } + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to resize file to expected size $fileFinalPath: ${ex.message}" + ) + } + + filesystem.openReadOnly(fileStagingPath).use { oldHandle -> + filesystem.openReadWrite(fileFinalPath).use { newHandle -> + // okio resize can OOM for large files on android. + for (match in copyChunks) { + ensureActive() + + val tmp = ByteArray(match.oldChunk.uncompressedLength) + oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) + newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) + } + } + } + + filesystem.delete(fileStagingPath) + } + } + } else { + // No old manifest or file not in old manifest. We must validate. + val fileSize = filesystem.metadata(fileFinalPath).size ?: 0L + if (fileSize.toULong() != file.totalSize.toULong()) { + try { + // okio resize can OOM for large files on android. + RandomAccessFile(fileFinalPath.toFile(), "rw").use { raf -> + raf.setLength(file.totalSize) + } + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to allocate file $fileFinalPath: ${ex.message}" + ) + } + } + + filesystem.openReadWrite(fileFinalPath).use { handle -> + logger?.debug("Validating $fileFinalPath") + notifyListeners { it.onStatusUpdate("Validating: ${file.fileName}") } + + neededChunks = Util.validateSteam3FileChecksums( + handle = handle, + chunkData = file.chunks.sortedBy { it.offset } + ).toMutableList() + } + } + + if (neededChunks!!.isEmpty()) { + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += file.totalSize + + val percentage = + (depotDownloadCounter.sizeDownloaded / depotDownloadCounter.completeDownloadSize.toFloat()) * 100.0f + logger?.debug("%.2f%% %s".format(percentage, fileFinalPath)) + } + + synchronized(downloadCounter) { + downloadCounter.completeDownloadSize -= file.totalSize + } + + return@withContext + } + + val sizeOnDisk = file.totalSize - neededChunks!!.sumOf { it.uncompressedLength } + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += sizeOnDisk + } + + synchronized(downloadCounter) { + downloadCounter.completeDownloadSize -= sizeOnDisk + } + } + + val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) + if (fileIsExecutable && + (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) + ) { + fileFinalPath.toFile().setExecutable(true) + } else if (!fileIsExecutable && + oldManifestFile != null && + oldManifestFile.flags.contains(EDepotFileFlag.Executable) + ) { + fileFinalPath.toFile().setExecutable(false) + } + + val fileStreamData = FileStreamData( + fileHandle = null, + fileLock = Mutex(), + chunksToDownload = AtomicInteger(neededChunks!!.size) + ) + + neededChunks!!.forEach { chunk -> + networkChunkQueue.send( + NetworkChunkItem( + fileStreamData = fileStreamData, + fileData = file, + chunk = chunk, + totalChunksForFile = neededChunks!!.size + ) + ) + } + } + + private suspend fun downloadSteam3DepotFileChunk( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + file: FileData, + fileStreamData: FileStreamData, + chunk: ChunkData, + ): Unit = withContext(Dispatchers.IO) { + ensureActive() + + val depot = depotFilesData.depotDownloadInfo + val depotDownloadCounter = depotFilesData.depotCounter + + val chunkID = Strings.toHex(chunk.chunkID) + + var written = 0 + val chunkBuffer = ByteArray(chunk.uncompressedLength) + + do { + ensureActive() + + var connection: Server? = null + + try { + connection = cdnClientPool?.getConnection() + ?: throw IllegalStateException("ContentDownloader already closed") + + var cdnToken: String? = null + + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") + } + } + + logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") + + written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( + depotId = depot.depotId, + chunk = chunk, + server = connection, + destination = chunkBuffer, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) + + cdnClientPool!!.returnConnection(connection) + + break + } catch (e: CancellationException) { + logger?.error(e) + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, + // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above + if (e.statusCode == 403 && + ( + !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host) || + steam3!!.cdnAuthTokens[depot.depotId to connection.host]?.isCompleted == false + ) + ) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + + cdnClientPool!!.returnConnection(connection) + + continue + } + + cdnClientPool!!.returnBrokenConnection(connection) + + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") + break + } + + logger?.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered unexpected error downloading chunk $chunkID", e) + } + } while (written == 0) + + if (written == 0) { + logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") + cancel() + } + + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() + + try { + fileStreamData.fileLock.lock() + + if (fileStreamData.fileHandle == null) { + val fileFinalPath = depot.installDir / file.fileName + fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) + } + + fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) + } finally { + fileStreamData.fileLock.unlock() + } + + val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() + if (remainingChunks == 0) { + fileStreamData.fileHandle?.close() + + // File completed - notify with percentage + val sizeDownloaded = synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + depotDownloadCounter.sizeDownloaded + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + + val fileFinalPath = depot.installDir / file.fileName + val depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) + + notifyListeners { listener -> + listener.onFileCompleted( + depotId = depot.depotId, + fileName = fileFinalPath.toString(), + depotPercentComplete = depotPercentage + ) + } + + logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) + } else { + // Just update counters without notifying + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + } + } + + private fun testIsFileIncluded(filename: String): Boolean { + if (!config.usingFileList) { + return true + } + + val normalizedFilename = filename.replace('\\', '/') + + if (normalizedFilename in config.filesToDownload) { + return true + } + + for (regex in config.filesToDownloadRegex) { + if (regex.matches(normalizedFilename)) { + return true + } + } + + return false + } + + // endregion + + // region [REGION] Listener Operations + + fun addListener(listener: IDownloadListener) { + listeners.add(listener) + } + + fun removeListener(listener: IDownloadListener) { + listeners.remove(listener) + } + + private fun notifyListeners(action: (IDownloadListener) -> Unit) { + listeners.forEach { listener -> action(listener) } + } + + // endregion + + // region [REGION] Queue Operations + + /** + * Add a singular item of either [AppItem], [PubFileItem], or [UgcItem] + */ + fun add(item: DownloadItem) { + runBlocking { + try { + processingChannel.send(item) + notifyListeners { it.onItemAdded(item) } + } catch (e: Exception) { + logger?.error(e) + throw e + } + } + } + + /** + * Add a list items of either [AppItem], [PubFileItem], or [UgcItem] + */ + fun addAll(items: List) { + runBlocking { + try { + items.forEach { item -> + processingChannel.send(item) + notifyListeners { it.onItemAdded(item) } + } + } catch (e: Exception) { + logger?.error(e) + throw e + } + } + } + + /** + * Signals that no more items will be added to the download queue. + * After calling this, the downloader will complete once all queued items finish. + * + * This is called automatically by [close], but you can call it explicitly + * if you want to wait for completion without closing the downloader. + */ + fun finishAdding() { + processingChannel.close() + } + + // endregion + + private suspend fun processItems() = coroutineScope { + if (useLanCache) { + ClientLancache.detectLancacheServer() + } + + if (ClientLancache.useLanCacheServer) { + logger?.debug("Detected Lan-Cache server! Downloads will be directed through the LanCache.") + if (maxDownloads == 8) { + maxDownloads = 25 + } + } + + for (item in processingChannel) { + try { + ensureActive() + + // Set configuration values + config = config.copy( + downloadManifestOnly = item.downloadManifestOnly, + installPath = item.installDirectory?.toPath(), + installToGameNameDirectory = item.installToGameNameDirectory, + ) + + when (item) { + is PubFileItem -> { + logger?.debug("Downloading PUB File for ${item.appId}") + notifyListeners { it.onDownloadStarted(item) } + downloadPubFile(item.appId, item.pubFile) + } + + is UgcItem -> { + logger?.debug("Downloading UGC File for ${item.appId}") + notifyListeners { it.onDownloadStarted(item) } + downloadUGC(item.appId, item.ugcId) + } + + is AppItem -> { + val branch = item.branch ?: DEFAULT_BRANCH + config = config.copy(betaPassword = item.branchPassword) + + if (!config.betaPassword.isNullOrBlank() && branch.isBlank()) { + logger?.error("Error: Cannot specify 'branchpassword' when 'branch' is not specified.") + continue + } + + config = config.copy(downloadAllPlatforms = item.downloadAllPlatforms) + val os = item.os + + if (config.downloadAllPlatforms && !os.isNullOrBlank()) { + logger?.error("Error: Cannot specify `os` when `all-platforms` is specified.") + continue + } + + config = config.copy(downloadAllArchs = item.downloadAllArchs) + val arch = item.osArch + + if (config.downloadAllArchs && !arch.isNullOrBlank()) { + logger?.error("Error: Cannot specify `osarch` when `all-archs` is specified.") + continue + } + + config = config.copy(downloadAllLanguages = item.downloadAllLanguages) + val language = item.language + + if (config.downloadAllLanguages && !language.isNullOrBlank()) { + logger?.error("Error: Cannot specify `language` when `all-languages` is specified.") + continue + } + + val depotManifestIds = mutableListOf>() + val depotIdList = item.depot + val manifestIdList = item.manifest + + if (manifestIdList.isNotEmpty()) { + if (depotIdList.size != manifestIdList.size) { + logger?.error("Error: `manifest` requires one id for every `depot` specified") + continue + } + depotManifestIds.addAll(depotIdList.zip(manifestIdList)) + } else { + depotManifestIds.addAll(depotIdList.map { it to INVALID_MANIFEST_ID }) + } + + logger?.debug("Downloading App for ${item.appId}") + notifyListeners { it.onDownloadStarted(item) } + downloadApp( + appId = item.appId, + depotManifestIds = depotManifestIds, + branch = branch, + os = os, + arch = arch, + language = language, + lv = item.lowViolence, + isUgc = false, + ) + } + } + + notifyListeners { it.onDownloadCompleted(item) } + } catch (e: Exception) { + logger?.error("Error downloading item ${item.appId}: ${e.message}", e) + notifyListeners { it.onDownloadFailed(item, e) } + } + } + + completionFuture.complete(null) + } + + /** + * Returns a CompletableFuture that completes when all queued downloads finish. + * @return CompletableFuture that completes when all downloads finish + */ + fun getCompletion(): CompletableFuture = completionFuture + + /** + * Blocks the current thread until all queued downloads complete. + * Convenience method that calls `getCompletion().join()`. + * @throws CompletionException if any download fails + */ + fun awaitCompletion() { + completionFuture.join() + } + + override fun close() { + processingChannel.close() + + scope.cancel("DepotDownloader Closing") + + httpClient.close() + + listeners.clear() + + steam3?.close() + steam3 = null + + cdnClientPool?.close() + cdnClientPool = null + + logger = null + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt new file mode 100644 index 00000000..4c3b4bbc --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt @@ -0,0 +1,11 @@ +package `in`.dragonbra.javasteam.depotdownloader + +/** + * Exception thrown when content download operations fail. + * Used to indicate errors during depot downloads, manifest retrieval, + * or other content downloader operations. + * + * @author Lossy + * @since Oct 1, 2025 + */ +class DepotDownloaderException(value: String) : Exception(value) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt new file mode 100644 index 00000000..67f35d48 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -0,0 +1,66 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.util.Versions +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.engine.cio.endpoint +import io.ktor.client.plugins.UserAgent +import kotlinx.coroutines.isActive +import java.io.Closeable + +/** + * HTTP client wrapper for content downloader operations. + * + * Provides a configured Ktor HTTP client optimized for Steam CDN downloads. + * Each instance maintains its own connection pool based on the specified + * maximum concurrent connections. + * + * @param maxConnections Maximum number of concurrent connections + * + * @author Lossy + * @since Oct 1, 2025 + */ +class HttpClient( + private val maxConnections: Int, +) : Closeable { + + private var httpClient: HttpClient? = null + + /** + * Returns the HTTP client instance, creating it lazily on first access. + * + * The client is configured with: + * - Custom User-Agent identifying JavaSteam DepotDownloader + * - Connection pooling based on [maxConnections] + * - 5 second keep-alive and connect timeout + * - 30 second request timeout + */ + fun getClient(): HttpClient { + if (httpClient?.isActive != true) { + httpClient = HttpClient(CIO) { + install(UserAgent) { + agent = "JavaSteam-DepotDownloader/${Versions.getVersion()}" + } + engine { + maxConnectionsCount = maxConnections + endpoint { + maxConnectionsPerRoute = (maxConnections / 2).coerceAtLeast(1) + pipelineMaxSize = maxConnections * 2 + keepAliveTime = 5000 + connectTimeout = 5000 + requestTimeout = 30000 + } + } + } + } + + return httpClient!! + } + + override fun close() { + if (httpClient?.isActive == true) { + httpClient?.close() + httpClient = null + } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt new file mode 100644 index 00000000..331f4a72 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -0,0 +1,60 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem + +/** + * Listener interface for receiving download progress and status events. + * + * All methods have default empty implementations, allowing listeners to + * implement only the callbacks they need. + * + * @author Lossy + * @since Oct 1, 2025 + */ +interface IDownloadListener { + /** + * Called when an item is added to the download queue. + */ + fun onItemAdded(item: DownloadItem) {} + + /** + * Called when a download begins processing. + */ + fun onDownloadStarted(item: DownloadItem) {} + + /** + * Called when a download completes successfully. + */ + fun onDownloadCompleted(item: DownloadItem) {} + + /** + * Called when a download fails with an error. + */ + fun onDownloadFailed(item: DownloadItem, error: Throwable) {} + + /** + * Called during file preparation with informational messages. + * Examples: "Pre-allocating depots\441\file.txt", "Validating file.cab" + */ + fun onStatusUpdate(message: String) {} + + /** + * Called when a file completes downloading. + * Use this for printing progress like "20.42% depots\441\maps\ctf_haarp.bsp" + * + * @param depotId The depot being downloaded + * @param fileName Relative file path + * @param depotPercentComplete Overall depot completion percentage (0f to 1f) + */ + fun onFileCompleted(depotId: Int, fileName: String, depotPercentComplete: Float) {} + + /** + * Called when a depot finishes downloading. + * Use this for printing summary like "Depot 228990 - Downloaded X bytes (Y bytes uncompressed)" + * + * @param depotId The depot that completed + * @param compressedBytes Bytes transferred (compressed) + * @param uncompressedBytes Actual data size (uncompressed) + */ + fun onDepotCompleted(depotId: Int, compressedBytes: Long, uncompressedBytes: Long) {} +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt new file mode 100644 index 00000000..de86be16 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -0,0 +1,358 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.enums.EResult +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesPublishedfileSteamclient +import `in`.dragonbra.javasteam.rpc.service.PublishedFile +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest +import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.SteamCloud +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.callback.UGCDetailsCallback +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.CDNAuthToken +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent +import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages +import `in`.dragonbra.javasteam.steam.handlers.steamuser.SteamUser +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.types.KeyValue +import `in`.dragonbra.javasteam.types.PublishedFileID +import `in`.dragonbra.javasteam.types.UGCHandle +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * Manages Steam protocol session state and API interactions for content downloading. + * + * All Steam API handlers are initialized during construction and must remain valid + * for the lifetime of this session. + * + * @param steamClient The connected Steam client instance + * @param debug If true, enables debug logging for all Steam API operations + * + * @author Lossy + * @since Oct 1, 2025 + */ +class Steam3Session( + internal val steamClient: SteamClient, + debug: Boolean, +) : Closeable { + + private var logger: Logger? = null + + private var isAborted: Boolean = false // Stubbed, no way to set true/false yet. + + internal val appTokens = ConcurrentHashMap() + internal val packageTokens = ConcurrentHashMap() + internal val depotKeys = ConcurrentHashMap() + internal val cdnAuthTokens = ConcurrentHashMap, CompletableDeferred>() + internal val appInfo = ConcurrentHashMap>() + internal val packageInfo = ConcurrentHashMap>() + internal val appBetaPasswords = ConcurrentHashMap() + + private var unifiedMessages: SteamUnifiedMessages? = null + internal var steamUser: SteamUser? = null + internal var steamContent: SteamContent? = null + internal var steamApps: SteamApps? = null + internal var steamCloud: SteamCloud? = null + internal var steamPublishedFile: PublishedFile? = null + + // ConcurrentHashMap can't have nullable Keys or Values + internal data class Optional(val value: T?) + + init { + if (debug) { + logger = LogManager.getLogger(Steam3Session::class.java) + } + + unifiedMessages = requireNotNull(steamClient.getHandler()) + steamApps = requireNotNull(steamClient.getHandler()) + steamCloud = requireNotNull(steamClient.getHandler()) + steamContent = requireNotNull(steamClient.getHandler()) + steamPublishedFile = requireNotNull(unifiedMessages?.createService()) + steamUser = requireNotNull(steamClient.getHandler()) + } + + override fun close() { + logger?.debug("Closing...") + + unifiedMessages = null + steamUser = null + steamContent = null + steamApps = null + steamCloud = null + steamPublishedFile = null + + cdnAuthTokens.values.forEach { it.cancel() } + cdnAuthTokens.clear() + + depotKeys.values.forEach { it.fill(0) } + depotKeys.clear() + appBetaPasswords.values.forEach { it.fill(0) } + appBetaPasswords.clear() + + appTokens.clear() + packageTokens.clear() + appInfo.clear() + packageInfo.clear() + + logger = null + } + + suspend fun requestAppInfo(appId: Int, bForce: Boolean = false) { + if ((appInfo.containsKey(appId) && !bForce) || isAborted) { + logger?.debug("requestAppInfo already has $appId or is aborting") + return + } + + val appTokens = steamApps!!.picsGetAccessTokens(appId).await() + + if (appTokens.appTokensDenied.contains(appId)) { + logger?.error("Insufficient privileges to get access token for app $appId") + } + + appTokens.appTokens.forEach { tokenDict -> + this.appTokens[tokenDict.key] = tokenDict.value + } + + val request = PICSRequest(appId) + + this.appTokens[appId]?.let { token -> + request.accessToken = token + } + + val appInfoMultiple = steamApps!!.picsGetProductInfo(request).await() + + logger?.debug( + "requestAppInfo($appId, $bForce) with \n" + + "${appTokens.appTokens.size} appTokens, \n" + + "${appTokens.appTokensDenied.size} appTokensDenied, \n" + + "${appTokens.packageTokens.size} packageTokens, and \n" + + "${appTokens.packageTokensDenied} packageTokensDenied. \n" + + "picsGetProductInfo result size: ${appInfoMultiple.results.size}" + ) + + appInfoMultiple.results.forEach { appInfo -> + appInfo.apps.forEach { appValue -> + val app = appValue.value + this.appInfo[app.id] = Optional(app) + } + appInfo.unknownApps.forEach { app -> + this.appInfo[app] = Optional(null) + } + } + } + + // TODO race condition (??) + private val packageInfoMutex = Mutex() + suspend fun requestPackageInfo(packageIds: List) { + packageInfoMutex.withLock { + logger?.debug("requestPackageInfo() invoked with ${packageIds.size} packageIds") + + // I have a silly race condition??? + val packages = packageIds.filter { !packageInfo.containsKey(it) } + + if (packages.isEmpty() || isAborted) { + return + } + + val packageRequests = arrayListOf() + + packages.forEach { pkg -> + val request = PICSRequest(id = pkg) + + packageTokens[pkg]?.let { token -> + request.accessToken = token + } + + packageRequests.add(request) + } + + val packageInfoMultiple = steamApps!!.picsGetProductInfo(emptyList(), packageRequests).await() + + logger?.debug("requestPackageInfo() picsGetProductInfo result size: ${packageInfoMultiple.results.size} ") + + packageInfoMultiple.results.forEach { pkgInfo -> + pkgInfo.packages.forEach { pkgValue -> + val pkg = pkgValue.value + packageInfo[pkg.id] = Optional(pkg) + } + pkgInfo.unknownPackages.forEach { pkgValue -> + packageInfo[pkgValue] = Optional(null) + } + } + } + } + + suspend fun requestFreeAppLicense(appId: Int): Boolean { + try { + val resultInfo = steamApps!!.requestFreeLicense(appId).await() + + logger?.debug("requestFreeAppLicense($appId) has result ${resultInfo.result}") + + return resultInfo.grantedApps.contains(appId) + } catch (e: Exception) { + logger?.error("Failed to request FreeOnDemand license for app $appId: ${e.message}") + return false + } + } + + suspend fun requestDepotKey(depotId: Int, appId: Int = 0) { + if (depotKeys.containsKey(depotId) || isAborted) { + logger?.debug("requestDepotKey already has $depotId or is aborting.") + return + } + + val depotKey = steamApps!!.getDepotDecryptionKey(depotId, appId).await() + + logger?.debug( + "requestDepotKey($depotId, $appId) " + + "Got depot key for ${depotKey.depotID} result: ${depotKey.result}" + ) + + if (depotKey.result != EResult.OK) { + logger?.error("requestDepotKey result was ${depotKey.result}") + return + } + + depotKeys[depotKey.depotID] = depotKey.depotKey + } + + suspend fun getDepotManifestRequestCode( + depotId: Int, + appId: Int, + manifestId: Long, + branch: String, + ): ULong = withContext(Dispatchers.IO) { + if (isAborted) { + logger?.debug("getDepotManifestRequestCode aborting.") + return@withContext 0UL + } + + val requestCode = steamContent!!.getManifestRequestCode( + depotId = depotId, + appId = appId, + manifestId = manifestId, + branch = branch, + branchPasswordHash = null, + parentScope = this // TODO am I passing this right? + ).await().toULong() + + if (requestCode == 0UL) { + logger?.error("No manifest request code was returned for depot $depotId from app $appId, manifest $manifestId") + + if (steamClient.isDisconnected) { + logger?.debug("Suggestion: Try logging in with a username as old manifests may not be available for anonymous accounts.") + } + } else { + logger?.debug("Got manifest request code for depot $depotId from app $appId, manifest $manifestId, result: $requestCode") + } + + logger?.debug( + "getDepotManifestRequestCode($depotId, $appId, $manifestId, $branch) " + + "got request code $requestCode" + ) + + return@withContext requestCode + } + + suspend fun requestCDNAuthToken(appId: Int, depotId: Int, server: Server) = withContext(Dispatchers.IO) { + val cdnKey = depotId to server.host!! + + if (cdnAuthTokens.containsKey(cdnKey)) { + logger?.debug("requestCDNAuthToken already has $cdnKey") + return@withContext + } + + val completion = CompletableDeferred() + + if (isAborted || cdnAuthTokens.putIfAbsent(cdnKey, completion) != null) { + logger?.debug("requestCDNAuthToken is aborting or unable to map $cdnKey") + return@withContext + } + + logger?.debug("Requesting CDN auth token for ${server.host}") + + val cdnAuth = steamContent!!.getCDNAuthToken(appId, depotId, server.host!!, this).await() + + logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") + + if (cdnAuth.result != EResult.OK) { + logger?.error("requestCDNAuthToken result was ${cdnAuth.result}") + return@withContext + } + + completion.complete(cdnAuth) + } + + suspend fun checkAppBetaPassword(appId: Int, password: String) { + val appPassword = steamApps!!.checkAppBetaPassword(appId, password).await() + + logger?.debug( + "checkAppBetaPassword($appId, )," + + "retrieved ${appPassword.betaPasswords.size} beta keys with result: ${appPassword.result}" + ) + + appPassword.betaPasswords.forEach { entry -> + this.appBetaPasswords[entry.key] = entry.value + } + } + + suspend fun getPrivateBetaDepotSection(appId: Int, branch: String): KeyValue { + // Should be filled by CheckAppBetaPassword + val branchPassword = appBetaPasswords[branch] ?: return KeyValue() + + // Should be filled by RequestAppInfo + val accessToken = appTokens[appId] ?: 0L + + val privateBeta = steamApps!!.picsGetPrivateBeta(appId, accessToken, branch, branchPassword).await() + + logger?.debug("getPrivateBetaDepotSection($appId, $branch) result: ${privateBeta.result}") + + return privateBeta.depotSection + } + + @Throws(DepotDownloaderException::class) + suspend fun getPublishedFileDetails( + appId: Int, + pubFile: PublishedFileID, + ): SteammessagesPublishedfileSteamclient.PublishedFileDetails? { + val pubFileRequest = + SteammessagesPublishedfileSteamclient.CPublishedFile_GetDetails_Request.newBuilder().apply { + this.appid = appId + this.includechildren = true + this.addPublishedfileids(pubFile.toLong()) + }.build() + + val details = steamPublishedFile!!.getDetails(pubFileRequest).await() + + logger?.debug("requestUGCDetails($appId, $pubFile) result: ${details.result}") + + if (details.result == EResult.OK) { + return details.body.publishedfiledetailsBuilderList.firstOrNull()?.build() + } + + throw DepotDownloaderException("EResult ${details.result.code()} (${details.result}) while retrieving file details for pubfile $pubFile.") + } + + suspend fun getUGCDetails(ugcHandle: UGCHandle): UGCDetailsCallback? { + val callback = steamCloud!!.requestUGCDetails(ugcHandle).await() + + logger?.debug("requestUGCDetails($ugcHandle) result: ${callback.result}") + + if (callback.result == EResult.OK) { + return callback + } else if (callback.result == EResult.FileNotFound) { + logger?.error("getUGCDetails got FileNotFound for ${ugcHandle.value}") + return null + } + + throw DepotDownloaderException("EResult ${callback.result.code()} (${callback.result}) while retrieving UGC details for ${ugcHandle.value}.") + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt new file mode 100644 index 00000000..a46951a0 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -0,0 +1,224 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.util.Adler32 +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import okio.FileHandle +import okio.FileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer +import org.apache.commons.lang3.SystemUtils +import java.io.IOException +import java.security.MessageDigest +import java.security.NoSuchAlgorithmException + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +object Util { + + private val logger: Logger = LogManager.getLogger() + + @JvmOverloads + @JvmStatic + fun getSteamOS(androidEmulation: Boolean = false): String { + if (SystemUtils.IS_OS_WINDOWS) { + return "windows" + } + if (SystemUtils.IS_OS_MAC_OSX) { + return "macos" + } + if (SystemUtils.IS_OS_LINUX) { + return "linux" + } + if (SystemUtils.IS_OS_FREE_BSD) { + // Return linux as freebsd steam client doesn't exist yet + return "linux" + } + + // Hack for PC emulation on android. (Pluvia, GameNative, GameHub) + if (androidEmulation && SystemUtils.IS_OS_ANDROID) { + return "windows" + } + + return "unknown" + } + + @JvmStatic + fun getSteamArch(): String { + val arch = try { + System.getProperty("os.arch")?.lowercase() ?: "" + } catch (e: Exception) { + logger.error(e) + "" + } + return when { + arch.contains("64") -> "64" + arch.contains("86") -> "32" + arch.contains("amd64") -> "64" + arch.contains("x86_64") -> "64" + arch.contains("aarch64") -> "64" + arch.contains("arm64") -> "64" + else -> "32" + } + } + + @JvmStatic + @Throws(IOException::class) + fun saveManifestToFile(directory: Path, manifest: DepotManifest): Boolean = try { + val filename = directory / "${manifest.depotID}_${manifest.manifestGID}.manifest" + manifest.saveToFile(filename.toString()) + + val shaFile = "$filename.sha".toPath() + FileSystem.SYSTEM.write(shaFile) { + write(fileSHAHash(filename)) + } + + true + } catch (e: IOException) { + logger.error(e) + false + } + + @JvmStatic + @Throws(NoSuchAlgorithmException::class, IllegalArgumentException::class, IOException::class) + fun loadManifestFromFile( + directory: Path, + depotId: Int, + manifestId: Long, + badHashWarning: Boolean, + ): DepotManifest? { + // Try loading Steam format manifest first. + val filename = directory / "${depotId}_$manifestId.manifest" + + if (FileSystem.SYSTEM.exists(filename)) { + val expectedChecksum = try { + FileSystem.SYSTEM.read("$filename.sha".toPath()) { + readByteArray() + } + } catch (e: IOException) { + logger.error(e) + null + } + + val currentChecksum = fileSHAHash(filename) + + if (expectedChecksum != null && expectedChecksum.contentEquals(currentChecksum)) { + return DepotManifest.loadFromFile(filename.toString()) + } else if (badHashWarning) { + logger.debug("Manifest $manifestId on disk did not match the expected checksum.") + } + } + + return null + } + + @JvmStatic + @Throws(NoSuchAlgorithmException::class, IllegalArgumentException::class, IOException::class) + fun fileSHAHash(filename: Path): ByteArray { + val digest = MessageDigest.getInstance("SHA-1") + + FileSystem.SYSTEM.source(filename).use { source -> + source.buffer().use { bufferedSource -> + val buffer = ByteArray(8192) + var bytesRead: Int + + while (bufferedSource.read(buffer).also { bytesRead = it } != -1) { + digest.update(buffer, 0, bytesRead) + } + } + } + + return digest.digest() + } + + /** + * Validate a file against Steam3 Chunk data + * + * @param handle FileHandle to read from + * @param chunkData Array of ChunkData to validate against + * @return List of ChunkData that are needed + * @throws IOException If there's an error reading the file + */ + @JvmStatic + @Throws(IOException::class) + fun validateSteam3FileChecksums(handle: FileHandle, chunkData: List): List { + val neededChunks = mutableListOf() + + for (data in chunkData) { + val chunk = ByteArray(data.uncompressedLength) + val read = handle.read(data.offset, chunk, 0, data.uncompressedLength) + + val tempChunk = if (read > 0 && read < data.uncompressedLength) { + chunk.copyOf(read) + } else { + chunk + } + + val adler = Adler32.calculate(tempChunk) + if (adler != data.checksum) { + neededChunks.add(data) + } + } + + return neededChunks + } + + @JvmStatic + @Throws(IOException::class) + fun dumpManifestToTextFile(depot: DepotDownloadInfo, manifest: DepotManifest) { + val txtManifest = depot.installDir / "manifest_${depot.depotId}_${depot.manifestId}.txt" + + FileSystem.SYSTEM.write(txtManifest) { + writeUtf8("Content Manifest for Depot ${depot.depotId}\n") + writeUtf8("\n") + writeUtf8("Manifest ID / date : ${depot.manifestId} / ${manifest.creationTime}\n") + + val uniqueChunks = manifest.files + .flatMap { it.chunks } + .mapNotNull { it.chunkID } + .toSet() + + writeUtf8("Total number of files : ${manifest.files.size}\n") + writeUtf8("Total number of chunks : ${uniqueChunks.size}\n") + writeUtf8("Total bytes on disk : ${manifest.totalUncompressedSize}\n") + writeUtf8("Total bytes compressed : ${manifest.totalCompressedSize}\n") + writeUtf8("\n") + writeUtf8("\n") + + writeUtf8(" Size Chunks File SHA Flags Name\n") + manifest.files.forEach { file -> + val sha1Hash = file.fileHash.toHexString().lowercase() + writeUtf8( + "%14d %6d %s %5x %s\n".format( + file.totalSize, + file.chunks.size, + sha1Hash, + EDepotFileFlag.code(file.flags), + file.fileName + ) + ) + } + } + } + + @JvmStatic + fun formatBytes(bytes: Long): String { + val units = arrayOf("B", "KB", "MB", "GB") + var size = bytes.toDouble() + var unitIndex = 0 + + while (size >= 1024 && unitIndex < units.size - 1) { + size /= 1024 + unitIndex++ + } + + return "%.2f %s".format(size, units[unitIndex]) + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt new file mode 100644 index 00000000..dafee0bd --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt @@ -0,0 +1,16 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.types.ChunkData + +/** + * Pairs matching chunks between old and new depot manifests for differential updates. + * Used during file validation to identify chunks that can be reused from existing + * files, avoiding unnecessary re-downloads when only portions of a file have changed. + * + * @property oldChunk Chunk from the previously installed manifest + * @property newChunk Corresponding chunk from the new manifest being downloaded + * + * @author Oxters + * @since Oct 29, 2024 + */ +data class ChunkMatch(val oldChunk: ChunkData, val newChunk: ChunkData) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt new file mode 100644 index 00000000..5b853ce4 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt @@ -0,0 +1,53 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import okio.Path + +/** + * Contains all information required to download a specific depot manifest and its content. + * This class aggregates the depot identification, authentication, and installation details + * needed to perform a complete depot download operation. It is created during the depot + * resolution phase and passed through the download pipeline. + * + * @property depotId The Steam depot identifier + * @property appId The owning application ID (may differ from the app being downloaded if the depot uses `depotfromapp` proxying) + * @property manifestId The specific manifest version to download + * @property branch The branch name this manifest belongs to (e.g., "public", "beta") + * @property installDir The target directory for downloaded files + * @property depotKey The AES decryption key for depot chunks. Cleared on disposal for security. + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotDownloadInfo( + val depotId: Int, + val appId: Int, + val manifestId: Long, + val branch: String, + val installDir: Path, + val depotKey: ByteArray, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is DepotDownloadInfo) return false + + if (depotId != other.depotId) return false + if (appId != other.appId) return false + if (manifestId != other.manifestId) return false + if (branch != other.branch) return false + if (installDir != other.installDir) return false + if (!depotKey.contentEquals(other.depotKey)) return false + + return true + } + + override fun hashCode(): Int { + var result = depotId + result = 31 * result + appId + result = 31 * result + manifestId.hashCode() + result = 31 * result + branch.hashCode() + result = 31 * result + installDir.hashCode() + result = 31 * result + depotKey.contentHashCode() + return result + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt new file mode 100644 index 00000000..15d4dda9 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt @@ -0,0 +1,32 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.types.FileData +import okio.Path + +/** + * Aggregates all data needed to process and download files for a single depot. + * Created during manifest processing and passed to the download phase. Contains both + * the current manifest and optional previous manifest to enable differential updates. + * + * @property depotDownloadInfo Core depot identification and authentication details + * @property depotCounter Progress tracking counters for this depot's download + * @property stagingDir Temporary directory for in-progress file writes + * @property manifest The current depot manifest being downloaded + * @property previousManifest The previously installed manifest, if any. Used to identify reusable chunks and deleted files. + * @property filteredFiles Files to download after applying platform, language, and user filters. Modified during processing to remove duplicates across depots. + * @property allFileNames Complete set of filenames in this depot, including directories. Used for cross-depot deduplication and cleanup of deleted files. + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotFilesData( + val depotDownloadInfo: DepotDownloadInfo, + val depotCounter: DepotDownloadCounter, + val stagingDir: Path, + val manifest: DepotManifest, + val previousManifest: DepotManifest?, + val filteredFiles: MutableList, + val allFileNames: HashSet, +) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt new file mode 100644 index 00000000..380463c2 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt @@ -0,0 +1,43 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +// https://kotlinlang.org/docs/coding-conventions.html#source-file-organization + +/** + * Tracks cumulative download statistics across all depots in a download session. + * Used for overall progress reporting and final download summary. All fields are + * accessed under synchronization to ensure thread-safe updates from concurrent downloads. + * + * @property completeDownloadSize Total bytes expected to download across all depots. Adjusted during validation when existing chunks are reused. + * @property totalBytesCompressed Total compressed bytes transferred from CDN servers + * @property totalBytesUncompressed Total uncompressed bytes written to disk + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class GlobalDownloadCounter( + var completeDownloadSize: Long = 0, + var totalBytesCompressed: Long = 0, + var totalBytesUncompressed: Long = 0, +) + +/** + * Tracks download statistics for a single depot. + * Used for depot-level progress reporting. All fields are accessed under synchronization + * to ensure thread-safe updates from concurrent chunk downloads. + * + * @property completeDownloadSize Total bytes expected to download for this depot. Calculated during file allocation phase. + * @property sizeDownloaded Bytes successfully downloaded and written so far + * @property depotBytesCompressed Compressed bytes transferred from CDN for this depot + * @property depotBytesUncompressed Uncompressed bytes written to disk for this depot + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotDownloadCounter( + var completeDownloadSize: Long = 0, + var sizeDownloaded: Long = 0, + var depotBytesCompressed: Long = 0, + var depotBytesUncompressed: Long = 0, +) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt new file mode 100644 index 00000000..813e0b52 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -0,0 +1,94 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +// https://kotlinlang.org/docs/coding-conventions.html#source-file-organization + +// TODO should these be a builder pattern for Java users? + +/** + * Base class for downloadable Steam content items. + * @property appId The Steam application ID + * @property installDirectory Optional custom installation directory path + * @property installToGameNameDirectory If true, installs to a directory named after the game + * @property downloadManifestOnly If true, only downloads the manifest file without actual content + * + * @author Lossy + * @since Oct 1, 2025 + */ +abstract class DownloadItem( + val appId: Int, + val installDirectory: String?, + val installToGameNameDirectory: Boolean, + val verify: Boolean, // TODO + val downloadManifestOnly: Boolean, +) + +/** + * Represents a Steam Workshop (UGC - User Generated Content) item for download. + * + * @property ugcId The unique UGC item identifier + * + * @author Lossy + * @since Oct 1, 2025 + */ +class UgcItem @JvmOverloads constructor( + appId: Int, + val ugcId: Long, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + verify: Boolean = false, + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) + +/** + * Represents a Steam published file for download. + * + * @property pubfile The published file identifier + * + * @author Lossy + * @since Oct 1, 2025 + */ +class PubFileItem @JvmOverloads constructor( + appId: Int, + val pubFile: Long, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + verify: Boolean = false, + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) + +/** + * Represents a Steam application/game for download from a depot. + * + * @property branch The branch name to download from (e.g., "public", "beta") + * @property branchPassword Password for password-protected branches + * @property downloadAllPlatforms If true, downloads depots for all platforms + * @property os Operating system filter (e.g., "windows", "macos", "linux") + * @property downloadAllArchs If true, downloads depots for all architectures + * @property osArch Architecture filter (e.g., "32", "64") + * @property downloadAllLanguages If true, downloads depots for all languages + * @property language Language filter (e.g., "english", "french") + * @property lowViolence If true, downloads low-violence versions where available + * @property depot List of specific depot IDs to download + * @property manifest List of specific manifest IDs corresponding to depot IDs + * + * @author Lossy + * @since Oct 1, 2025 + */ +class AppItem @JvmOverloads constructor( + appId: Int, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + val branch: String? = null, + val branchPassword: String? = null, + val downloadAllPlatforms: Boolean = false, + val os: String? = null, + val downloadAllArchs: Boolean = false, + val osArch: String? = null, + val downloadAllLanguages: Boolean = false, + val language: String? = null, + val lowViolence: Boolean = false, + val depot: List = emptyList(), + val manifest: List = emptyList(), + verify: Boolean = false, + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt new file mode 100644 index 00000000..1bc68fac --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt @@ -0,0 +1,24 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import kotlinx.coroutines.sync.Mutex +import okio.FileHandle +import java.util.concurrent.atomic.AtomicInteger + +/** + * Internal state for managing concurrent chunk writes to a single file. + * Coordinates writes from multiple download workers to ensure thread-safe file access + * and tracks when all chunks have been written. + * + * @property fileHandle Shared file handle for all chunk writes. Lazily opened on first write. + * @property fileLock Mutex protecting concurrent access to [fileHandle] + * @property chunksToDownload Atomic counter of remaining chunks. File is closed when this reaches zero. + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class FileStreamData( + var fileHandle: FileHandle?, + val fileLock: Mutex = Mutex(), + var chunksToDownload: AtomicInteger = AtomicInteger(0), +) diff --git a/javasteam-samples/build.gradle.kts b/javasteam-samples/build.gradle.kts index 9deffead..1e1aa774 100644 --- a/javasteam-samples/build.gradle.kts +++ b/javasteam-samples/build.gradle.kts @@ -14,12 +14,13 @@ java { dependencies { implementation(rootProject) implementation(project(":javasteam-cs")) + implementation(project(":javasteam-depotdownloader")) implementation(libs.bouncyCastle) implementation(libs.gson) implementation(libs.kotlin.coroutines) implementation(libs.okHttp) - implementation(libs.protobuf.java) // To access protobufs directly as shown in Sample #2 + implementation(libs.protobuf.java) // Protobuf access implementation(libs.qrCode) implementation(libs.zstd) // Content Downloading. implementation(libs.xz) // Content Downloading. diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index e22d7e08..1305d187 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -1,9 +1,15 @@ package in.dragonbra.javasteamsamples._023_downloadapp; +import in.dragonbra.javasteam.depotdownloader.DepotDownloader; +import in.dragonbra.javasteam.depotdownloader.IDownloadListener; +import in.dragonbra.javasteam.depotdownloader.data.*; import in.dragonbra.javasteam.enums.EResult; -import in.dragonbra.javasteam.steam.contentdownloader.ContentDownloader; -import in.dragonbra.javasteam.steam.handlers.steamapps.SteamApps; -import in.dragonbra.javasteam.steam.handlers.steamapps.callback.FreeLicenseCallback; +import in.dragonbra.javasteam.steam.authentication.AuthPollResult; +import in.dragonbra.javasteam.steam.authentication.AuthSessionDetails; +import in.dragonbra.javasteam.steam.authentication.AuthenticationException; +import in.dragonbra.javasteam.steam.authentication.UserConsoleAuthenticator; +import in.dragonbra.javasteam.steam.handlers.steamapps.License; +import in.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback; import in.dragonbra.javasteam.steam.handlers.steamuser.LogOnDetails; import in.dragonbra.javasteam.steam.handlers.steamuser.SteamUser; import in.dragonbra.javasteam.steam.handlers.steamuser.callback.LoggedOffCallback; @@ -12,10 +18,20 @@ import in.dragonbra.javasteam.steam.steamclient.callbackmgr.CallbackManager; import in.dragonbra.javasteam.steam.steamclient.callbacks.ConnectedCallback; import in.dragonbra.javasteam.steam.steamclient.callbacks.DisconnectedCallback; +import in.dragonbra.javasteam.steam.steamclient.configuration.SteamConfiguration; import in.dragonbra.javasteam.util.log.DefaultLogListener; import in.dragonbra.javasteam.util.log.LogManager; - -import java.io.File; +import okhttp3.OkHttpClient; +import org.jetbrains.annotations.NotNull; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.TimeUnit; /** * @author Oxters @@ -25,23 +41,20 @@ *

* this sample introduces the usage of the content downloader API *

- * content downloader lets you download an app from a Steam depot given - * an app ID + * content downloader lets you download an app, pub file, or ugc item given some parameters. *

- * in this case, this sample will demonstrate how to download the free game - * called Rocky Mayhem + * in this case, this sample will ask which game app id you'd like to download. + * You can find the app id of a game by the url of the store page. + * For example "store.steampowered.com/app/1303350/Rocky_Mayhem/", where 1303350 is the app id. */ @SuppressWarnings("FieldCanBeLocal") -public class SampleDownloadApp implements Runnable { - - private final int ROCKY_MAYHEM_APP_ID = 1303350; - private final int ROCKY_MAYHEM_DEPOT_ID = 1303351; +public class SampleDownloadApp implements Runnable, IDownloadListener { private SteamClient steamClient; private CallbackManager manager; + private SteamUser steamUser; - private SteamApps steamApps; private boolean isRunning; @@ -49,12 +62,13 @@ public class SampleDownloadApp implements Runnable { private final String pass; - private final String twoFactor; + private List subscriptions; + + private List licenseList; - public SampleDownloadApp(String user, String pass, String twoFactor) { + public SampleDownloadApp(String user, String pass) { this.user = user; this.pass = pass; - this.twoFactor = twoFactor; } public static void main(String[] args) { @@ -65,68 +79,120 @@ public static void main(String[] args) { LogManager.addListener(new DefaultLogListener()); - String twoFactor = null; - if (args.length == 3) - twoFactor = args[2]; - new SampleDownloadApp(args[0], args[1], twoFactor).run(); + new SampleDownloadApp(args[0], args[1]).run(); } @Override public void run() { - // create our steamclient instance - steamClient = new SteamClient(); + // Most everything has been described in earlier samples. + // Anything pertaining to this sample will be commented. + + // Depot chunks are downloaded using OKHttp, it's best to set some timeouts. + var config = SteamConfiguration.create(builder -> { + builder.withHttpClient( + new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) // Time to establish connection + .readTimeout(60, TimeUnit.SECONDS) // Max inactivity between reads + .writeTimeout(30, TimeUnit.SECONDS) // Time for writes + .build() + ); + }); + + steamClient = new SteamClient(config); - // create the callback manager which will route callbacks to function calls manager = new CallbackManager(steamClient); - // get the steamuser handler, which is used for logging on after successfully connecting steamUser = steamClient.getHandler(SteamUser.class); - steamApps = steamClient.getHandler(SteamApps.class); - - // register a few callbacks we're interested in - // these are registered upon creation to a callback manager, which will then route the callbacks - // to the functions specified - manager.subscribe(ConnectedCallback.class, this::onConnected); - manager.subscribe(DisconnectedCallback.class, this::onDisconnected); - manager.subscribe(LoggedOnCallback.class, this::onLoggedOn); - manager.subscribe(LoggedOffCallback.class, this::onLoggedOff); + subscriptions = new ArrayList<>(); - manager.subscribe(FreeLicenseCallback.class, this::onFreeLicense); + subscriptions.add(manager.subscribe(ConnectedCallback.class, this::onConnected)); + subscriptions.add(manager.subscribe(DisconnectedCallback.class, this::onDisconnected)); + subscriptions.add(manager.subscribe(LoggedOnCallback.class, this::onLoggedOn)); + subscriptions.add(manager.subscribe(LoggedOffCallback.class, this::onLoggedOff)); + subscriptions.add(manager.subscribe(LicenseListCallback.class, this::onLicenseList)); isRunning = true; System.out.println("Connecting to steam..."); - // initiate the connection steamClient.connect(); - // create our callback handling loop while (isRunning) { - // in order for the callbacks to get routed, they need to be handled by the manager manager.runWaitCallbacks(1000L); } + + System.out.println("Closing " + subscriptions.size() + " subscriptions."); + for (var subscription : subscriptions) { + try { + subscription.close(); + } catch (IOException e) { + System.out.println("Couldn't close a callback."); + } + } } private void onConnected(ConnectedCallback callback) { System.out.println("Connected to Steam! Logging in " + user + "..."); - LogOnDetails details = new LogOnDetails(); - details.setUsername(user); - details.setPassword(pass); - if (twoFactor != null) { - details.setTwoFactorCode(twoFactor); - } + AuthSessionDetails authDetails = new AuthSessionDetails(); + authDetails.username = user; + authDetails.password = pass; + authDetails.deviceFriendlyName = "JavaSteam - Sample 023"; + authDetails.persistentSession = true; + + authDetails.authenticator = new UserConsoleAuthenticator(); + + try { + var path = Paths.get("refreshtoken.txt"); - // Set LoginID to a non-zero value if you have another client connected using the same account, - // the same private ip, and same public ip. - details.setLoginID(149); + String accountName; + String refreshToken; + if (!Files.exists(path)) { + System.out.println("No existing refresh token found. Beginning Authentication"); - steamUser.logOn(details); + var authSession = steamClient.getAuthentication().beginAuthSessionViaCredentials(authDetails).get(); + + AuthPollResult pollResponse = authSession.pollingWaitForResult().get(); + + accountName = pollResponse.getAccountName(); + refreshToken = pollResponse.getRefreshToken(); + + // Save our refresh token for automatic login on next sample run. + Files.writeString(path, pollResponse.getRefreshToken()); + } else { + System.out.println("Existing refresh token found"); + var token = Files.readString(path); + + accountName = user; + refreshToken = token; + } + + LogOnDetails details = new LogOnDetails(); + details.setUsername(accountName); + details.setAccessToken(refreshToken); + details.setShouldRememberPassword(true); + + details.setLoginID(149); + + System.out.println("Logging in..."); + + steamUser.logOn(details); + } catch (Exception e) { + if (e instanceof AuthenticationException) { + System.err.println("An Authentication error has occurred. " + e.getMessage()); + } else if (e instanceof CancellationException) { + System.err.println("An Cancellation exception was raised. Usually means a timeout occurred. " + e.getMessage()); + } else { + System.err.println("An error occurred:" + e.getMessage()); + } + + steamUser.logOff(); + } } private void onDisconnected(DisconnectedCallback callback) { - System.out.println("Disconnected from Steam"); + System.out.println("Disconnected from Steam, UserInitiated: " + callback.isUserInitiated()); if (callback.isUserInitiated()) { isRunning = false; @@ -143,9 +209,6 @@ private void onDisconnected(DisconnectedCallback callback) { private void onLoggedOn(LoggedOnCallback callback) { if (callback.getResult() != EResult.OK) { if (callback.getResult() == EResult.AccountLogonDenied) { - // if we receive AccountLogonDenied or one of its flavors (AccountLogonDeniedNoMailSent, etc.) - // then the account we're logging into is SteamGuard protected - // see sample 5 for how SteamGuard can be handled System.out.println("Unable to logon to Steam: This account is SteamGuard protected."); isRunning = false; @@ -161,36 +224,23 @@ private void onLoggedOn(LoggedOnCallback callback) { System.out.println("Successfully logged on!"); - // now that we are logged in, we can request a free license for Rocky Mayhem - steamApps.requestFreeLicense(ROCKY_MAYHEM_APP_ID); + // at this point, we'd be able to perform actions on Steam + // The sample continues in onLicenseList } - private void onFreeLicense(FreeLicenseCallback callback) { + private void onLicenseList(LicenseListCallback callback) { if (callback.getResult() != EResult.OK) { - System.out.println("Failed to get a free license for Rocky Mayhem"); + System.out.println("Failed to obtain licenses the account owns."); steamClient.disconnect(); return; } - // we have successfully received a free license for Rocky Mayhem so now we can start the download process - // note: it is okay to see some errors about ContentDownloader failing to download a chunk, it will retry and continue. - new File("steamapps/staging/").mkdirs(); - var contentDownloader = new ContentDownloader(steamClient); - contentDownloader.downloadApp( - ROCKY_MAYHEM_APP_ID, - ROCKY_MAYHEM_DEPOT_ID, - "steamapps/", - "steamapps/staging/", - "public", - 8, - progress -> System.out.println("Download progress: " + progress) - ).thenAccept(success -> { - if (success) { - System.out.println("Download completed successfully"); - } - steamClient.disconnect(); - }); + licenseList = callback.getLicenseList(); + + System.out.println("Got " + licenseList.size() + " licenses from account!"); + + downloadApp(); } private void onLoggedOff(LoggedOffCallback callback) { @@ -198,4 +248,118 @@ private void onLoggedOff(LoggedOffCallback callback) { isRunning = false; } + + private void downloadApp() { + // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. + // You will need to subscribe to LicenseListCallback to obtain your app licenses. + try (var depotDownloader = new DepotDownloader(steamClient, licenseList, true)) { + + // Add this class as a listener of IDownloadListener + depotDownloader.addListener(this); + + var pubItem = new PubFileItem( + /* (Required) appId */ 0, + /* (Required) pubFile */ 0, + /* (Optional) installToGameNameDirectory */ false, + /* (Optional) installDirectory */ null, + /* (Optional) verify */ false, + /* (Optional) downloadManifestOnly */ false + ); + + var ugcItem = new UgcItem( + /* (Required) appId */0, + /* (Required) ugcId */ 0, + /* (Optional) installToGameNameDirectory */ false, + /* (Optional) installDirectory */ null, + /* (Optional) verify */ false, + /* (Optional) downloadManifestOnly */ false + ); + + var appItem = new AppItem( + /* (Required) appId */ 1303350, + /* (Optional) installToGameNameDirectory */ true, + /* (Optional) installDirectory */ "steamapps", + /* (Optional) branch */ "public", + /* (Optional) branchPassword */ "", + /* (Optional) downloadAllPlatforms */ false, + /* (Optional) os */ "windows", + /* (Optional) downloadAllArchs */ false, + /* (Optional) osArch */ "64", + /* (Optional) downloadAllLanguages */ false, + /* (Optional) language */ "english", + /* (Optional) lowViolence */ false, + /* (Optional) depot */ List.of(), + /* (Optional) manifest */ List.of(), + /* (Optional) verify */ false, + /* (Optional) downloadManifestOnly */ false + ); + + // Items added are downloaded automatically in a FIFO (First-In, First-Out) queue. + + // Add a singular item to process. + depotDownloader.add(appItem); + + // You can add a List of items to be processed. + // depotDownloader.add(List.of(a, b, c)); + + // Signal the downloader that no more items will be added. + // Once all items in queue are done, 'completion' will signal that everything had finished. + depotDownloader.finishAdding(); + + // Block until we're done downloading. + // Note: If you did not call `finishAdding()` before awaiting, depotDownloader will be expecting + // more items to be added to queue. It may look like a hang. You could call `close()` to finish too. + depotDownloader.awaitCompletion(); + + // Kotlin users can use: + // depotDownloader.getCompletion().await() + + // Remove this class as a listener of IDownloadListener + depotDownloader.removeListener(this); + } finally { + System.out.println("Done Downloading"); + steamUser.logOff(); + } + } + + // Depot Downloader Callbacks. + + @Override + public void onItemAdded(@NotNull DownloadItem item) { + System.out.println("Item " + item.getAppId() + " added to queue."); + } + + @Override + public void onDownloadStarted(@NotNull DownloadItem item) { + System.out.println("Item " + item.getAppId() + " download started."); + } + + @Override + public void onDownloadCompleted(@NotNull DownloadItem item) { + System.out.println("Item " + item.getAppId() + " download completed."); + } + + @Override + public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { + System.out.println("Item " + item.getAppId() + " failed to download"); + System.err.println(error.getMessage()); + } + + @Override + public void onStatusUpdate(@NotNull String message) { + System.out.println("Status: " + message); + } + + @Override + public void onFileCompleted(int depotId, @NotNull String fileName, float depotPercentComplete) { + var complete = String.format("%.2f%%", depotPercentComplete * 100f); + System.out.println("Depot " + depotId + " with file " + fileName + " completed. " + complete); + } + + @Override + public void onDepotCompleted(int depotId, long compressedBytes, long uncompressedBytes) { + System.out.println("Depot " + depotId + " completed."); + System.out.println("\t" + compressedBytes + " compressed bytes"); + System.out.println("\t" + uncompressedBytes + " uncompressed bytes"); + } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 1dc5b8bc..fa1f8579 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,7 @@ rootProject.name = "javasteam" -include(":javasteam-samples") include(":javasteam-cs") +include(":javasteam-deadlock") +include(":javasteam-depotdownloader") +include(":javasteam-dota2") +include(":javasteam-samples") include(":javasteam-tf") -include("javasteam-deadlock") -include("javasteam-dota2") diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index 9f7591f4..c34da41c 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -6,22 +6,22 @@ import `in`.dragonbra.javasteam.types.ChunkData import `in`.dragonbra.javasteam.types.DepotManifest import `in`.dragonbra.javasteam.util.SteamKitWebRequestException import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger -import `in`.dragonbra.javasteam.util.stream.MemoryStream import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.future.future -import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import okhttp3.HttpUrl import okhttp3.OkHttpClient import okhttp3.Request -import java.io.ByteArrayOutputStream +import okhttp3.coroutines.executeAsync +import java.io.ByteArrayInputStream import java.io.Closeable import java.io.IOException -import java.util.concurrent.* -import java.util.zip.DataFormatException +import java.util.concurrent.CompletableFuture import java.util.zip.ZipInputStream /** @@ -32,64 +32,66 @@ import java.util.zip.ZipInputStream */ class Client(steamClient: SteamClient) : Closeable { - private val httpClient: OkHttpClient = steamClient.configuration.httpClient - - private val defaultScope = CoroutineScope(Dispatchers.IO) + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) companion object { - private val logger: Logger = LogManager.getLogger() - - /** - * Default timeout to use when making requests - */ - var requestTimeout = 10000L - - /** - * Default timeout to use when reading the response body - */ - var responseBodyTimeout = 60000L + private val logger: Logger = LogManager.getLogger(Client::class.java) - @JvmStatic - @JvmOverloads - fun buildCommand( + private fun buildCommand( server: Server, command: String, query: String? = null, proxyServer: Server? = null, ): HttpUrl { - // TODO look into this to mimic SK's method. Should be able to remove if/else and only have the if. - val httpUrl: HttpUrl + val scheme = if (server.protocol == Server.ConnectionProtocol.HTTP) "http" else "https" + var host = server.vHost ?: server.host ?: "" + var port = server.port + var path = command + if (proxyServer != null && proxyServer.useAsProxy && proxyServer.proxyRequestPathTemplate != null) { - httpUrl = HttpUrl.Builder() - .scheme(if (proxyServer.protocol == Server.ConnectionProtocol.HTTP) "http" else "https") - .host(proxyServer.vHost) - .port(proxyServer.port) - .addPathSegment(server.vHost) - .addPathSegments(command) - .run { - query?.let { this.query(it) } ?: this - }.build() - } else { - httpUrl = HttpUrl.Builder() - .scheme(if (server.protocol == Server.ConnectionProtocol.HTTP) "http" else "https") - .host(server.vHost) - .port(server.port) - .addPathSegments(command) - .run { - query?.let { this.query(it) } ?: this - }.build() + val pathTemplate = proxyServer.proxyRequestPathTemplate!! + .replace("%host%", host) + .replace("%path%", "/$command") + + host = proxyServer.vHost ?: proxyServer.host ?: "" + port = proxyServer.port + path = pathTemplate + } + + val urlBuilder = HttpUrl.Builder() + .scheme(scheme) + .host(host) + .port(port) + .addPathSegments(path.trimStart('/')) + + query?.let { queryString -> + if (queryString.isNotEmpty()) { + val params = queryString.split("&") + for (param in params) { + val keyValue = param.split("=", limit = 2) + if (keyValue.size == 2) { + urlBuilder.addQueryParameter(keyValue[0], keyValue[1]) + } else if (keyValue.size == 1 && keyValue[0].isNotEmpty()) { + urlBuilder.addQueryParameter(keyValue[0], "") + } + } + } } - return httpUrl + return urlBuilder.build() } } + private val httpClient: OkHttpClient = steamClient.configuration.httpClient + /** * Disposes of this object. */ override fun close() { + scope.cancel() httpClient.connectionPool.evictAll() + httpClient.dispatcher.executorService.shutdown() } /** @@ -106,7 +108,6 @@ class Client(steamClient: SteamClient) : Closeable { * @exception IllegalArgumentException [server] was null. * @exception IOException A network error occurred when performing the request. * @exception SteamKitWebRequestException A network error occurred when performing the request. - * @exception DataFormatException When the data received is not as expected */ suspend fun downloadManifest( depotId: Int, @@ -116,8 +117,9 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): DepotManifest { + ): DepotManifest = withContext(Dispatchers.IO) { val manifestVersion = 5 + val url = if (manifestRequestCode > 0U) { "depot/$depotId/manifest/$manifestId/$manifestVersion/$manifestRequestCode" } else { @@ -128,8 +130,10 @@ class Client(steamClient: SteamClient) : Closeable { .url(buildCommand(server, url, cdnAuthToken, proxyServer)) .build() - return withTimeout(requestTimeout) { - val response = httpClient.newCall(request).execute() + logger.debug("Request URL is: $request") + + try { + val response = httpClient.newCall(request).executeAsync() if (!response.isSuccessful) { throw SteamKitWebRequestException( @@ -138,93 +142,36 @@ class Client(steamClient: SteamClient) : Closeable { ) } - val depotManifest = withTimeout(responseBodyTimeout) { - val contentLength = response.header("Content-Length")?.toIntOrNull() + return@withContext response.use { resp -> + val responseBody = resp.body?.bytes() + ?: throw SteamKitWebRequestException("Response body is null") - if (contentLength == null) { - logger.debug("Manifest response does not have Content-Length, falling back to unbuffered read.") + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") } - response.body.byteStream().use { inputStream -> - ByteArrayOutputStream().use { bs -> - val bytesRead = inputStream.copyTo(bs, contentLength ?: DEFAULT_BUFFER_SIZE) + // Decompress the zipped manifest data + ZipInputStream(ByteArrayInputStream(responseBody)).use { zipInputStream -> + zipInputStream.nextEntry + ?: throw SteamKitWebRequestException("Expected the zip to contain at least one file") - if (bytesRead != contentLength?.toLong()) { - throw DataFormatException("Length mismatch after downloading depot manifest! (was $bytesRead, but should be $contentLength)") - } + val manifestData = zipInputStream.readBytes() - val contentBytes = bs.toByteArray() - - MemoryStream(contentBytes).use { ms -> - ZipInputStream(ms).use { zip -> - var entryCount = 0 - while (zip.nextEntry != null) { - entryCount++ - } - if (entryCount > 1) { - logger.debug("Expected the zip to contain only one file") - } - } - } + val depotManifest = DepotManifest.deserialize(ByteArrayInputStream(manifestData)) - // Decompress the zipped manifest data - MemoryStream(contentBytes).use { ms -> - ZipInputStream(ms).use { zip -> - zip.nextEntry - DepotManifest.deserialize(zip) - } - } + if (depotKey != null) { + depotManifest.decryptFilenames(depotKey) } - } - } - depotKey?.let { key -> - // if we have the depot key, decrypt the manifest filenames - depotManifest.decryptFilenames(key) + depotManifest + } } - - depotManifest + } catch (e: Exception) { + logger.error("Failed to download manifest ${request.url}: ${e.message}", e) + throw e } } - /** - * Java Compat: - * Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided. - * @param depotId The id of the depot being accessed. - * @param manifestId The unique identifier of the manifest to be downloaded. - * @param manifestRequestCode The manifest request code for the manifest that is being downloaded. - * @param server The content server to connect to. - * @param depotKey The depot decryption key for the depot that will be downloaded. - * This is used for decrypting filenames (if needed) in depot manifests. - * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. - * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. - * @return A [DepotManifest] instance that contains information about the files present within a depot. - * @exception IllegalArgumentException [server] was null. - * @exception IOException A network error occurred when performing the request. - * @exception SteamKitWebRequestException A network error occurred when performing the request. - * @exception DataFormatException When the data received is not as expected - */ - @JvmOverloads - fun downloadManifestFuture( - depotId: Int, - manifestId: Long, - manifestRequestCode: Long, - server: Server, - depotKey: ByteArray? = null, - proxyServer: Server? = null, - cdnAuthToken: String? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadManifest( - depotId = depotId, - manifestId = manifestId, - manifestRequestCode = manifestRequestCode.toULong(), - server = server, - depotKey = depotKey, - proxyServer = proxyServer, - cdnAuthToken = cdnAuthToken, - ) - } - /** * Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided. * This function will also validate the length of the downloaded chunk with the value of [ChunkData.compressedLength], @@ -252,7 +199,7 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): Int { + ): Int = withContext(Dispatchers.IO) { require(chunk.chunkID != null) { "Chunk must have a ChunkID." } if (depotKey == null) { @@ -268,82 +215,123 @@ class Client(steamClient: SteamClient) : Closeable { val chunkID = Strings.toHex(chunk.chunkID) val url = "depot/$depotId/chunk/$chunkID" - val request: Request = if (ClientLancache.useLanCacheServer) { - ClientLancache.buildLancacheRequest(server, url, cdnAuthToken) + val request = if (ClientLancache.useLanCacheServer) { + ClientLancache.buildLancacheRequest(server = server, command = url, query = cdnAuthToken) } else { - Request.Builder().url(buildCommand(server, url, cdnAuthToken, proxyServer)).build() + val url = buildCommand(server = server, command = url, query = cdnAuthToken, proxyServer = proxyServer) + Request.Builder() + .url(url) + .build() } - withTimeout(requestTimeout) { - httpClient.newCall(request).execute() - }.use { response -> - if (!response.isSuccessful) { - throw SteamKitWebRequestException( - "Response status code does not indicate success: ${response.code} (${response.message})", - response - ) - } + try { + val response = httpClient.newCall(request).executeAsync() - var contentLength = chunk.compressedLength + response.use { resp -> + if (!resp.isSuccessful) { + throw SteamKitWebRequestException( + "Response status code does not indicate success: ${resp.code} (${resp.message})", + resp + ) + } - response.header("Content-Length")?.toLongOrNull()?.let { responseContentLength -> - contentLength = responseContentLength.toInt() + var contentLength = chunk.compressedLength - // assert that lengths match only if the chunk has a length assigned. - if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { - throw IllegalStateException("Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})") - } - } ?: run { - if (contentLength > 0) { - logger.debug("Response does not have Content-Length, falling back to chunk.compressedLength.") + if (resp.body.contentLength().toInt() > 0) { + contentLength = resp.body.contentLength().toInt() + + // assert that lengths match only if the chunk has a length assigned. + if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { + throw SteamKitWebRequestException( + "Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})" + ) + } + } else if (contentLength > 0) { + logger.debug("Response does not have Content-Length, falling back to chunk.CompressedLength.") } else { throw SteamKitWebRequestException( - "Response does not have Content-Length and chunk.compressedLength is not set.", + "Response does not have Content-Length and chunk.CompressedLength is not set.", response ) } - } - // If no depot key is provided, stream into the destination buffer without renting - if (depotKey == null) { - val bytesRead = withTimeout(responseBodyTimeout) { - response.body.byteStream().use { input -> - input.readNBytesCompat(destination, 0, contentLength) - } + val responseBody = resp.body.bytes() + + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") } - if (bytesRead != contentLength) { - throw IOException("Length mismatch after downloading depot chunk! (was $bytesRead, but should be $contentLength)") + if (responseBody.size != contentLength) { + throw SteamKitWebRequestException( + "Length mismatch after downloading depot chunk! (was ${responseBody.size}, but should be $contentLength)" + ) } - return contentLength - } + if (depotKey == null) { + System.arraycopy(responseBody, 0, destination, 0, contentLength) + return@withContext contentLength + } - // We have to stream into a temporary buffer because a decryption will need to be performed - val buffer = ByteArray(contentLength) + return@withContext DepotChunk.process(chunk, responseBody, destination, depotKey) + } + } catch (e: Exception) { + logger.error("Failed to download a depot chunk ${request.url}: ${e.message}", e) + throw e + } + } - try { - val bytesRead = withTimeout(responseBodyTimeout) { - response.body.byteStream().use { input -> - input.readNBytesCompat(buffer, 0, contentLength) - } - } + // region Java Compatibility - if (bytesRead != contentLength) { - throw IOException("Length mismatch after downloading encrypted depot chunk! (was $bytesRead, but should be $contentLength)") - } + /** + * Java-compatible version of downloadManifest that returns a CompletableFuture. + * Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided. + * @param depotId The id of the depot being accessed. + * @param manifestId The unique identifier of the manifest to be downloaded. + * @param manifestRequestCode The manifest request code for the manifest that is being downloaded. + * @param server The content server to connect to. + * @param depotKey The depot decryption key for the depot that will be downloaded. + * This is used for decrypting filenames (if needed) in depot manifests. + * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. + * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. + * @return A CompletableFuture that will complete with a [DepotManifest] instance that contains information about the files present within a depot. + * @exception IllegalArgumentException [server] was null. + * @exception IOException A network error occurred when performing the request. + * @exception SteamKitWebRequestException A network error occurred when performing the request. + */ + @JvmOverloads + fun downloadManifestFuture( + depotId: Int, + manifestId: Long, + manifestRequestCode: Long, + server: Server, + depotKey: ByteArray? = null, + proxyServer: Server? = null, + cdnAuthToken: String? = null, + ): CompletableFuture { + val future = CompletableFuture() - // process the chunk immediately - return DepotChunk.process(chunk, buffer, destination, depotKey) - } catch (ex: Exception) { - logger.error("Failed to download a depot chunk ${request.url}", ex) - throw ex + scope.launch { + try { + val result = downloadManifest( + depotId = depotId, + manifestId = manifestId, + manifestRequestCode = manifestRequestCode.toULong(), + server = server, + depotKey = depotKey, + proxyServer = proxyServer, + cdnAuthToken = cdnAuthToken + ) + future.complete(result) + } catch (e: Exception) { + future.completeExceptionally(e) } } + + return future } /** - * Java Compat: + * Java-compatible version of downloadDepotChunk that returns a CompletableFuture. * Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided. * This function will also validate the length of the downloaded chunk with the value of [ChunkData.compressedLength], * if it has been assigned a value. @@ -357,7 +345,7 @@ class Client(steamClient: SteamClient) : Closeable { * This is used to process the chunk data. * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. - * @return The total number of bytes written to [destination]. + * @return A CompletableFuture that will complete with the total number of bytes written to [destination]. * @exception IllegalArgumentException Thrown if the chunk's [ChunkData.chunkID] was null or if the [destination] buffer is too small. * @exception IllegalStateException Thrown if the downloaded data does not match the expected length. * @exception SteamKitWebRequestException A network error occurred when performing the request. @@ -371,15 +359,28 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadDepotChunk( - depotId = depotId, - chunk = chunk, - server = server, - destination = destination, - depotKey = depotKey, - proxyServer = proxyServer, - cdnAuthToken = cdnAuthToken, - ) + ): CompletableFuture { + val future = CompletableFuture() + + scope.launch { + try { + val bytesWritten = downloadDepotChunk( + depotId = depotId, + chunk = chunk, + server = server, + destination = destination, + depotKey = depotKey, + proxyServer = proxyServer, + cdnAuthToken = cdnAuthToken + ) + future.complete(bytesWritten) + } catch (e: Exception) { + future.completeExceptionally(e) + } + } + + return future } + + // endregion } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt index eb3238ae..0eb9e699 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt @@ -1,80 +1,126 @@ package `in`.dragonbra.javasteam.steam.cdn -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.future.future -import okhttp3.HttpUrl +import kotlinx.coroutines.withContext +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Request import java.net.Inet4Address import java.net.Inet6Address import java.net.InetAddress -import java.util.concurrent.* /** * @author Lossy * @since 31/12/2024 */ -@Suppress("unused") object ClientLancache { - - private const val TRIGGER_DOMAIN: String = "lancache.steamcontent.com" - /** - * When set to true, will attempt to download from a Lancache instance on the LAN rather than going out to Steam's CDNs. + * When set to true, will attempt to download from a Lancache instance on the LAN + * rather than going out to Steam's CDNs. */ var useLanCacheServer: Boolean = false + private set + + private const val TRIGGER_DOMAIN = "lancache.steamcontent.com" /** - * Attempts to automatically resolve a Lancache on the local network. - * If detected, SteamKit will route all downloads through the cache rather than through Steam's CDN. + * Attempts to automatically resolve a Lancache on the local network. If detected, + * SteamKit will route all downloads through the cache rather than through Steam's CDN. * Will try to detect the Lancache through the poisoned DNS entry for lancache.steamcontent.com * - * This is a modified version from the original source : + * This is a modified version from the original source: * https://github.com/tpill90/lancache-prefill-common/blob/main/dotnet/LancacheIpResolver.cs */ - @JvmStatic - @JvmOverloads - fun detectLancacheServer(dispatcher: CoroutineDispatcher = Dispatchers.IO): CompletableFuture = - CoroutineScope(dispatcher).future { - val ipAddresses = InetAddress.getAllByName(TRIGGER_DOMAIN) - .filter { it is Inet4Address || it is Inet6Address } + suspend fun detectLancacheServer() { + withContext(Dispatchers.IO) { + try { + val addresses = InetAddress.getAllByName(TRIGGER_DOMAIN) + val ipAddresses = addresses.filter { address -> + address is Inet4Address || address is Inet6Address + } - useLanCacheServer = ipAddresses.any { isPrivateAddress(it) } + useLanCacheServer = ipAddresses.any { isPrivateAddress(it) } + } catch (_: Exception) { + useLanCacheServer = false + } } + } /** * Determines if an IP address is a private address, as specified in RFC1918 + * * @param toTest The IP address that will be tested - * @return true if the IP is a private address, false if it isn't private + * @return Returns true if the IP is a private address, false if it isn't private */ @JvmStatic fun isPrivateAddress(toTest: InetAddress): Boolean { - if (toTest.isLoopbackAddress) return true + if (toTest.isLoopbackAddress) { + return true + } + val bytes = toTest.address - return when (toTest) { - is Inet4Address -> when (bytes[0].toInt() and 0xFF) { + + // IPv4 + if (toTest is Inet4Address) { + // Convert signed byte to unsigned for comparison + val firstOctet = bytes[0].toInt() and 0xFF + + return when (firstOctet) { 10 -> true - 172 -> (bytes[1].toInt() and 0xFF) in 16..31 - 192 -> bytes[1].toInt() and 0xFF == 168 + 172 -> { + val secondOctet = bytes[1].toInt() and 0xFF + secondOctet in 16..<32 + } + + 192 -> { + val secondOctet = bytes[1].toInt() and 0xFF + secondOctet == 168 + } + else -> false } - is Inet6Address -> (bytes[0].toInt() and 0xFE) == 0xFC || toTest.isLinkLocalAddress - else -> false } + + // IPv6 + if (toTest is Inet6Address) { + // Check for Unique Local Address (fc00::/7) and link-local + val firstByte = bytes[0].toInt() and 0xFF + return (firstByte and 0xFE) == 0xFC || toTest.isLinkLocalAddress + } + + return false } - fun buildLancacheRequest(server: Server, command: String, query: String?): Request = Request.Builder() - .url( - HttpUrl.Builder() - .scheme("http") - .host("lancache.steamcontent.com") - .port(80) - .addPathSegments(command) - .query(query) - .build() - ) - .header("Host", server.host) - .header("User-Agent", "Valve/Steam HTTP Client 1.0") - .build() + /** + * Builds an HTTP request for Lancache with proper headers + * + * @param server The server to route the request through + * @param command The API command/path + * @param query Optional query parameters + * @return OkHttp Request object configured for Lancache + */ + fun buildLancacheRequest(server: Server, command: String, query: String? = null): Request { + val urlBuilder = "http://lancache.steamcontent.com:80".toHttpUrl().newBuilder() + .addPathSegments(command.trimStart('/')) + + query?.let { queryString -> + if (queryString.isNotEmpty()) { + val params = queryString.split("&") + for (param in params) { + val keyValue = param.split("=", limit = 2) + if (keyValue.size == 2) { + urlBuilder.addQueryParameter(keyValue[0], keyValue[1]) + } else if (keyValue.size == 1 && keyValue[0].isNotEmpty()) { + urlBuilder.addQueryParameter(keyValue[0], "") + } + } + } + } + + return Request.Builder() + .url(urlBuilder.build()) + .header("Host", server.host ?: "") + // User agent must match the Steam client in order for Lancache to correctly identify and cache Valve's CDN content + .header("User-Agent", "Valve/Steam HTTP Client 1.0") + .build() + } } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt deleted file mode 100644 index ee377dd2..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt +++ /dev/null @@ -1,142 +0,0 @@ -package `in`.dragonbra.javasteam.steam.cdn - -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Job -import kotlinx.coroutines.async -import kotlinx.coroutines.cancel -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import java.util.concurrent.ConcurrentLinkedDeque -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -/** - * [ClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed - */ -class ClientPool(internal val steamClient: SteamClient, private val appId: Int, private val parentScope: CoroutineScope) { - - companion object { - private const val SERVER_ENDPOINT_MIN_SIZE = 8 - } - - val cdnClient: Client = Client(steamClient) - - var proxyServer: Server? = null - private set - - private val activeConnectionPool = ConcurrentLinkedDeque() - - private val availableServerEndpoints = ConcurrentLinkedQueue() - - private val populatePoolEvent = CountDownLatch(1) - - private val monitorJob: Job - - private val logger: Logger = LogManager.getLogger() - - init { - monitorJob = parentScope.launch { connectionPoolMonitor().await() } - } - - fun shutdown() { - monitorJob.cancel() - } - - private fun fetchBootstrapServerList(): Deferred?> = parentScope.async { - return@async try { - steamClient.getHandler(SteamContent::class.java)?.getServersForSteamPipe(parentScope = parentScope)?.await() - } catch (ex: Exception) { - logger.error("Failed to retrieve content server list", ex) - - null - } - } - - private fun connectionPoolMonitor() = parentScope.async { - var didPopulate = false - - while (isActive) { - populatePoolEvent.await(1, TimeUnit.SECONDS) - - if (availableServerEndpoints.size < SERVER_ENDPOINT_MIN_SIZE && steamClient.isConnected) { - val servers = fetchBootstrapServerList().await() - - if (servers.isNullOrEmpty()) { - logger.error("Servers is empty or null, exiting connection pool monitor") - parentScope.cancel() - return@async - } - - proxyServer = servers.find { it.useAsProxy } - - val weightedCdnServers = servers - .filter { server -> - val isEligibleForApp = server.allowedAppIds.isEmpty() || appId in server.allowedAppIds - isEligibleForApp && (server.type == "SteamCache" || server.type == "CDN") - } - .sortedBy { it.weightedLoad } - - for (server in weightedCdnServers) { - repeat(server.numEntries) { - availableServerEndpoints.offer(server) - } - } - - didPopulate = true - } else if (availableServerEndpoints.isEmpty() && !steamClient.isConnected && didPopulate) { - logger.error("Available server endpoints is empty and steam is not connected, exiting connection pool monitor") - - parentScope.cancel() - - return@async - } - } - } - - private fun buildConnection(): Deferred = parentScope.async { - return@async try { - if (availableServerEndpoints.size < SERVER_ENDPOINT_MIN_SIZE) { - populatePoolEvent.countDown() - } - - var output: Server? = null - - while (isActive && availableServerEndpoints.poll().also { output = it } == null) { - delay(1000) - } - - output - } catch (e: Exception) { - logger.error("Failed to build connection", e) - - null - } - } - - internal fun getConnection(): Deferred = parentScope.async { - return@async try { - val server = activeConnectionPool.poll() ?: buildConnection().await() - server - } catch (e: Exception) { - logger.error("Failed to get/build connection", e) - - null - } - } - - internal fun returnConnection(server: Server?) { - server?.let { activeConnectionPool.push(it) } - } - - @Suppress("unused") - internal fun returnBrokenConnection(server: Server?) { - // Broken connections are not returned to the pool - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt index 6285449f..db744e3c 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt @@ -51,15 +51,16 @@ object DepotChunk { require(iv.size == ivBytesRead) { "Failed to decrypt depot chunk iv (${iv.size} != $ivBytesRead)" } + val aes = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) + aes.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) + // With CBC and padding, the decrypted size will always be smaller val buffer = ByteArray(data.size - iv.size) - val cbcCipher = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) - cbcCipher.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) val writtenDecompressed: Int try { - val written = cbcCipher.doFinal(data, iv.size, data.size - iv.size, buffer) + val written = aes.doFinal(data, iv.size, data.size - iv.size, buffer) // Per SK: // Steam client checks for like 20 bytes for pkzip, and 22 bytes for vzip, diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt index 7b9aa104..82a2035f 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt @@ -1,24 +1,35 @@ package `in`.dragonbra.javasteam.steam.cdn +import java.net.InetSocketAddress + /** * Represents a single Steam3 'Steampipe' content server. */ -class Server @JvmOverloads constructor( - protocol: ConnectionProtocol = ConnectionProtocol.HTTP, - host: String, - vHost: String, - port: Int, - type: String? = null, - sourceID: Int = 0, - cellID: Int = 0, - load: Int = 0, - weightedLoad: Float = 0f, - numEntries: Int = 0, - steamChinaOnly: Boolean = false, - useAsProxy: Boolean = false, - proxyRequestPathTemplate: String? = null, - allowedAppIds: IntArray = IntArray(0), -) { +class Server { + + companion object { + /** + * Creates a Server from an InetSocketAddress. + */ + @JvmStatic + fun fromInetSocketAddress(endPoint: InetSocketAddress): Server = Server().apply { + protocol = if (endPoint.port == 443) ConnectionProtocol.HTTPS else ConnectionProtocol.HTTP + host = endPoint.address.hostAddress + vHost = endPoint.address.hostAddress + port = endPoint.port + } + + /** + * Creates a Server from hostname and port. + */ + @JvmStatic + fun fromHostAndPort(hostname: String, port: Int): Server = Server().apply { + protocol = if (port == 443) ConnectionProtocol.HTTPS else ConnectionProtocol.HTTP + host = hostname + vHost = hostname + this.port = port + } + } /** * The protocol used to connect to this server @@ -38,86 +49,85 @@ class Server @JvmOverloads constructor( /** * Gets the supported connection protocol of the server. */ - var protocol = protocol + var protocol: ConnectionProtocol = ConnectionProtocol.HTTP internal set /** * Gets the hostname of the server. */ - var host = host + var host: String? = null internal set /** * Gets the virtual hostname of the server. */ - var vHost = vHost + var vHost: String? = null internal set /** * Gets the port of the server. */ - var port = port + var port: Int = 0 internal set /** * Gets the type of the server. */ - var type = type + var type: String? = null internal set /** * Gets the SourceID this server belongs to. */ - @Suppress("unused") - var sourceID = sourceID + var sourceId: Int = 0 internal set /** * Gets the CellID this server belongs to. */ - var cellID = cellID + var cellId: Int = 0 internal set /** * Gets the load value associated with this server. */ - var load = load + var load: Int = 0 internal set /** * Gets the weighted load. */ - var weightedLoad = weightedLoad + var weightedLoad: Float = 0F internal set /** * Gets the number of entries this server is worth. */ - var numEntries = numEntries + var numEntries: Int = 0 internal set /** * Gets the flag whether this server is for Steam China only. */ - var steamChinaOnly = steamChinaOnly + var steamChinaOnly: Boolean = false internal set /** * Gets the download proxy status. */ - var useAsProxy = useAsProxy + var useAsProxy: Boolean = false internal set /** * Gets the transformation template applied to request paths. */ - var proxyRequestPathTemplate = proxyRequestPathTemplate + var proxyRequestPathTemplate: String? = null internal set /** * Gets the list of app ids this server can be used with. */ - var allowedAppIds = allowedAppIds + var allowedAppIds: IntArray = intArrayOf() internal set /** diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt deleted file mode 100644 index 2004de2b..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.ChunkData - -data class ChunkMatch( - val oldChunk: ChunkData, - val newChunk: ChunkData, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt deleted file mode 100644 index 2a638d65..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt +++ /dev/null @@ -1,724 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.enums.EDepotFileFlag -import `in`.dragonbra.javasteam.enums.EResult -import `in`.dragonbra.javasteam.steam.cdn.ClientPool -import `in`.dragonbra.javasteam.steam.cdn.Server -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest -import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient -import `in`.dragonbra.javasteam.types.ChunkData -import `in`.dragonbra.javasteam.types.DepotManifest -import `in`.dragonbra.javasteam.types.FileData -import `in`.dragonbra.javasteam.types.KeyValue -import `in`.dragonbra.javasteam.util.Adler32 -import `in`.dragonbra.javasteam.util.SteamKitWebRequestException -import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.Utils -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.future.future -import kotlinx.coroutines.isActive -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit -import java.io.File -import java.io.FileInputStream -import java.io.FileOutputStream -import java.io.RandomAccessFile -import java.nio.ByteBuffer -import java.nio.file.Paths -import java.time.Instant -import java.time.temporal.ChronoUnit -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentLinkedQueue - -@Suppress("unused", "SpellCheckingInspection") -class ContentDownloader(val steamClient: SteamClient) { - - companion object { - private const val HTTP_UNAUTHORIZED = 401 - private const val HTTP_FORBIDDEN = 403 - private const val HTTP_NOT_FOUND = 404 - private const val SERVICE_UNAVAILABLE = 503 - - internal const val INVALID_APP_ID = Int.MAX_VALUE - internal const val INVALID_MANIFEST_ID = Long.MAX_VALUE - - private val logger: Logger = LogManager.getLogger(ContentDownloader::class.java) - } - - private val defaultScope = CoroutineScope(Dispatchers.IO) - - private fun requestDepotKey( - appId: Int, - depotId: Int, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.getDepotDecryptionKey(depotId, appId)?.await() - - return@async Pair(callback?.result ?: EResult.Fail, callback?.depotKey) - } - - private fun getDepotManifestId( - app: PICSProductInfo, - depotId: Int, - branchId: String, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val depot = app.keyValues["depots"][depotId.toString()] - if (depot == KeyValue.INVALID) { - logger.error("Could not find depot $depotId of ${app.id}") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val manifest = depot["manifests"][branchId] - if (manifest != KeyValue.INVALID) { - return@async Pair(app.id, manifest["gid"].asLong()) - } - - val depotFromApp = depot["depotfromapp"].asInteger(INVALID_APP_ID) - if (depotFromApp == app.id || depotFromApp == INVALID_APP_ID) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val innerApp = getAppInfo(depotFromApp, parentScope).await() - if (innerApp == null) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - return@async getDepotManifestId(innerApp, depotId, branchId, parentScope).await() - } - - private fun getAppDirName(app: PICSProductInfo): String { - val installDirKeyValue = app.keyValues["config"]["installdir"] - - return if (installDirKeyValue != KeyValue.INVALID) installDirKeyValue.value!! else app.id.toString() - } - - private fun getAppInfo( - appId: Int, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.picsGetProductInfo(PICSRequest(appId))?.await() - val apps = callback?.results?.flatMap { it.apps.values } - - if (apps.isNullOrEmpty()) { - logger.error("Received empty apps list in PICSProductInfo response for $appId") - return@async null - } - - if (apps.size > 1) { - logger.debug("Received ${apps.size} apps from PICSProductInfo for $appId, using first result") - } - - return@async apps.first() - } - - /** - * Kotlin coroutines version - */ - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope = defaultScope, - ): Deferred = parentScope.async { - downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = onDownloadProgress, - scope = parentScope - ) - } - - /** - * Java-friendly version that returns a CompletableFuture - */ - @JvmOverloads - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - progressCallback: ProgressCallback? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = progressCallback?.let { callback -> { progress -> callback.onProgress(progress) } }, - scope = defaultScope - ) - } - - private suspend fun downloadAppInternal( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - scope: CoroutineScope, - ): Boolean { - if (!scope.isActive) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - return false - } - - val cdnPool = ClientPool(steamClient, appId, scope) - - val shiftedAppId: Int - val manifestId: Long - val appInfo = getAppInfo(appId, scope).await() - - if (appInfo == null) { - logger.error("Could not retrieve PICSProductInfo of $appId") - return false - } - - getDepotManifestId(appInfo, depotId, branch, scope).await().apply { - shiftedAppId = first - manifestId = second - } - - val depotKeyResult = requestDepotKey(shiftedAppId, depotId, scope).await() - - if (depotKeyResult.first != EResult.OK || depotKeyResult.second == null) { - logger.error("Depot key request for $appId failed with result ${depotKeyResult.first}") - return false - } - - val depotKey = depotKeyResult.second!! - - var newProtoManifest = steamClient.configuration.depotManifestProvider.fetchManifest(depotId, manifestId) - var oldProtoManifest = steamClient.configuration.depotManifestProvider.fetchLatestManifest(depotId) - - if (oldProtoManifest?.manifestGID == manifestId) { - oldProtoManifest = null - } - - // In case we have an early exit, this will force equiv of verifyall next run. - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, INVALID_MANIFEST_ID) - - try { - if (newProtoManifest == null) { - newProtoManifest = - downloadFilesManifestOf(shiftedAppId, depotId, manifestId, branch, depotKey, cdnPool, scope).await() - } else { - logger.debug("Already have manifest $manifestId for depot $depotId.") - } - - if (newProtoManifest == null) { - logger.error("Failed to retrieve files manifest for app: $shiftedAppId depot: $depotId manifest: $manifestId branch: $branch") - return false - } - - if (!scope.isActive) { - return false - } - - val downloadCounter = GlobalDownloadCounter() - val installDir = Paths.get(installPath, getAppDirName(appInfo)).toString() - val stagingDir = Paths.get(stagingPath, getAppDirName(appInfo)).toString() - val depotFileData = DepotFilesData( - depotDownloadInfo = DepotDownloadInfo(depotId, shiftedAppId, manifestId, branch, installDir, depotKey), - depotCounter = DepotDownloadCounter( - completeDownloadSize = newProtoManifest.totalUncompressedSize - ), - stagingDir = stagingDir, - manifest = newProtoManifest, - previousManifest = oldProtoManifest - ) - - downloadDepotFiles(cdnPool, downloadCounter, depotFileData, maxDownloads, onDownloadProgress, scope).await() - - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, manifestId) - - cdnPool.shutdown() - - // delete the staging directory of this app - File(stagingDir).deleteRecursively() - - logger.debug( - "Depot $depotId - Downloaded ${depotFileData.depotCounter.depotBytesCompressed} " + - "bytes (${depotFileData.depotCounter.depotBytesUncompressed} bytes uncompressed)" - ) - - return true - } catch (e: CancellationException) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - - return false - } catch (e: Exception) { - logger.error("Error occurred while downloading app $shiftedAppId", e) - - return false - } - } - - private fun downloadDepotFiles( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - maxDownloads: Int, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!parentScope.isActive) { - return@async - } - - depotFilesData.manifest.files.forEach { file -> - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - if (file.flags.contains(EDepotFileFlag.Directory)) { - File(fileFinalPath).mkdirs() - File(fileStagingPath).mkdirs() - } else { - // Some manifests don't explicitly include all necessary directories - File(fileFinalPath).parentFile.mkdirs() - File(fileStagingPath).parentFile.mkdirs() - } - } - - logger.debug("Downloading depot ${depotFilesData.depotDownloadInfo.depotId}") - - val files = depotFilesData.manifest.files.filter { !it.flags.contains(EDepotFileFlag.Directory) }.toTypedArray() - val networkChunkQueue = ConcurrentLinkedQueue>() - - val downloadSemaphore = Semaphore(maxDownloads) - files.map { file -> - async { - downloadSemaphore.withPermit { - downloadDepotFile(depotFilesData, file, networkChunkQueue, onDownloadProgress, parentScope).await() - } - } - }.awaitAll() - - networkChunkQueue.map { (fileStreamData, fileData, chunk) -> - async { - downloadSemaphore.withPermit { - downloadSteam3DepotFileChunk( - cdnPool = cdnPool, - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = fileData, - fileStreamData = fileStreamData, - chunk = chunk, - onDownloadProgress = onDownloadProgress, - parentScope = parentScope - ).await() - } - } - }.awaitAll() - - // Check for deleted files if updating the depot. - depotFilesData.previousManifest?.apply { - val previousFilteredFiles = files.asSequence().map { it.fileName }.toMutableSet() - - // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names - previousFilteredFiles.removeAll(depotFilesData.manifest.files.map { it.fileName }.toSet()) - - for (existingFileName in previousFilteredFiles) { - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, existingFileName).toString() - - if (!File(fileFinalPath).exists()) { - continue - } - - File(fileFinalPath).delete() - logger.debug("Deleted $fileFinalPath") - } - } - } - - private fun downloadDepotFile( - depotFilesData: DepotFilesData, - file: FileData, - networkChunkQueue: ConcurrentLinkedQueue>, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depotDownloadCounter = depotFilesData.depotCounter - val oldManifestFile = depotFilesData.previousManifest?.files?.find { it.fileName == file.fileName } - - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - // This may still exist if the previous run exited before cleanup - File(fileStagingPath).takeIf { it.exists() }?.delete() - - val neededChunks: MutableList - val fi = File(fileFinalPath) - val fileDidExist = fi.exists() - - if (!fileDidExist) { - // create new file. need all chunks - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - } - - neededChunks = file.chunks.toMutableList() - } else { - // open existing - if (oldManifestFile != null) { - neededChunks = mutableListOf() - - val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) - if (!hashMatches) { - logger.debug("Validating $fileFinalPath") - - val matchingChunks = mutableListOf() - - for (chunk in file.chunks) { - val oldChunk = oldManifestFile.chunks.find { it.chunkID.contentEquals(chunk.chunkID) } - if (oldChunk != null) { - matchingChunks.add(ChunkMatch(oldChunk, chunk)) - } else { - neededChunks.add(chunk) - } - } - - val orderedChunks = matchingChunks.sortedBy { it.oldChunk.offset } - - val copyChunks = mutableListOf() - - FileInputStream(fileFinalPath).use { fsOld -> - for (match in orderedChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - val adler = Adler32.calculate(tmp) - if (adler != match.oldChunk.checksum) { - neededChunks.add(match.newChunk) - } else { - copyChunks.add(match) - } - } - } - - if (neededChunks.isNotEmpty()) { - File(fileFinalPath).renameTo(File(fileStagingPath)) - - FileInputStream(fileStagingPath).use { fsOld -> - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - - for (match in copyChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - fs.channel.position(match.newChunk.offset) - fs.write(tmp) - } - } - } - - File(fileStagingPath).delete() - } - } - } else { - // No old manifest or file not in old manifest. We must validate. - RandomAccessFile(fileFinalPath, "rw").use { fs -> - if (fi.length() != file.totalSize) { - fs.channel.truncate(file.totalSize) - } - - logger.debug("Validating $fileFinalPath") - neededChunks = Utils.validateSteam3FileChecksums( - fs, - file.chunks.sortedBy { it.offset }.toTypedArray() - ) - } - } - - if (neededChunks.isEmpty()) { - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += file.totalSize - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - - return@async - } - - val sizeOnDisk = file.totalSize - neededChunks.sumOf { it.uncompressedLength.toLong() } - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += sizeOnDisk - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - } - - val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) - if (fileIsExecutable && - (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) - ) { - File(fileFinalPath).setExecutable(true) - } else if (!fileIsExecutable && oldManifestFile != null && oldManifestFile.flags.contains(EDepotFileFlag.Executable)) { - File(fileFinalPath).setExecutable(false) - } - - val fileStreamData = FileStreamData( - fileStream = null, - fileLock = Semaphore(1), - chunksToDownload = neededChunks.size - ) - - for (chunk in neededChunks) { - networkChunkQueue.add(Triple(fileStreamData, file, chunk)) - } - } - - private fun downloadSteam3DepotFileChunk( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - file: FileData, - fileStreamData: FileStreamData, - chunk: ChunkData, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depot = depotFilesData.depotDownloadInfo - val depotDownloadCounter = depotFilesData.depotCounter - - val chunkID = Strings.toHex(chunk.chunkID) - - var outputChunkData = ByteArray(chunk.uncompressedLength) - var writtenBytes = 0 - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - outputChunkData = ByteArray(chunk.uncompressedLength) - writtenBytes = cdnPool.cdnClient.downloadDepotChunk( - depotId = depot.depotId, - chunk = chunk, - server = connection!!, - destination = outputChunkData, - depotKey = depot.depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - when (e.statusCode) { - HTTP_UNAUTHORIZED, HTTP_FORBIDDEN -> { - logger.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") - break - } - - else -> logger.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") - } - } catch (e: NoClassDefFoundError) { - // Zstd is a 'compileOnly' dependency. - throw CancellationException(e.message) - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered unexpected error downloading chunk $chunkID", e) - } - } while (isActive && writtenBytes <= 0) - - if (writtenBytes <= 0) { - logger.error("Failed to find any server with chunk $chunkID for depot ${depot.depotId}. Aborting.") - throw CancellationException("Failed to download chunk") - } - - try { - fileStreamData.fileLock.acquire() - - if (fileStreamData.fileStream == null) { - val fileFinalPath = Paths.get(depot.installDir, file.fileName).toString() - val randomAccessFile = RandomAccessFile(fileFinalPath, "rw") - fileStreamData.fileStream = randomAccessFile.channel - } - - fileStreamData.fileStream?.position(chunk.offset) - fileStreamData.fileStream?.write(ByteBuffer.wrap(outputChunkData, 0, writtenBytes)) - } finally { - fileStreamData.fileLock.release() - } - - val remainingChunks = synchronized(fileStreamData) { - --fileStreamData.chunksToDownload - } - if (remainingChunks <= 0) { - fileStreamData.fileStream?.close() - } - - var sizeDownloaded: Long - synchronized(depotDownloadCounter) { - sizeDownloaded = depotDownloadCounter.sizeDownloaded + outputChunkData.size - depotDownloadCounter.sizeDownloaded = sizeDownloaded - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - - onDownloadProgress?.invoke( - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - ) - } - - private fun downloadFilesManifestOf( - appId: Int, - depotId: Int, - manifestId: Long, - branch: String, - depotKey: ByteArray, - cdnPool: ClientPool, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - if (!isActive) { - return@async null - } - - var depotManifest: DepotManifest? = null - var manifestRequestCode = 0UL - var manifestRequestCodeExpiration = Instant.MIN - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - if (connection == null) continue - - val now = Instant.now() - - // In order to download this manifest, we need the current manifest request code - // The manifest request code is only valid for a specific period of time - if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { - val steamContent = steamClient.getHandler(SteamContent::class.java)!! - - manifestRequestCode = steamContent.getManifestRequestCode( - depotId = depotId, - appId = appId, - manifestId = manifestId, - branch = branch, - parentScope = parentScope - ).await() - - // This code will hopefully be valid for one period following the issuing period - manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) - - // If we could not get the manifest code, this is a fatal error - if (manifestRequestCode == 0UL) { - throw CancellationException("No manifest request code was returned for manifest $manifestId in depot $depotId") - } - } - - depotManifest = cdnPool.cdnClient.downloadManifest( - depotId = depotId, - manifestId = manifestId, - manifestRequestCode = manifestRequestCode, - server = connection, - depotKey = depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: CancellationException) { - logger.error("Connection timeout downloading depot manifest $depotId $manifestId") - - return@async null - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - val statusName = when (e.statusCode) { - HTTP_UNAUTHORIZED -> HTTP_UNAUTHORIZED::class.java.name - HTTP_FORBIDDEN -> HTTP_FORBIDDEN::class.java.name - HTTP_NOT_FOUND -> HTTP_NOT_FOUND::class.java.name - SERVICE_UNAVAILABLE -> SERVICE_UNAVAILABLE::class.java.name - else -> null - } - - logger.error( - "Downloading of manifest $manifestId failed for depot $depotId with " + - if (statusName != null) { - "response of $statusName(${e.statusCode})" - } else { - "status code of ${e.statusCode}" - } - ) - - return@async null - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered error downloading manifest for depot $depotId $manifestId", e) - - return@async null - } - } while (isActive && depotManifest == null) - - if (depotManifest == null) { - throw CancellationException("Unable to download manifest $manifestId for depot $depotId") - } - - val newProtoManifest = depotManifest - steamClient.configuration.depotManifestProvider.updateManifest(newProtoManifest) - - return@async newProtoManifest - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt deleted file mode 100644 index dd31905a..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -data class DepotDownloadCounter( - var completeDownloadSize: Long = 0, - var sizeDownloaded: Long = 0, - var depotBytesCompressed: Long = 0, - var depotBytesUncompressed: Long = 0, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt deleted file mode 100644 index ff31b973..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt +++ /dev/null @@ -1,11 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -@Suppress("ArrayInDataClass") -data class DepotDownloadInfo( - val depotId: Int, - val appId: Int, - val manifestId: Long, - val branch: String, - val installDir: String, - val depotKey: ByteArray?, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt deleted file mode 100644 index f2e01726..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt +++ /dev/null @@ -1,11 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -data class DepotFilesData( - val depotDownloadInfo: DepotDownloadInfo, - val depotCounter: DepotDownloadCounter, - val stagingDir: String, - val manifest: DepotManifest, - val previousManifest: DepotManifest?, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt deleted file mode 100644 index ec7201b6..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt +++ /dev/null @@ -1,210 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import `in`.dragonbra.javasteam.util.stream.MemoryStream -import java.io.ByteArrayOutputStream -import java.io.File -import java.io.IOException -import java.nio.ByteBuffer -import java.nio.file.Files -import java.nio.file.Path -import java.util.zip.CRC32 -import java.util.zip.ZipEntry -import java.util.zip.ZipInputStream -import java.util.zip.ZipOutputStream - -/** - * Depot manifest provider that stores depot manifests in a zip file. - * @constructor Instantiates a [FileManifestProvider] object. - * @param file the file that will store the depot manifests - * - * @author Oxters - * @since 2024-11-07 - */ -@Suppress("unused") -class FileManifestProvider(private val file: Path) : IManifestProvider { - - /** - * Instantiates a [FileManifestProvider] object. - * @param file the file that will store the depot manifests. - */ - constructor(file: File) : this(file.toPath()) - - /** - * Instantiates a [FileManifestProvider] object. - * @param filename the filename that will store the depot manifests. - */ - constructor(filename: String) : this(Path.of(filename)) - - init { - require(file.fileName.toString().isNotBlank()) { "FileName must not be blank" } - } - - companion object { - private val logger: Logger = LogManager.getLogger(FileManifestProvider::class.java) - - private fun getLatestEntryName(depotID: Int): String = "$depotID${File.separator}latest" - - private fun getEntryName(depotID: Int, manifestID: Long): String = "$depotID${File.separator}$manifestID.bin" - - private fun seekToEntry(zipStream: ZipInputStream, entryName: String): ZipEntry? { - var zipEntry: ZipEntry? - - do { - zipEntry = zipStream.nextEntry - if (zipEntry?.name.equals(entryName, true)) { - break - } - } while (zipEntry != null) - - return zipEntry - } - - private fun copyZip(from: ZipInputStream, to: ZipOutputStream, vararg excludeEntries: String) { - var entry = from.nextEntry - - while (entry != null) { - if (!excludeEntries.contains(entry.name) && (entry.isDirectory || (!entry.isDirectory && entry.size > 0))) { - to.putNextEntry(entry) - - if (!entry.isDirectory) { - val entryBytes = ByteArray(entry.size.toInt()) - - from.readNBytesCompat(entryBytes, 0, entryBytes.size) - to.write(entryBytes) - } - - to.closeEntry() - } - - entry = from.nextEntry - } - } - - private fun zipUncompressed(zip: ZipOutputStream, entryName: String, bytes: ByteArray) { - val entry = ZipEntry(entryName).apply { - method = ZipEntry.STORED - size = bytes.size.toLong() - compressedSize = bytes.size.toLong() - crc = CRC32().run { - update(bytes) - value - } - } - - zip.putNextEntry(entry) - zip.write(bytes) - zip.closeEntry() - } - } - - override fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? = runCatching { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zip -> - seekToEntry(zip, getEntryName(depotID, manifestID))?.let { - if (it.size > 0) { - DepotManifest.deserialize(zip) - } else { - null - } - } - } - } - }.fold( - onSuccess = { it }, - onFailure = { error -> - when (error) { - is NoSuchFileException -> logger.debug("File doesn't exist") - else -> logger.error("Unknown error occurred", error) - } - - null - } - ) - - override fun fetchLatestManifest(depotID: Int): DepotManifest? = runCatching { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zip -> - seekToEntry(zip, getLatestEntryName(depotID))?.let { idEntry -> - if (idEntry.size > 0) { - ByteBuffer.wrap(zip.readNBytesCompat(idEntry.size.toInt())).getLong() - } else { - null - } - } - } - }?.let { manifestId -> - fetchManifest(depotID, manifestId) - } - }.fold( - onSuccess = { it }, - onFailure = { error -> - when (error) { - is NoSuchFileException -> logger.debug("File doesn't exist") - else -> logger.error("Unknown error occurred", error) - } - - null - } - ) - - override fun setLatestManifestId(depotID: Int, manifestID: Long) { - ByteArrayOutputStream().use { bs -> - ZipOutputStream(bs).use { zip -> - // copy old file only if it exists - if (Files.exists(file)) { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zs -> - copyZip(zs, zip, getLatestEntryName(depotID)) - } - } - } - // write manifest id as uncompressed data - ByteBuffer.allocate(Long.SIZE_BYTES).apply { - putLong(manifestID) - zipUncompressed(zip, getLatestEntryName(depotID), array()) - } - } - // save all data to the file - try { - Files.newOutputStream(file).use { fos -> - fos.write(bs.toByteArray()) - } - } catch (e: IOException) { - logger.error("Failed to write manifest ID to file ${file.fileName}", e) - } - } - } - - override fun updateManifest(manifest: DepotManifest) { - ByteArrayOutputStream().use { bs -> - ZipOutputStream(bs).use { zip -> - // copy old file only if it exists - if (Files.exists(file)) { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zs -> - copyZip(zs, zip, getEntryName(manifest.depotID, manifest.manifestGID)) - } - } - } - // add manifest as uncompressed data - MemoryStream().use { ms -> - manifest.serialize(ms.asOutputStream()) - - zipUncompressed(zip, getEntryName(manifest.depotID, manifest.manifestGID), ms.toByteArray()) - } - } - // save all data to the file - try { - Files.newOutputStream(file).use { fos -> - fos.write(bs.toByteArray()) - } - } catch (e: IOException) { - logger.error("Failed to write manifest to file ${file.fileName}", e) - } - } - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt deleted file mode 100644 index 87970b90..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt +++ /dev/null @@ -1,10 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import kotlinx.coroutines.sync.Semaphore -import java.nio.channels.FileChannel - -data class FileStreamData( - var fileStream: FileChannel?, - val fileLock: Semaphore, - var chunksToDownload: Int, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt deleted file mode 100644 index 76929969..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt +++ /dev/null @@ -1,6 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -data class GlobalDownloadCounter( - var totalBytesCompressed: Long = 0, - var totalBytesUncompressed: Long = 0, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt deleted file mode 100644 index 05373c88..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt +++ /dev/null @@ -1,36 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -/** - * An interface for persisting depot manifests for Steam content downloading - * - * @author Oxters - * @since 2024-11-06 - */ -interface IManifestProvider { - - /** - * Ask a provider to fetch a specific depot manifest - * @return A [Pair] object with a [DepotManifest] and its checksum if it exists otherwise null - */ - fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? - - /** - * Ask a provider to fetch the most recent manifest used of a depot - * @return A [Pair] object with a [DepotManifest] and its checksum if it exists otherwise null - */ - fun fetchLatestManifest(depotID: Int): DepotManifest? - - /** - * Ask a provider to set the most recent manifest ID used of a depot - */ - fun setLatestManifestId(depotID: Int, manifestID: Long) - - /** - * Update the persistent depot manifest - * @param manifest The depot manifest - * @return The checksum of the depot manifest - */ - fun updateManifest(manifest: DepotManifest) -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt deleted file mode 100644 index 8f680953..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt +++ /dev/null @@ -1,28 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -/** - * @author Oxters - * @since 2024-11-06 - */ -class MemoryManifestProvider : IManifestProvider { - - private val depotManifests = mutableMapOf>() - - private val latestManifests = mutableMapOf() - - override fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? = - depotManifests[depotID]?.get(manifestID) - - override fun fetchLatestManifest(depotID: Int): DepotManifest? = - latestManifests[depotID]?.let { fetchManifest(depotID, it) } - - override fun setLatestManifestId(depotID: Int, manifestID: Long) { - latestManifests[depotID] = manifestID - } - - override fun updateManifest(manifest: DepotManifest) { - depotManifests.getOrPut(manifest.depotID) { mutableMapOf() }[manifest.manifestGID] = manifest - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt deleted file mode 100644 index afe5adc3..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -/** - * Interface for Java to implement for progress updates - */ -fun interface ProgressCallback { - fun onProgress(progress: Float) -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt similarity index 74% rename from src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt rename to src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt index 807670f5..1d9068aa 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt @@ -1,14 +1,14 @@ -package `in`.dragonbra.javasteam.steam.cdn +package `in`.dragonbra.javasteam.steam.handlers.steamcontent import `in`.dragonbra.javasteam.enums.EResult -import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetCDNAuthToken_Response +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.callback.ServiceMethodResponse import java.util.Date /** * This is received when a CDN auth token is received */ -class AuthToken(message: ServiceMethodResponse) { +class CDNAuthToken(message: ServiceMethodResponse) { /** * Result of the operation diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt index c38bdd3c..5e66c4e6 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt @@ -5,9 +5,9 @@ import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystem import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetManifestRequestCode_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetServersForSteamPipe_Request import `in`.dragonbra.javasteam.rpc.service.ContentServerDirectory -import `in`.dragonbra.javasteam.steam.cdn.AuthToken import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.CDNAuthToken import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages import `in`.dragonbra.javasteam.steam.webapi.ContentServerDirectoryService import kotlinx.coroutines.CoroutineScope @@ -38,7 +38,7 @@ class SteamContent : ClientMsgHandler() { parentScope: CoroutineScope, ): Deferred> = parentScope.async { val request = CContentServerDirectory_GetServersForSteamPipe_Request.newBuilder().apply { - this.cellId = cellId ?: client.cellID?.toInt() ?: 0 + this.cellId = cellId ?: client.cellID ?: 0 maxNumServers?.let { this.maxServers = it } }.build() @@ -65,7 +65,7 @@ class SteamContent : ClientMsgHandler() { branch: String? = null, branchPasswordHash: String? = null, parentScope: CoroutineScope, - ): Deferred = parentScope.async { + ): Deferred = parentScope.async { var localBranch = branch var localBranchPasswordHash = branchPasswordHash @@ -89,24 +89,24 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getManifestRequestCode(request).await() val response = message.body.build() - return@async response.manifestRequestCode.toULong() + return@async response.manifestRequestCode } /** * Request product information for an app or package - * Results are returned in a [AuthToken]. + * Results are returned in a [CDNAuthToken]. * * @param app App id requested. * @param depot Depot id requested. * @param hostName CDN host name being requested. - * @return The [AuthToken] containing the result. + * @return The [CDNAuthToken] containing the result. */ fun getCDNAuthToken( app: Int, depot: Int, hostName: String, parentScope: CoroutineScope, - ): Deferred = parentScope.async { + ): Deferred = parentScope.async { val request = CContentServerDirectory_GetCDNAuthToken_Request.newBuilder().apply { this.appId = app this.depotId = depot @@ -115,7 +115,7 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getCDNAuthToken(request).await() - return@async AuthToken(message) + return@async CDNAuthToken(message) } /** diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt index 1872ff77..950ed9d9 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt @@ -109,6 +109,16 @@ class SteamClient @JvmOverloads constructor( addHandlerCore(handler) } + /** + * Kotlin Helper: + * Adds a new handler to the internal list of message handlers. + * @param T The handler to add. + */ + inline fun addHandler() { + val handler = T::class.java.getDeclaredConstructor().newInstance() + addHandler(handler) + } + private fun addHandlerCore(handler: ClientMsgHandler) { handler.setup(this) handlers[handler.javaClass] = handler @@ -122,6 +132,15 @@ class SteamClient @JvmOverloads constructor( handlers.remove(handler) } + /** + * Kotlin Helper: + * Removes a registered handler by name. + * @param T The handler name to remove. + */ + inline fun removeHandler() { + removeHandler(T::class.java) + } + /** * Removes a registered handler. * @param handler The handler name to remove. diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt index 0bb0a2af..9c91907a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import okhttp3.OkHttpClient import java.util.* @@ -96,14 +95,6 @@ interface ISteamConfigurationBuilder { */ fun withServerListProvider(provider: IServerListProvider): ISteamConfigurationBuilder - /** - * Configures the depot manifest provider for this [SteamConfiguration]. - * - * @param provider The depot manifest provider to use. - * @return A builder with modified configuration. - */ - fun withManifestProvider(provider: IManifestProvider): ISteamConfigurationBuilder - /** * Configures the Universe that this [SteamConfiguration] belongs to. * diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt index 13212928..44a7c41a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import `in`.dragonbra.javasteam.steam.discovery.SmartCMServerList import `in`.dragonbra.javasteam.steam.steamclient.SteamClient @@ -79,12 +78,6 @@ class SteamConfiguration internal constructor(private val state: SteamConfigurat val serverListProvider: IServerListProvider get() = state.serverListProvider - /** - * The depot manifest provider to use. - */ - val depotManifestProvider: IManifestProvider - get() = state.depotManifestProvider - /** * The Universe to connect to. This should always be [EUniverse.Public] unless you work at Valve and are using this internally. If this is you, hello there. */ diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt index 6ce754d2..c275477a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt @@ -4,8 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider -import `in`.dragonbra.javasteam.steam.contentdownloader.MemoryManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import `in`.dragonbra.javasteam.steam.discovery.MemoryServerListProvider import `in`.dragonbra.javasteam.steam.webapi.WebAPI @@ -72,11 +70,6 @@ class SteamConfigurationBuilder : ISteamConfigurationBuilder { return this } - override fun withManifestProvider(provider: IManifestProvider): ISteamConfigurationBuilder { - state.depotManifestProvider = provider - return this - } - override fun withUniverse(universe: EUniverse): ISteamConfigurationBuilder { state.universe = universe return this @@ -108,7 +101,6 @@ class SteamConfigurationBuilder : ISteamConfigurationBuilder { httpClient = OkHttpClient(), protocolTypes = EnumSet.of(ProtocolTypes.TCP, ProtocolTypes.WEB_SOCKET), serverListProvider = MemoryServerListProvider(), - depotManifestProvider = MemoryManifestProvider(), universe = EUniverse.Public, webAPIBaseAddress = WebAPI.DEFAULT_BASE_ADDRESS, cellID = 0, diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt index fbce399a..00bf1ff4 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import okhttp3.OkHttpClient import java.util.* @@ -22,7 +21,6 @@ data class SteamConfigurationState( var httpClient: OkHttpClient, var protocolTypes: EnumSet, var serverListProvider: IServerListProvider, - var depotManifestProvider: IManifestProvider, var universe: EUniverse, var webAPIBaseAddress: String, var webAPIKey: String?, diff --git a/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt b/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt index 54565d32..d30c5548 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt @@ -8,6 +8,7 @@ import `in`.dragonbra.javasteam.steam.cdn.Server */ object ContentServerDirectoryService { + @JvmStatic internal fun convertServerList( response: CContentServerDirectory_GetServersForSteamPipe_Response, ): List = response.serversList.map { child -> @@ -18,21 +19,21 @@ object ContentServerDirectoryService { Server.ConnectionProtocol.HTTP } - Server( - protocol = protocol, - host = child.host, - vHost = child.vhost, - port = if (protocol == Server.ConnectionProtocol.HTTPS) 443 else 80, - type = child.type, - sourceID = child.sourceId, - cellID = child.cellId, - load = child.load, - weightedLoad = child.weightedLoad, - numEntries = child.numEntriesInClientList, - steamChinaOnly = child.steamChinaOnly, - useAsProxy = child.useAsProxy, - proxyRequestPathTemplate = child.proxyRequestPathTemplate, - allowedAppIds = child.allowedAppIdsList.toIntArray() - ) + Server().apply { + this.protocol = protocol + this.host = child.host + this.vHost = child.vhost + this.port = if (protocol == Server.ConnectionProtocol.HTTPS) 443 else 80 + this.type = child.type + this.sourceId = child.sourceId + this.cellId = child.cellId + this.load = child.load + this.weightedLoad = child.weightedLoad + this.numEntries = child.numEntriesInClientList + this.steamChinaOnly = child.steamChinaOnly + this.useAsProxy = child.useAsProxy + this.proxyRequestPathTemplate = child.proxyRequestPathTemplate + this.allowedAppIds = child.allowedAppIdsList.toIntArray() + } } } diff --git a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt index ec30be2d..8069c9d6 100644 --- a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt @@ -23,7 +23,6 @@ import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec import kotlin.NoSuchElementException import kotlin.collections.ArrayList -import kotlin.collections.HashSet /** * Represents a Steam3 depot manifest. @@ -400,25 +399,23 @@ class DepotManifest { * @param output The stream to which the serialized depot manifest will be written. */ fun serialize(output: OutputStream) { - val payload = ContentManifestPayload.newBuilder() - val uniqueChunks = object : HashSet() { - // This acts like "ChunkIdComparer" - private val items = mutableListOf() - - override fun add(element: ByteArray): Boolean { - if (contains(element)) return false - items.add(element) - return true + // Basically like ChunkIdComparer from DepotDownloader + class ByteArrayKey(private val byteArray: ByteArray) { + override fun equals(other: Any?): Boolean = other is ByteArrayKey && byteArray.contentEquals(other.byteArray) + + override fun hashCode(): Int = if (byteArray.size >= 4) { + ((byteArray[0].toInt() and 0xFF)) or + ((byteArray[1].toInt() and 0xFF) shl 8) or + ((byteArray[2].toInt() and 0xFF) shl 16) or + ((byteArray[3].toInt() and 0xFF) shl 24) + } else { + byteArray.contentHashCode() } - - override fun contains(element: ByteArray): Boolean = items.any { it.contentEquals(element) } - - override fun iterator(): MutableIterator = items.iterator() - - override val size: Int - get() = items.size } + val payload = ContentManifestPayload.newBuilder() + val uniqueChunks = hashSetOf() + files.forEach { file -> val protofile = ContentManifestPayload.FileMapping.newBuilder().apply { this.size = file.totalSize @@ -457,7 +454,7 @@ class DepotManifest { }.build() protofile.addChunks(protochunk) - uniqueChunks.add(chunk.chunkID!!) + uniqueChunks.add(ByteArrayKey(chunk.chunkID!!)) } payload.addMappings(protofile.build()) diff --git a/src/main/java/in/dragonbra/javasteam/types/FileData.kt b/src/main/java/in/dragonbra/javasteam/types/FileData.kt index 53a7c2f4..41ee3ead 100644 --- a/src/main/java/in/dragonbra/javasteam/types/FileData.kt +++ b/src/main/java/in/dragonbra/javasteam/types/FileData.kt @@ -17,7 +17,6 @@ import java.util.EnumSet * @param fileHash Gets SHA-1 hash of this file. * @param linkTarget Gets symlink target of this file. */ -@Suppress("ArrayInDataClass") data class FileData( var fileName: String = "", var fileNameHash: ByteArray = byteArrayOf(), @@ -48,4 +47,32 @@ data class FileData( fileHash = hash, linkTarget = linkTarget, ) + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as FileData + + if (totalSize != other.totalSize) return false + if (fileName != other.fileName) return false + if (!fileNameHash.contentEquals(other.fileNameHash)) return false + if (chunks != other.chunks) return false + if (flags != other.flags) return false + if (!fileHash.contentEquals(other.fileHash)) return false + if (linkTarget != other.linkTarget) return false + + return true + } + + override fun hashCode(): Int { + var result = totalSize.hashCode() + result = 31 * result + fileName.hashCode() + result = 31 * result + fileNameHash.contentHashCode() + result = 31 * result + chunks.hashCode() + result = 31 * result + flags.hashCode() + result = 31 * result + fileHash.contentHashCode() + result = 31 * result + (linkTarget?.hashCode() ?: 0) + return result + } } diff --git a/src/main/java/in/dragonbra/javasteam/types/PubFile.kt b/src/main/java/in/dragonbra/javasteam/types/PubFile.kt new file mode 100644 index 00000000..fe50ce11 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/types/PubFile.kt @@ -0,0 +1,108 @@ +package `in`.dragonbra.javasteam.types + +/** + * The base class used for wrapping common ULong types, to introduce type safety and distinguish between common types. + */ +@Suppress("unused") +abstract class UInt64Handle : Any { + + /** + * Gets or sets the value. + */ + protected open var value: ULong = 0UL + + /** + * @constructor Initializes a new instance of the [UInt64Handle] class. + */ + protected constructor() + + /** + * Initializes a new instance of the [UInt64Handle] class. + * @param value The value to initialize this handle to. + */ + protected constructor(value: ULong) { + this.value = value + } + + /** + * Returns a hash code for this instance. + * @return A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table. + */ + override fun hashCode(): Int = value.hashCode() + + /** + * Determines whether the specified object is equal to this instance. + * @param other The object to compare with this instance. + * @return true if the specified object is equal to this instance; otherwise, false. + */ + override fun equals(other: Any?): Boolean { + if (other is UInt64Handle) { + return other.value == value + } + + return false + } + + /** + * Returns a string that represents this instance. + * @return A string that represents this instance. + */ + override fun toString(): String = value.toString() + + /** + * TODO + */ + fun toLong(): Long = value.toLong() + + /** + * Indicates whether the current object is equal to another object of the same type. + * @param other An object to compare with this object. + * @return true if the current object is equal to the other parameter; otherwise, false. + */ + fun equals(other: UInt64Handle?): Boolean { + if (other == null) { + return false + } + return value == other.value + } +} + +/** + * Represents a handle to a published file on the Steam workshop. + */ +class PublishedFileID : UInt64Handle { + + /** + * Initializes a new instance of the PublishedFileID class. + * @param fileId The file id. + */ + constructor(fileId: Long = Long.MAX_VALUE) : super(fileId.toULong()) + + companion object { + /** + * Implements the operator ==. + * @param a The first published file. + * @param b The second published file. + * @return The result of the operator. + */ + fun equals(a: PublishedFileID?, b: PublishedFileID?): Boolean { + if (a === b) { + return true + } + + if (a == null || b == null) { + return false + } + + return a.value == b.value + } + + /** + * Implements the operator !=. + * @param a The first published file. + * @param b The second published file. + * @return The result of the operator. + */ + fun notEquals(a: PublishedFileID?, b: PublishedFileID?): Boolean = !equals(a, b) + } +} diff --git a/src/main/java/in/dragonbra/javasteam/util/Utils.java b/src/main/java/in/dragonbra/javasteam/util/Utils.java index ecd8d3f1..f614c5c8 100644 --- a/src/main/java/in/dragonbra/javasteam/util/Utils.java +++ b/src/main/java/in/dragonbra/javasteam/util/Utils.java @@ -193,40 +193,4 @@ public static long crc32(byte[] bytes) { checksum.update(bytes, 0, bytes.length); return checksum.getValue(); } - - /** - * Validate a file against Steam3 Chunk data - * - * @param fs FileInputStream to read from - * @param chunkData Array of ChunkData to validate against - * @return List of ChunkData that are needed - * @throws IOException If there's an error reading the file - * @throws ClosedChannelException If this channel is closed - * @throws IllegalArgumentException If the new position is negative - */ - @SuppressWarnings("resource") - public static List validateSteam3FileChecksums(RandomAccessFile fs, ChunkData[] chunkData) throws IOException { - List neededChunks = new ArrayList<>(); - int read; - - for (ChunkData data : chunkData) { - byte[] chunk = new byte[data.getUncompressedLength()]; - fs.getChannel().position(data.getOffset()); - read = fs.read(chunk, 0, data.getUncompressedLength()); - - byte[] tempChunk; - if (read > 0 && read < data.getUncompressedLength()) { - tempChunk = Arrays.copyOf(chunk, read); - } else { - tempChunk = chunk; - } - - int adler = Adler32.calculate(tempChunk); - if (adler != data.getChecksum()) { - neededChunks.add(data); - } - } - - return neededChunks; - } } diff --git a/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java b/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java index 30e9bd95..2bf4ef4a 100644 --- a/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java +++ b/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java @@ -42,7 +42,7 @@ public void throwsSteamKitWebExceptionOnUnsuccessfulWebResponseForManifest() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); // JVM will throw ExecutionException Exception exception = Assertions.assertThrows(ExecutionException.class, () -> @@ -64,7 +64,7 @@ public void throwsSteamKitWebExceptionOnUnsuccessfulWebResponseForChunk() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0L, 0, 0); // JVM will throw ExecutionException @@ -87,7 +87,7 @@ public void throwsWhenNoChunkIDIsSet() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(); // JVM will throw ExecutionException @@ -111,7 +111,7 @@ public void throwsWhenDestinationBufferSmaller() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0, 32, 64); // JVM will throw ExecutionException @@ -135,7 +135,7 @@ public void throwsWhenDestinationBufferSmallerWithDepotKey() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0, 32, 64); // JVM will throw ExecutionException