Add Xet support for faster downloads

Author: mattt

Package.resolved

@@ -0,0 +1,51 @@
+// swift-tools-version: 6.1
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "swift-huggingface",
+    platforms: [
+        .macOS(.v14),
+        .macCatalyst(.v14),
+        .iOS(.v16),
+        .watchOS(.v10),
+        .tvOS(.v17),
+        .visionOS(.v1),
+    ],
+    products: [
+        .library(
+            name: "HuggingFace",
+            targets: ["HuggingFace"]
+        )
+    ],
+    dependencies: [
+        .package(url: "https://github.com/mattt/EventSource.git", from: "1.0.0"),
+        .package(url: "https://github.com/mattt/swift-xet.git", from: "0.1.0")
+    ],
+    traits: [
+        .default(
+            enabledTraits: ["XetSupport"]
+        ),
+        .init(
+            name: "XetSupport",
+            description: "Enable Xet for faster downloads",
+            enabledTraits: []
+        )
+    ],
+    targets: [
+        .target(
+            name: "HuggingFace",
+            dependencies: [
+                .product(name: "EventSource", package: "EventSource"),
+                .product(name: "Xet", package: "swift-xet", condition: .when(traits: ["XetSupport"]))
+            ],
+            path: "Sources/HuggingFace"
+        ),
+        .testTarget(
+            name: "HuggingFaceTests",
+            dependencies: ["HuggingFace"]
+        ),
+    ]
+)
+

Package@swift-6.1.swift

@@ -1000,3 +1000,8 @@ let response = try await client.speechToText(
 
 print("Transcription: \(response.text)")
 ```
+
+## License
+
+This project is available under the MIT license.
+See the LICENSE file for more info.

README.md

@@ -6,6 +6,10 @@ import UniformTypeIdentifiers
     import FoundationNetworking
 #endif
 
+#if canImport(Xet)
+    import Xet
+#endif
+
 // MARK: - Upload Operations
 
 public extension HubClient {
@@ -183,6 +187,22 @@ public extension HubClient {
         useRaw: Bool = false,
         cachePolicy: URLRequest.CachePolicy = .useProtocolCachePolicy
     ) async throws -> Data {
+        #if canImport(Xet)
+            // Try Xet first for faster downloads
+            do {
+                let xetClient = try await createXetClient()
+                let content = try xetClient.getFileContent(
+                    repo: repo.rawValue,
+                    path: repoPath,
+                    revision: revision
+                )
+                return content
+            } catch {
+                // Fallback to LFS if Xet fails
+            }
+        #endif
+
+        // Fallback to existing LFS download method
         let endpoint = useRaw ? "raw" : "resolve"
         let urlPath = "/\(repo)/\(endpoint)/\(revision)/\(repoPath)"
         var request = try httpClient.createRequest(.get, urlPath)
@@ -215,6 +235,28 @@ public extension HubClient {
         cachePolicy: URLRequest.CachePolicy = .useProtocolCachePolicy,
         progress: Progress? = nil
     ) async throws -> URL {
+        #if canImport(Xet)
+            // Try Xet first for faster downloads
+            do {
+                let xetClient = try await createXetClient()
+                try xetClient.downloadFile(
+                    repo: repo.rawValue,
+                    path: repoPath,
+                    destination: destination.path,
+                    revision: revision
+                )
+
+                // Update progress if provided
+                progress?.totalUnitCount = 100
+                progress?.completedUnitCount = 100
+
+                return destination
+            } catch {
+                // Fallback to LFS if Xet fails
+            }
+        #endif
+
+        // Fallback to existing LFS download method
         let endpoint = useRaw ? "raw" : "resolve"
         let urlPath = "/\(repo)/\(endpoint)/\(revision)/\(repoPath)"
         var request = try httpClient.createRequest(.get, urlPath)
@@ -513,6 +555,36 @@ public extension HubClient {
                 continue
             }
 
+            #if canImport(Xet)
+                // Try Xet first if available
+                do {
+                    let xetClient = try await createXetClient()
+                    try xetClient.downloadFile(
+                        repo: repo.rawValue,
+                        path: filename,
+                        destination: fileDestination.path,
+                        revision: revision
+                    )
+                    fileProgress.completedUnitCount = 100
+
+                    if let etag = remoteFile.etag, let revision = remoteFile.revision {
+                        try writeDownloadMetadata(
+                            commitHash: revision,
+                            etag: etag,
+                            to: metadataDestination
+                        )
+                    }
+
+                    if Task.isCancelled {
+                        return repoDestination
+                    }
+
+                    continue
+                } catch {
+                    // Fallback to LFS if Xet fails
+                }
+            #endif
+
             _ = try await downloadFile(
                 at: filename,
                 from: repo,

Sources/HuggingFace/Hub/HubClient+Files.swift

@@ -4,6 +4,10 @@ import Foundation
     import FoundationNetworking
 #endif
 
+#if canImport(Xet)
+import Xet
+#endif
+
 /// A Hugging Face Hub API client.
 ///
 /// This client provides methods to interact with the Hugging Face Hub API,
@@ -134,4 +138,21 @@ public final class HubClient: Sendable {
         }
         return defaultHost
     }
+
+    // MARK: - Xet Client
+
+    #if canImport(Xet)
+    /// Creates a Xet client for faster downloads.
+    ///
+    /// The client is initialized with the bearer token if available.
+    ///
+    /// - Returns: A Xet client instance.
+    internal func createXetClient() async throws -> XetClient {
+        if let token = try? await httpClient.tokenProvider.getToken() {
+            return try XetClient.withToken(token: token)
+        } else {
+            return try XetClient()
+        }
+    }
+    #endif
 }