Added a potential solution for windows

Author: cehongwang

py/torch_tensorrt/dynamo/conversion/_conversion.py

@@ -34,7 +34,7 @@ def infer_module_output_dtypes(
     """
     outputs = [node for node in module.graph.nodes if node.op == "output"]
     outputs = outputs[0].args
-    return get_output_dtypes(outputs, truncate_double)  # type: ignore[no-any-return]
+    return get_output_dtypes(outputs, truncate_double)
 
 
 def interpret_module_to_result(
@@ -70,6 +70,29 @@ def interpret_module_to_result(
     )
 
     interpreter_result = interpreter.run()
+    # Delete the frozen parameters from the module to release CPU memory
+    del interpreter
+    for attr in dir(module):
+        if attr.startswith("_frozen_param"):
+            delattr(module, attr)
+    release_memory()
+    logger.debug(
+        f"CPU memory usage after clearing frozen parameters and building memory in conversion: {get_cpu_memory_usage()} MB"
+    )
+
+    serialized_engine = interpreter_result.engine.serialize()
+    with io.BytesIO() as engine_bytes:
+        engine_bytes.write(serialized_engine)
+        serialized_engine = engine_bytes.getvalue()
+
+    interpreter_result = TRTInterpreterResult(
+        engine=serialized_engine,
+        input_names=interpreter_result.input_names,
+        output_names=interpreter_result.output_names,
+        weight_name_map=interpreter_result.weight_name_map,
+        requires_output_allocator=interpreter_result.requires_output_allocator,
+    )
+
     return interpreter_result
 
 
@@ -108,22 +131,8 @@ def convert_module(
             "Since Torch-TensorRT runtime is not available, using Python Runtime, some features may not be available"
         )
 
-    # Delete the frozen parameters from the module to release CPU memory
-    for attr in dir(module):
-        if attr.startswith("_frozen_param"):
-            delattr(module, attr)
-    release_memory()
-    logger.debug(
-        f"CPU memory usage after clearing frozen parameters and building memory in conversion: {get_cpu_memory_usage()} MB"
-    )
-
-    serialized_engine = interpreter_result.engine.serialize()
-    with io.BytesIO() as engine_bytes:
-        engine_bytes.write(serialized_engine)
-        serialized_engine = engine_bytes.getvalue()
-    breakpoint()
     return rt_cls(
-        serialized_engine=serialized_engine,
+        serialized_engine=interpreter_result.engine,
         input_binding_names=list(interpreter_result.input_names),
         output_binding_names=list(interpreter_result.output_names),
         name=name,

py/torch_tensorrt/dynamo/utils.py

@@ -868,6 +868,7 @@ def get_cpu_memory_usage() -> Any:
 
 
 def release_memory() -> None:
+    gc.collect()
     if torch.cuda.is_available():
         torch.cuda.synchronize()
         torch.cuda.empty_cache()
@@ -881,3 +882,32 @@ def release_memory() -> None:
                 logger.warning("Failed to release CPU memory.")
         except Exception:
             logger.warning("Failed to release CPU memory.")
+
+    elif platform.system() == "Windows":
+        from ctypes import wintypes
+
+        kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+        psapi = ctypes.WinDLL("psapi", use_last_error=True)
+
+        GetCurrentProcess = kernel32.GetCurrentProcess
+        GetCurrentProcess.restype = wintypes.HANDLE
+        hproc = GetCurrentProcess()
+
+        HeapSetInformation = kernel32.HeapSetInformation
+        HeapSetInformation.argtypes = [
+            wintypes.HANDLE,
+            ctypes.c_int,
+            ctypes.c_void_p,
+            ctypes.c_size_t,
+        ]
+        HeapSetInformation.restype = wintypes.BOOL
+        GetProcessHeap = kernel32.GetProcessHeap
+        GetProcessHeap.restype = wintypes.HANDLE
+        ok = False
+        try:
+            HeapOptimizeResources = 3
+            hheap = GetProcessHeap()
+            if HeapSetInformation(hheap, HeapOptimizeResources, None, 0):
+                ok = True
+        except Exception:
+            logger.warning("Failed to release CPU memory.")