add Intel XPU platform support for benchmark_v2 #42993
Conversation
Signed-off-by: Liu, Kaixuan <[email protected]>
|
cc @IlyasMoutawwakil maybe? |
Signed-off-by: Liu, Kaixuan <[email protected]>
Signed-off-by: Liu, Kaixuan <[email protected]>
Signed-off-by: Liu, Kaixuan <[email protected]>
Signed-off-by: Liu, Kaixuan <[email protected]>
|
@IlyasMoutawwakil Hi, pls help review when you are available, thx! |
Signed-off-by: Liu, Kaixuan <[email protected]>
| type=str, | ||
| default="cuda", | ||
| help="Device to run benchmarks on (cuda, xpu, cpu). If not specified, will auto-detect.", | ||
| ) |
There was a problem hiding this comment.
inconsistent, the docstring says if not specified, will auto detect, but the default is set to cuda rather than auto
| self.logger = logger if logger is not None else logging.getLogger(__name__) | ||
| self.device_type = device_type | ||
|
|
||
| # Detect available accelerators |
There was a problem hiding this comment.
you mean "detect the number of available accelerators"?
| device_type = None | ||
| if hasattr(torch, "cuda") and torch.cuda.is_available(): | ||
| device_type = "cuda" | ||
| elif hasattr(torch, "xpu") and torch.xpu.is_available(): |
There was a problem hiding this comment.
use is_torch_xpu_available from transformers
|
|
||
| def __init__(self) -> None: | ||
| self.device_name, self.device_memory_total = get_device_name_and_memory_total() | ||
| def __init__(self, device_type: str = "cuda") -> None: |
| self.gpu_name, self.gpu_memory_total_gb = get_device_name_and_memory_total() | ||
| # Auto-detect device type | ||
| device_type = None | ||
| if hasattr(torch, "cuda") and torch.cuda.is_available(): |
There was a problem hiding this comment.
suppose torch.cuda.is_available() is enough
| Args: | ||
| device_type: The type of device to query ('cuda', 'xpu', etc.) | ||
| """ |
There was a problem hiding this comment.
seems device_type is unnecessary, just detect the available device and use torch_accelerator_module like:
`device_type = torch.accelerator.current_accelerator().type if is_torch_accelerator_available() else "cuda"
torch_accelerator_module = getattr(torch, device_type, torch.cuda)
torch_accelerator_module .get_device_properties(0).name
...
`
| f"Time generate done in {e2e_latency:.2f} seconds. Memory usage: {torch.cuda.memory_allocated() / 1024**2:.2f} MB" | ||
| ) | ||
|
|
||
| # Get memory usage based on device type |
There was a problem hiding this comment.
just use device_type to get torch_accelerator_module?
| """Profile the latency of a call to model.generate() with the given (inputs) and (max_new_tokens).""" | ||
| # Build activities list based on available devices | ||
| activities = [torch.profiler.ProfilerActivity.CPU] | ||
| if hasattr(torch, "xpu") and torch.xpu.is_available(): |
There was a problem hiding this comment.
use is_torch_xpu_available from transformers
|
Close this PR first. |
3 participants
No description provided.