renderer.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148
  1. from math import ceil
  2. class Renderer:
  3. def __init__(
  4. self,
  5. progress_bar_width=50,
  6. columns=1,
  7. use_space_lines=True,
  8. node_names=None,
  9. display_power=True,
  10. display_users=False,
  11. ) -> None:
  12. self.progress_bar_width = progress_bar_width
  13. self.columns = columns
  14. self.use_space_lines = use_space_lines
  15. self.node_names = node_names
  16. self.display_power = display_power
  17. self.display_users = display_users
  18. def render_info_dict(self, info_dict):
  19. line_blocks = []
  20. for node_dict in info_dict:
  21. if self.node_names is None or node_dict["name"] in self.node_names:
  22. lines = self.render_node(node_dict)
  23. line_blocks.append(lines)
  24. first_line = "|" + "=" * (len(lines[-1]) - 2) + "|"
  25. final_lines = []
  26. n_rows = ceil(len(line_blocks) / self.columns)
  27. # Format rows and columns
  28. for row in range(n_rows):
  29. lines = []
  30. max_size = -1
  31. # Find max size of the blocks in the same row:
  32. for col in range(self.columns):
  33. if col * n_rows + row < len(line_blocks):
  34. if len(line_blocks[col * n_rows + row]) > max_size:
  35. max_size = len(line_blocks[col * n_rows + row])
  36. for col in range(self.columns):
  37. if col * n_rows + row < len(line_blocks):
  38. if len(lines) == 0:
  39. lines.extend(
  40. self.expand_rendered_block_to_size(
  41. line_blocks[col * n_rows + row],
  42. max_size
  43. )
  44. )
  45. else:
  46. for i, line in enumerate(self.expand_rendered_block_to_size(line_blocks[col * n_rows + row], max_size)):
  47. lines[i] += line
  48. final_lines.extend(lines)
  49. final_lines.insert(0, first_line * self.columns)
  50. #lines.append("=" * len(lines[-1]))
  51. return "\n".join(final_lines)
  52. def render_node(self, node_dict):
  53. name = node_dict["name"]
  54. mem_used = node_dict["latest_info"]["used_memory_mb"]
  55. mem_total = node_dict["total_memory_mb"]
  56. utilization = node_dict["latest_info"]["cpu_utilization"]
  57. temp = node_dict["latest_info"]["temperature"]
  58. head_line = "|- Node: " + name + " "
  59. info_line = f"| CPU: {utilization:>4.1f}% Memory: {mem_used:>6}/{mem_total:<6} MB Temp: {temp:>3}°C"
  60. lines = []
  61. for i, gpu_dict in enumerate(node_dict["gpus"]):
  62. lines.extend(self.get_rendered_gpu_lines(gpu_dict))
  63. head_line = head_line + "-" * (len(lines[-1]) - len(head_line) - 1) + "|"
  64. info_line = info_line + " " * (len(lines[-1]) - len(info_line) - 1) + "|"
  65. pad_line = "|" + "-" * (len(lines[-1]) - 2) + "|"
  66. pad_line_empty = "|" + " " * (len(lines[-1]) - 2) + "|"
  67. lines.append(pad_line_empty)
  68. lines.append("|" + "=" * (len(lines[-1]) - 2) + "|")
  69. lines.insert(0, pad_line)
  70. if self.use_space_lines:
  71. lines.insert(0, pad_line_empty)
  72. lines.insert(0, info_line)
  73. if self.use_space_lines:
  74. lines.insert(0, pad_line_empty)
  75. lines.insert(0, head_line)
  76. return lines
  77. def get_rendered_gpu_lines(self, gpu_dict):
  78. gpu_type = gpu_dict["type"]
  79. index = gpu_dict["index"]
  80. mem_used = gpu_dict["latest_info"]["used_memory_mb"]
  81. mem_total = gpu_dict["total_memory_mb"]
  82. utilization = gpu_dict["latest_info"]["utilization"]
  83. temp = gpu_dict["latest_info"]["temperature"]
  84. n_processes = len(gpu_dict["running_processes"])
  85. power = gpu_dict["latest_info"]["power_draw"]
  86. mem_used_percent = int(self.progress_bar_width * mem_used / mem_total)
  87. rest_mem = self.progress_bar_width - mem_used_percent
  88. line_util = "| [" + \
  89. "=" * mem_used_percent + \
  90. " " * rest_mem + "]" + \
  91. f"{mem_used:>6}/{mem_total:<6} MB, Util: {int(utilization):>3}% |"
  92. line_meta = f"| GPU #{index} ({gpu_type}): #Proc.: {n_processes} Temp: {temp:>3}°C " + (f"Pow: {int(power):>3} W" if self.display_power else "")
  93. line_meta = line_meta + " " * (len(line_util) - len(line_meta) - 1) + "|"
  94. empty_line = "|" + " " * (len(line_meta) - 2) + "|"
  95. lines = [line_meta, line_util]
  96. if self.display_users:
  97. lines.append(self.get_rendered_users_line(gpu_dict["running_processes"], len(line_meta)))
  98. if self.use_space_lines:
  99. lines = [empty_line] + lines
  100. return lines
  101. def get_rendered_users_line(self, processes_list, line_len):
  102. user_names = [p_dict["user_name"] for p_dict in processes_list]
  103. line = "| Users: " + ", ".join(user_names)
  104. line += " " * (line_len - len(line) - 1) + "|"
  105. return line
  106. def expand_rendered_block_to_size(self, block_lines, max_size):
  107. buffer_line = "|" + " " * (len(block_lines[-1]) - 2) + "|"
  108. block_lines_new = block_lines[:-1] + [buffer_line] * (max_size - len(block_lines)) + block_lines[-1:]
  109. return block_lines_new