lowvram.py 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147
  1. import torch
  2. from modules import devices, shared
  3. module_in_gpu = None
  4. cpu = torch.device("cpu")
  5. def send_everything_to_cpu():
  6. global module_in_gpu
  7. if module_in_gpu is not None:
  8. module_in_gpu.to(cpu)
  9. module_in_gpu = None
  10. def is_needed(sd_model):
  11. return shared.cmd_opts.lowvram or shared.cmd_opts.medvram or shared.cmd_opts.medvram_sdxl and hasattr(sd_model, 'conditioner')
  12. def apply(sd_model):
  13. enable = is_needed(sd_model)
  14. shared.parallel_processing_allowed = not enable
  15. if enable:
  16. setup_for_low_vram(sd_model, not shared.cmd_opts.lowvram)
  17. else:
  18. sd_model.lowvram = False
  19. def setup_for_low_vram(sd_model, use_medvram):
  20. if getattr(sd_model, 'lowvram', False):
  21. return
  22. sd_model.lowvram = True
  23. parents = {}
  24. def send_me_to_gpu(module, _):
  25. """send this module to GPU; send whatever tracked module was previous in GPU to CPU;
  26. we add this as forward_pre_hook to a lot of modules and this way all but one of them will
  27. be in CPU
  28. """
  29. global module_in_gpu
  30. module = parents.get(module, module)
  31. if module_in_gpu == module:
  32. return
  33. if module_in_gpu is not None:
  34. module_in_gpu.to(cpu)
  35. module.to(devices.device)
  36. module_in_gpu = module
  37. # see below for register_forward_pre_hook;
  38. # first_stage_model does not use forward(), it uses encode/decode, so register_forward_pre_hook is
  39. # useless here, and we just replace those methods
  40. first_stage_model = sd_model.first_stage_model
  41. first_stage_model_encode = sd_model.first_stage_model.encode
  42. first_stage_model_decode = sd_model.first_stage_model.decode
  43. def first_stage_model_encode_wrap(x):
  44. send_me_to_gpu(first_stage_model, None)
  45. return first_stage_model_encode(x)
  46. def first_stage_model_decode_wrap(z):
  47. send_me_to_gpu(first_stage_model, None)
  48. return first_stage_model_decode(z)
  49. to_remain_in_cpu = [
  50. (sd_model, 'first_stage_model'),
  51. (sd_model, 'depth_model'),
  52. (sd_model, 'embedder'),
  53. (sd_model, 'model'),
  54. (sd_model, 'embedder'),
  55. ]
  56. is_sdxl = hasattr(sd_model, 'conditioner')
  57. is_sd2 = not is_sdxl and hasattr(sd_model.cond_stage_model, 'model')
  58. if is_sdxl:
  59. to_remain_in_cpu.append((sd_model, 'conditioner'))
  60. elif is_sd2:
  61. to_remain_in_cpu.append((sd_model.cond_stage_model, 'model'))
  62. else:
  63. to_remain_in_cpu.append((sd_model.cond_stage_model, 'transformer'))
  64. # remove several big modules: cond, first_stage, depth/embedder (if applicable), and unet from the model
  65. stored = []
  66. for obj, field in to_remain_in_cpu:
  67. module = getattr(obj, field, None)
  68. stored.append(module)
  69. setattr(obj, field, None)
  70. # send the model to GPU.
  71. sd_model.to(devices.device)
  72. # put modules back. the modules will be in CPU.
  73. for (obj, field), module in zip(to_remain_in_cpu, stored):
  74. setattr(obj, field, module)
  75. # register hooks for those the first three models
  76. if is_sdxl:
  77. sd_model.conditioner.register_forward_pre_hook(send_me_to_gpu)
  78. elif is_sd2:
  79. sd_model.cond_stage_model.model.register_forward_pre_hook(send_me_to_gpu)
  80. sd_model.cond_stage_model.model.token_embedding.register_forward_pre_hook(send_me_to_gpu)
  81. parents[sd_model.cond_stage_model.model] = sd_model.cond_stage_model
  82. parents[sd_model.cond_stage_model.model.token_embedding] = sd_model.cond_stage_model
  83. else:
  84. sd_model.cond_stage_model.transformer.register_forward_pre_hook(send_me_to_gpu)
  85. parents[sd_model.cond_stage_model.transformer] = sd_model.cond_stage_model
  86. sd_model.first_stage_model.register_forward_pre_hook(send_me_to_gpu)
  87. sd_model.first_stage_model.encode = first_stage_model_encode_wrap
  88. sd_model.first_stage_model.decode = first_stage_model_decode_wrap
  89. if sd_model.depth_model:
  90. sd_model.depth_model.register_forward_pre_hook(send_me_to_gpu)
  91. if sd_model.embedder:
  92. sd_model.embedder.register_forward_pre_hook(send_me_to_gpu)
  93. if use_medvram:
  94. sd_model.model.register_forward_pre_hook(send_me_to_gpu)
  95. else:
  96. diff_model = sd_model.model.diffusion_model
  97. # the third remaining model is still too big for 4 GB, so we also do the same for its submodules
  98. # so that only one of them is in GPU at a time
  99. stored = diff_model.input_blocks, diff_model.middle_block, diff_model.output_blocks, diff_model.time_embed
  100. diff_model.input_blocks, diff_model.middle_block, diff_model.output_blocks, diff_model.time_embed = None, None, None, None
  101. sd_model.model.to(devices.device)
  102. diff_model.input_blocks, diff_model.middle_block, diff_model.output_blocks, diff_model.time_embed = stored
  103. # install hooks for bits of third model
  104. diff_model.time_embed.register_forward_pre_hook(send_me_to_gpu)
  105. for block in diff_model.input_blocks:
  106. block.register_forward_pre_hook(send_me_to_gpu)
  107. diff_model.middle_block.register_forward_pre_hook(send_me_to_gpu)
  108. for block in diff_model.output_blocks:
  109. block.register_forward_pre_hook(send_me_to_gpu)
  110. def is_enabled(sd_model):
  111. return sd_model.lowvram