network_glora.py 1.2 KB

123456789101112131415161718192021222324252627282930313233
  1. import network
  2. class ModuleTypeGLora(network.ModuleType):
  3. def create_module(self, net: network.Network, weights: network.NetworkWeights):
  4. if all(x in weights.w for x in ["a1.weight", "a2.weight", "alpha", "b1.weight", "b2.weight"]):
  5. return NetworkModuleGLora(net, weights)
  6. return None
  7. # adapted from https://github.com/KohakuBlueleaf/LyCORIS
  8. class NetworkModuleGLora(network.NetworkModule):
  9. def __init__(self, net: network.Network, weights: network.NetworkWeights):
  10. super().__init__(net, weights)
  11. if hasattr(self.sd_module, 'weight'):
  12. self.shape = self.sd_module.weight.shape
  13. self.w1a = weights.w["a1.weight"]
  14. self.w1b = weights.w["b1.weight"]
  15. self.w2a = weights.w["a2.weight"]
  16. self.w2b = weights.w["b2.weight"]
  17. def calc_updown(self, orig_weight):
  18. w1a = self.w1a.to(orig_weight.device)
  19. w1b = self.w1b.to(orig_weight.device)
  20. w2a = self.w2a.to(orig_weight.device)
  21. w2b = self.w2b.to(orig_weight.device)
  22. output_shape = [w1a.size(0), w1b.size(1)]
  23. updown = ((w2b @ w1b) + ((orig_weight.to(dtype = w1a.dtype) @ w2a) @ w1a))
  24. return self.finalize_updown(updown, orig_weight, output_shape)