fix stable diffusers textual_inversion finetune accurary issue. (#1533) (#1594)

the generated imaged by the finetune model unrelated with prompt

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
Co-authored-by: Wang, Yi <yi.a.wang@intel.com>

Author: jianan-gu

intel_extension_for_pytorch/nn/utils/_weight_cast.py

@@ -1,5 +1,6 @@
 import torch
 import torch.nn as nn
+import sys
 from intel_extension_for_pytorch.optim import _optimizer_utils, _lamb
 import types
 from ._model_convert import _LSTM
@@ -112,11 +113,16 @@ def convert(m):
                         for name, para in sub_m.named_parameters():
                             cast_attr(sub_m, name, master_weight_split, params_attr, optimizer)
         return m
-      
+
+    def isCLIPTextEmbeddings(m):
+        mod = 'transformers.models.clip.modeling_clip'
+        return mod in sys.modules and hasattr(sys.modules[mod], 'CLIPTextEmbeddings') and isinstance(m, sys.modules[mod].CLIPTextEmbeddings)
+
     def convert_rec(m):
         new_m = convert(m)
         for name, sub_m in m.named_children():
-            setattr(new_m, name, convert_rec(sub_m))
+            if not isCLIPTextEmbeddings(sub_m):
+                setattr(new_m, name, convert_rec(sub_m))
         return new_m
 
     casted_model, casted_optimizer, params_attr = convert_rec(module), optimizer, params_attr