[platform]: add scripts and adapt for ascend, mlu, metal

.gitignore

@@ -23,6 +23,8 @@
 *.ipynb*
 build/
 dist/
+!configs/platforms/ascend_npu/dist/
+!configs/platforms/ascend_npu/dist/*.json
 .cache/
 server_cache/
 app/.gradio/

configs/platforms/ascend_npu/dist/flux2_dev.json

@@ -0,0 +1,27 @@
+{
+    "model_cls": "flux2_dev",
+    "task": "t2i",
+    "infer_steps": 50,
+    "aspect_ratio": "16:9",
+    "sample_guide_scale": 4.0,
+    "vae_scale_factor": 16,
+    "feature_caching": "None",
+    "enable_cfg": false,
+    "patch_size": 2,
+    "tokenizer_max_length": 512,
+    "text_encoder_out_layers": [
+        10,
+        20,
+        30
+    ],
+    "attn_type": "npu_flash_attn",
+    "cpu_offload": false,
+    "offload_granularity": "block",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "parallel": {
+        "tensor_p_size": 8
+    }
+}

configs/platforms/ascend_npu/dist/hunyuan_video_t2v_480p.json

@@ -0,0 +1,25 @@
+{
+    "infer_steps": 50,
+    "transformer_model_name": "480p_i2v",
+    "fps": 24,
+    "target_video_length": 121,
+    "vae_stride": [
+        4,
+        16,
+        16
+    ],
+    "sample_shift": 5.0,
+    "sample_guide_scale": 6.0,
+    "aspect_ratio": "16:9",
+    "enable_cfg": true,
+    "attn_type": "npu_flash_attn",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "parallel": {
+        "seq_p_size": 4,
+        "seq_p_attn_type": "ulysses",
+        "cfg_p_size": 2
+    }
+}

configs/platforms/ascend_npu/dist/longcat_image_t2i.json

@@ -0,0 +1,24 @@
+{
+    "infer_steps": 50,
+    "aspect_ratio": "16:9",
+    "attn_type": "npu_flash_attn",
+    "enable_cfg": true,
+    "sample_guide_scale": 4.0,
+    "enable_cfg_renorm": true,
+    "cfg_renorm_min": 0.0,
+    "axes_dims_rope": [
+        16,
+        56,
+        56
+    ],
+    "dit_quant_scheme": "Default",
+    "rms_norm_type": "torch",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "parallel": {
+        "seq_p_size": 4,
+        "seq_p_attn_type": "ulysses",
+        "cfg_p_size": 2
+    }
+}

configs/platforms/ascend_npu/dist/ltx2_3.json

@@ -0,0 +1,59 @@
+{
+    "infer_steps": 30,
+    "target_video_length": 121,
+    "target_height": 512,
+    "target_width": 768,
+    "attn_type": "npu_flash_attn",
+    "sample_guide_scale": 3.0,
+    "sample_shift": [
+        2.05,
+        0.95
+    ],
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "gemma_cpu_offload": true,
+    "vae_cpu_offload": false,
+    "audio_mel_cpu_offload": true,
+    "offload_granularity": "model",
+    "norm_modulate_backend": "torch",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "num_channels_latents": 128,
+    "fps": 24,
+    "audio_fps": 24000,
+    "audio_mel_bins": 16,
+    "double_precision_rope": false,
+    "use_tiling_vae": false,
+    "dit_original_ckpt": "/data/wushuo1/models/LTX-2.3/ltx-2.3-22b-dev.safetensors",
+    "caption_proj_before_connector": true,
+    "cross_attention_adaln": true,
+    "apply_gated_attention": true,
+    "mm_guider": {
+        "enabled": true,
+        "video": {
+            "cfg_scale": 3.0,
+            "stg_scale": 1.0,
+            "stg_blocks": [
+                28
+            ],
+            "modality_scale": 3.0,
+            "rescale_scale": 0.7,
+            "skip_step": 0
+        },
+        "audio": {
+            "cfg_scale": 7.0,
+            "stg_scale": 1.0,
+            "stg_blocks": [
+                28
+            ],
+            "modality_scale": 3.0,
+            "rescale_scale": 0.7,
+            "skip_step": 0
+        }
+    },
+    "parallel": {
+        "tensor_p_size": 8
+    }
+}

configs/platforms/ascend_npu/dist/qwen_image_t2i_2512.json

@@ -0,0 +1,21 @@
+{
+    "infer_steps": 50,
+    "aspect_ratio": "16:9",
+    "prompt_template_encode": "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
+    "prompt_template_encode_start_idx": 34,
+    "attn_type": "npu_flash_attn",
+    "enable_cfg": true,
+    "sample_guide_scale": 4.0,
+    "cpu_offload": false,
+    "offload_granularity": "model",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "vae_decode_parallel": true,
+    "parallel": {
+        "seq_p_size": 4,
+        "seq_p_attn_type": "ulysses",
+        "cfg_p_size": 2
+    }
+}

configs/platforms/ascend_npu/dist/wan_moe_t2v.json

@@ -0,0 +1,28 @@
+{
+    "infer_steps": 40,
+    "target_video_length": 81,
+    "text_len": 512,
+    "target_height": 480,
+    "target_width": 832,
+    "self_attn_1_type": "npu_flash_attn",
+    "cross_attn_1_type": "npu_flash_attn",
+    "cross_attn_2_type": "npu_flash_attn",
+    "sample_guide_scale": [
+        4.0,
+        3.0
+    ],
+    "sample_shift": 12.0,
+    "enable_cfg": true,
+    "cpu_offload": true,
+    "offload_granularity": "model",
+    "boundary": 0.875,
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "parallel": {
+        "seq_p_size": 4,
+        "seq_p_attn_type": "ulysses",
+        "cfg_p_size": 2
+    }
+}

configs/platforms/ascend_npu/dist/wan_t2v.json

@@ -0,0 +1,24 @@
+{
+    "infer_steps": 50,
+    "target_video_length": 81,
+    "text_len": 512,
+    "target_height": 480,
+    "target_width": 832,
+    "self_attn_1_type": "npu_flash_attn",
+    "cross_attn_1_type": "npu_flash_attn",
+    "cross_attn_2_type": "npu_flash_attn",
+    "sample_guide_scale": 6,
+    "sample_shift": 8,
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "offload_granularity": "model",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "parallel": {
+        "seq_p_size": 4,
+        "seq_p_attn_type": "ulysses",
+        "cfg_p_size": 2
+    }
+}

configs/platforms/ascend_npu/dist/wan_t2v_sf.json

@@ -0,0 +1,37 @@
+{
+    "infer_steps": 4,
+    "target_video_length": 81,
+    "text_len": 512,
+    "target_height": 480,
+    "target_width": 832,
+    "self_attn_1_type": "npu_flash_attn",
+    "cross_attn_1_type": "npu_flash_attn",
+    "cross_attn_2_type": "npu_flash_attn",
+    "sample_guide_scale": 1,
+    "sample_shift": 5.0,
+    "enable_cfg": false,
+    "cpu_offload": false,
+    "dit_original_ckpt": "/data/wushuo1/models/Self-Forcing/checkpoints/self_forcing_dmd.pt",
+    "causal_rope_type": "torch",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "ar_config": {
+        "local_attn_size": -1,
+        "num_frame_per_chunk": 3,
+        "timesteps_index": [
+            0,
+            179,
+            358,
+            679
+        ],
+        "kv_offload": false,
+        "async_vae_decode": false
+    },
+    "parallel": {
+        "seq_p_size": 4,
+        "cfg_p_size": 2,
+        "seq_p_attn_type": "ulysses"
+    }
+}

configs/platforms/ascend_npu/dist/z_image_turbo_t2i.json

@@ -0,0 +1,19 @@
+{
+    "aspect_ratio": "16:9",
+    "num_channels_latents": 16,
+    "infer_steps": 9,
+    "attn_type": "npu_flash_attn",
+    "enable_cfg": false,
+    "sample_guide_scale": 0.0,
+    "patch_size": 2,
+    "cpu_offload": false,
+    "offload_granularity": "model",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "parallel": {
+        "seq_p_size": 2,
+        "seq_p_attn_type": "ulysses"
+    }
+}

configs/platforms/ascend_npu/single/flux2_dev.json

@@ -0,0 +1,20 @@
+{
+    "model_cls": "flux2_dev",
+    "task": "t2i",
+    "infer_steps": 50,
+    "aspect_ratio": "16:9",
+    "sample_guide_scale": 4.0,
+    "vae_scale_factor": 16,
+    "feature_caching": "None",
+    "enable_cfg": false,
+    "patch_size": 2,
+    "tokenizer_max_length": 512,
+    "text_encoder_out_layers": [10, 20, 30],
+    "attn_type": "npu_flash_attn",
+    "cpu_offload": true,
+    "offload_granularity": "block",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch"
+}

configs/platforms/ascend_npu/single/hunyuan_video_t2v_480p.json

@@ -0,0 +1,16 @@
+{
+    "infer_steps": 50,
+    "transformer_model_name": "480p_i2v",
+    "fps": 24,
+    "target_video_length": 121,
+    "vae_stride": [4, 16, 16],
+    "sample_shift": 5.0,
+    "sample_guide_scale": 6.0,
+    "aspect_ratio": "16:9",
+    "enable_cfg": true,
+    "attn_type": "npu_flash_attn",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch"
+}

configs/platforms/ascend_npu/single/ltx2_3.json

@@ -0,0 +1,46 @@
+{
+    "infer_steps": 30,
+    "target_video_length": 121,
+    "target_height": 512,
+    "target_width": 768,
+    "attn_type": "npu_flash_attn",
+    "sample_guide_scale": 3.0,
+    "sample_shift": [2.05, 0.95],
+    "enable_cfg": true,
+    "cpu_offload": true,
+    "offload_granularity": "model",
+    "norm_modulate_backend": "torch",
+    "modulate_type": "torch",
+    "rope_type": "torch",
+    "layer_norm_type": "torch",
+    "rms_norm_type": "torch",
+    "num_channels_latents": 128,
+    "fps": 24,
+    "audio_fps": 24000,
+    "audio_mel_bins":16,
+    "double_precision_rope": false,
+    "use_tiling_vae": false,
+    "dit_original_ckpt": "/data/wushuo1/models/LTX-2.3/ltx-2.3-22b-dev.safetensors",
+    "caption_proj_before_connector": true,
+    "cross_attention_adaln": true,
+    "apply_gated_attention": true,
+    "mm_guider": {
+        "enabled": true,
+        "video": {
+            "cfg_scale": 3.0,
+            "stg_scale": 1.0,
+            "stg_blocks": [28],
+            "modality_scale": 3.0,
+            "rescale_scale": 0.7,
+            "skip_step": 0
+        },
+        "audio": {
+            "cfg_scale": 7.0,
+            "stg_scale": 1.0,
+            "stg_blocks": [28],
+            "modality_scale": 3.0,
+            "rescale_scale": 0.7,
+            "skip_step": 0
+        }
+    }
+}

configs/platforms/ascend_npu/qwen_image_t2i_2512.json → configs/platforms/ascend_npu/single/qwen_image_t2i_2512.json

@@ -3,7 +3,7 @@
     "aspect_ratio": "16:9",
     "prompt_template_encode": "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
     "prompt_template_encode_start_idx": 34,
-    "attn_type": "flash_attn3",
+    "attn_type": "npu_flash_attn",
     "enable_cfg": true,
     "sample_guide_scale": 4.0,
     "cpu_offload": true,