add more ns examples (intel#1346)

Author: VincyZhang

docs/weightonlyquant.md

@@ -136,6 +136,7 @@ python -m pip install torch==2.1.0a0  -f https://developer.intel.com/ipex-whl-st
 
 source /opt/intel/oneapi/setvars.sh
 
+# Build IPEX from Source Code
 git clone https://github.com/intel/intel-extension-for-pytorch.git ipex-gpu
 cd ipex-gpu
 git checkout -b dev/QLLM origin/dev/QLLM
@@ -144,6 +145,7 @@ export USE_AOT_DEVLIST='pvc,ats-m150'
 export BUILD_WITH_CPU=OFF
 
 pip install -r requirements.txt
+
 python setup.py install
 ```
 

examples/.config/neural_speed_deploy.json

@@ -22,6 +22,48 @@
             "params": {
                 "model_name": "/tf_dataset2/models/nlp_toolkit/neural-chat-7b-v3-1",
                 "model_format": "runtime",
+                "tasks": "piqa"
+            }
+        },
+        "launcher":{}
+    },
+    "neural_chat_v3-3_autoround_neural_speed": {
+        "working_dir": "huggingface/neural_speed",
+        "data_dir": "",
+        "hf_model_name": "Intel/neural-chat-7b-v3-3",
+        "benchmark": {
+            "cmd": "python run_accuracy.py",
+            "params": {
+                "model_name": "/tf_dataset2/models/auto_round/neuralchat_v3-3",
+                "model_format": "runtime",
+                "tasks": "lambada_openai"
+            }
+        },
+        "launcher":{}
+    },
+    "mistral_7b_neural_speed": {
+        "working_dir": "huggingface/neural_speed",
+        "data_dir": "",
+        "hf_model_name": "mistralai/Mistral-7B-v0.1",
+        "benchmark": {
+            "cmd": "python run_accuracy.py",
+            "params": {
+                "model_name": "/tf_dataset2/models/pytorch/Mistral-7B-v0.1",
+                "model_format": "runtime",
+                "tasks": "piqa"
+            }
+        },
+        "launcher":{}
+    },
+    "qwen_neural_speed": {
+        "working_dir": "huggingface/neural_speed",
+        "data_dir": "",
+        "hf_model_name": "Qwen/Qwen-7B-Chat",
+        "benchmark": {
+            "cmd": "python run_accuracy.py",
+            "params": {
+                "model_name": "/tf_dataset2/models/nlp_toolkit/Qwen-7B-Chat",
+                "model_format": "runtime",
                 "tasks": "lambada_openai"
             }
         },

examples/huggingface/neural_speed/requirements.txt

@@ -7,3 +7,5 @@ gguf
 torch==2.2.0+cpu
 transformers
 intel_extension_for_pytorch==2.2.0
+tiktoken
+transformers_stream_generator

examples/huggingface/neural_speed/run_accuracy.py

@@ -24,7 +24,7 @@
     parser.add_argument('--batch_size', type=int, default=1)
     args = parser.parse_args()
     print(args)
-    model_args=f'pretrained="{args.model_name}",dtype=float32'
+    model_args=f'pretrained="{args.model_name}",dtype=float32,trust_remote_code=True'
     if args.use_gptq:
         model_args += ",use_gptq=True"
     if args.model_format == "runtime":

examples/huggingface/neural_speed/run_autoround_qdq.py

@@ -24,7 +24,7 @@
 
     results = evaluate(
         model="hf-causal",
-        model_args=f'pretrained="{args.model_name}",dtype=float32',
+        model_args=f'pretrained="{args.model_name}",dtype=float32,trust_remote_code=True',
         tasks=[f"{args.tasks}"]
     )
 

examples/huggingface/neural_speed/run_inference.py

@@ -39,7 +39,7 @@ def main(args_in: Optional[List[str]] = None) -> None:
     streamer = TextStreamer(tokenizer)
     inputs = tokenizer(prompt, return_tensors="pt").input_ids
 
-    model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=woq_config)
+    model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=woq_config, trust_remote_code=True)
 
     outputs = model.generate(inputs, streamer=streamer, ctx_size=args.n_ctx, max_new_tokens=args.max_new_tokens)
 

intel_extension_for_transformers/neural_chat/README.md

@@ -61,10 +61,6 @@ pip install -r requirements.txt
 >**Note**: Suggest using fastapi==0.103.2
 
 
->**Note**: Suggest using fastapi==0.103.2
-
-
-
 # Getting Started
 
 ## OpenAI-Compatible RESTful APIs