run.yaml

#
#
# Copyright Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: 2
image_name: redhat-ai-dev-llama-stack
apis:
  - agents
  - inference
  - safety
  - tool_runtime
  - vector_io
  - files
container_image:
external_providers_dir:
providers:
  agents:
    - config:
        persistence:
          agent_state:
            namespace: agents
            backend: kv_default
          responses:
            table_name: responses
            backend: sql_default
      provider_id: meta-reference
      provider_type: inline::meta-reference
  inference:
    - provider_id: ${env.ENABLE_VLLM:+vllm}
      provider_type: remote::vllm
      config:
        url: ${env.VLLM_URL:=}
        api_token: ${env.VLLM_API_KEY:=}
        max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
        tls_verify: ${env.VLLM_TLS_VERIFY:=true}
    - provider_id: ${env.ENABLE_OLLAMA:+ollama}
      provider_type: remote::ollama
      config:
        url: ${env.OLLAMA_URL:=http://localhost:11434}
    - provider_id: ${env.ENABLE_OPENAI:+openai}
      provider_type: remote::openai
      config:
        api_key: ${env.OPENAI_API_KEY:=}
    - provider_id: ${env.ENABLE_VERTEX_AI:+vertexai}
      provider_type: remote::vertexai
      config:
        project: ${env.VERTEX_AI_PROJECT:=}
        location: ${env.VERTEX_AI_LOCATION:=us-central1}
    - provider_id: safety-guard
      provider_type: remote::vllm
      config:
        url: ${env.SAFETY_URL:=http://host.docker.internal:11434/v1}
        api_token: ${env.SAFETY_API_KEY:=token}
    - provider_id: sentence-transformers
      provider_type: inline::sentence-transformers
      config: {}
  safety:
    - provider_id: llama-guard
      provider_type: inline::llama-guard
      config:
        excluded_categories: []
  tool_runtime:
    - provider_id: model-context-protocol
      provider_type: remote::model-context-protocol
      config: {}
    - provider_id: rag-runtime
      provider_type: inline::rag-runtime
      config: {}
  vector_io:
    - provider_id: faiss
      provider_type: inline::faiss
      config:
        persistence:
          namespace: vector_io::faiss
          backend: faiss_kv
  files:
    - provider_id: localfs
      provider_type: inline::localfs
      config:
        storage_dir: /tmp/llama-stack-files
        metadata_store:
          table_name: files_metadata
          backend: sql_files
storage:
  backends:
    kv_default:
      type: kv_sqlite
      db_path: /tmp/kvstore.db
    sql_default:
      type: sql_sqlite
      db_path: /tmp/sql_store.db
    sql_files:
      type: sql_sqlite
      db_path: /rag-content/vector_db/rhdh_product_docs/1.9/files_metadata.db
    faiss_kv:
      type: kv_sqlite
      db_path: /rag-content/vector_db/rhdh_product_docs/1.9/faiss_store.db
  stores:
    metadata:
      namespace: registry
      backend: faiss_kv
    inference:
      table_name: inference_store
      backend: sql_default
      max_write_queue_size: 10000
      num_writers: 4
    conversations:
      table_name: openai_conversations
      backend: sql_default
registered_resources:
  models:
    - model_id: sentence-transformers/all-mpnet-base-v2
      metadata:
        embedding_dimension: 768
      model_type: embedding
      provider_id: sentence-transformers
      provider_model_id: /rag-content/embeddings_model
    - model_id: ${env.SAFETY_MODEL:=llama-guard3:8b}
      provider_id: safety-guard
      provider_model_id: ${env.SAFETY_MODEL:=llama-guard3:8b}
      model_type: llm
      metadata: {}
  shields:
    - shield_id: llama-guard-shield
      provider_id: llama-guard
      provider_shield_id: safety-guard/${env.SAFETY_MODEL:=llama-guard3:8b}
  tool_groups:
    - provider_id: rag-runtime
      toolgroup_id: builtin::rag
  vector_dbs:
    - vector_db_id: rhdh-product-docs-1_8
      embedding_model: sentence-transformers/all-mpnet-base-v2
      embedding_dimension: 768
      provider_id: faiss
server:
  auth:
  host:
  port: 8321
  quota:
  tls_cafile:
  tls_certfile:
  tls_keyfile: