feat(cmd): add datasource template (#449)

* feat: add template for datasource plugins

Signed-off-by: jingfelix <[email protected]>

* fix: description and missing category

Signed-off-by: jingfelix <[email protected]>

---------

Signed-off-by: jingfelix <[email protected]>

Author: jingfelix

cmd/commandline/plugin/category.go

@@ -23,12 +23,13 @@ const PLUGIN_GUIDE = `Before starting, here's some basic knowledge about Plugin
 ` + BOLD + `- Endpoint` + RESET + `: ` + GREEN + `Similar to Service API in Dify and Ingress in Kubernetes. Extend HTTP services as endpoints with custom logic.` + RESET + `
 ` + BOLD + `- Agent Strategy` + RESET + `: ` + GREEN + `Implement your own agent strategies like Function Calling, ReAct, ToT, CoT, etc.` + RESET + `
 
-Based on the ability you want to extend, Plugins are divided into four types: ` + BOLD + `Tool` + RESET + `, ` + BOLD + `Model` + RESET + `, ` + BOLD + `Extension` + RESET + `, and ` + BOLD + `Agent Strategy` + RESET + `.
+Based on the ability you want to extend, Plugins are divided into five types: ` + BOLD + `Tool` + RESET + `, ` + BOLD + `Model` + RESET + `, ` + BOLD + `Extension` + RESET + `, ` + BOLD + `Agent Strategy` + RESET + `, and ` + BOLD + `Datasource` + RESET + `.
 
 ` + BOLD + `- Tool` + RESET + `: ` + YELLOW + `A tool provider that can also implement endpoints. For example, building a Discord Bot requires both ` + BLUE + `Sending` + RESET + YELLOW + ` and ` + BLUE + `Receiving Messages` + RESET + YELLOW + `, so both ` + BOLD + `Tool` + RESET + YELLOW + ` and ` + BOLD + `Endpoint` + RESET + YELLOW + ` functionality.` + RESET + `
 ` + BOLD + `- Model` + RESET + `: ` + YELLOW + `Strictly for model providers, no other extensions allowed.` + RESET + `
 ` + BOLD + `- Extension` + RESET + `: ` + YELLOW + `For simple HTTP services that extend functionality.` + RESET + `
 ` + BOLD + `- Agent Strategy` + RESET + `: ` + YELLOW + `Implement custom agent logic with a focused approach.` + RESET + `
+` + BOLD + `- Datasource` + RESET + `: ` + YELLOW + `Provide datasource for Dify RAG Pipeline.` + RESET + `
 
 We've provided templates to help you get started. Choose one of the options below:
 `
@@ -47,6 +48,7 @@ var categories = []string{
 	"speech2text",
 	"moderation",
 	"extension",
+	"datasource",
 }
 
 func newCategory() category {

cmd/commandline/plugin/init.go

@@ -156,6 +156,7 @@ func InitPluginWithFlags(
 			"tts",
 			"extension",
 			"agent-strategy",
+			"datasource",
 		}
 		valid := false
 		for _, cat := range validCategories {
@@ -443,6 +444,10 @@ func (m model) createPlugin() {
 		manifest.Plugins.AgentStrategies = []string{fmt.Sprintf("provider/%s.yaml", manifest.Name)}
 	}
 
+	if categoryString == "datasource" {
+		manifest.Plugins.Datasources = []string{fmt.Sprintf("provider/%s.yaml", manifest.Name)}
+	}
+
 	manifest.Meta = plugin_entities.PluginMeta{
 		Version: "0.0.1",
 		Arch: []constants.Arch{

cmd/commandline/plugin/python.go

@@ -91,6 +91,18 @@ var PYTHON_AGENT_STRATEGY_MANIFEST_TEMPLATE []byte
 //go:embed templates/python/agent_strategy.py
 var PYTHON_AGENT_STRATEGY_TEMPLATE []byte
 
+//go:embed templates/python/datasource.yaml
+var PYTHON_DATASOURCE_MANIFEST_TEMPLATE []byte
+
+//go:embed templates/python/datasource.py
+var PYTHON_DATASOURCE_TEMPLATE []byte
+
+//go:embed templates/python/datasource_provider.yaml
+var PYTHON_DATASOURCE_PROVIDER_MANIFEST_TEMPLATE []byte
+
+//go:embed templates/python/datasource_provider.py
+var PYTHON_DATASOURCE_PROVIDER_PY_TEMPLATE []byte
+
 //go:embed templates/python/GUIDE.md
 var PYTHON_GUIDE []byte
 
@@ -231,5 +243,11 @@ func createPythonEnvironment(
 		}
 	}
 
+	if category == "datasource" {
+		if err := createPythonDatasource(root, manifest); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

cmd/commandline/plugin/python_categories.go

@@ -270,3 +270,25 @@ func createPythonAgentStrategy(root string, manifest *plugin_entities.PluginDecl
 
 	return nil
 }
+
+func createPythonDatasource(root string, manifest *plugin_entities.PluginDeclaration) error {
+	datasourceFileContent, err := renderTemplate(PYTHON_DATASOURCE_TEMPLATE, manifest, []string{""})
+	if err != nil {
+		return err
+	}
+	datasourceFilePath := filepath.Join(root, "datasources", fmt.Sprintf("%s.py", manifest.Name))
+	if err := writeFile(datasourceFilePath, datasourceFileContent); err != nil {
+		return err
+	}
+
+	datasourceManifestFilePath := filepath.Join(root, "datasources", fmt.Sprintf("%s.yaml", manifest.Name))
+	datasourceManifestFileContent, err := renderTemplate(PYTHON_DATASOURCE_MANIFEST_TEMPLATE, manifest, []string{""})
+	if err != nil {
+		return err
+	}
+	if err := writeFile(datasourceManifestFilePath, datasourceManifestFileContent); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/commandline/plugin/templates/python/datasource.py

@@ -0,0 +1,147 @@
+# For online document, you can use the following code:
+import logging
+import re
+import urllib.parse
+from collections.abc import Generator
+from typing import Any
+
+from dify_plugin.entities.datasource import (
+    DatasourceGetPagesResponse,
+    DatasourceMessage,
+    GetOnlineDocumentPageContentRequest,
+    OnlineDocumentInfo,
+    OnlineDocumentPage,
+)
+from dify_plugin.interfaces.datasource.online_document import OnlineDocumentDatasource
+
+logger = logging.getLogger(__name__)
+
+
+class {{ .PluginName | SnakeToCamel }}DataSource(OnlineDocumentDatasource):
+
+    def _get_pages(self, datasource_parameters: dict[str, Any]) -> DatasourceGetPagesResponse:
+        page = OnlineDocumentPage(
+            page_name="",
+            page_id="",
+            type="page",
+            last_edited_time="",
+            parent_id="",
+            page_icon=None,
+        )
+        # Get workspace info from credentials
+        workspace_name = self.runtime.credentials.get("workspace_name", "{{ .PluginName }}")
+        workspace_id = self.runtime.credentials.get("workspace_id", "unknown")
+        workspace_icon = self.runtime.credentials.get("workspace_icon", "")
+
+        online_document_info = OnlineDocumentInfo(
+            workspace_name=workspace_name,
+            workspace_icon=workspace_icon,
+            workspace_id=workspace_id,
+            pages=[page],
+            total=1,
+        )
+        return DatasourceGetPagesResponse(result=[online_document_info])
+
+    def _get_content(self, page: GetOnlineDocumentPageContentRequest) -> Generator[DatasourceMessage, None, None]:
+       
+        yield self.create_variable_message("content", "")
+        yield self.create_variable_message("page_id", "")
+        yield self.create_variable_message("workspace_id", "")
+
+
+# For website crawl, you can use the following code:
+# from typing import Any, Generator
+
+# from dify_plugin.entities.datasource import WebSiteInfo, WebSiteInfoDetail
+# from dify_plugin.entities.tool import ToolInvokeMessage
+# from dify_plugin.interfaces.datasource.website import WebsiteCrawlDatasource
+
+
+# class {{ .PluginName | SnakeToCamel }}DataSource(WebsiteCrawlDatasource):
+
+#     def _get_website_crawl(
+#         self, datasource_parameters: dict[str, Any]
+#     ) -> Generator[ToolInvokeMessage, None, None]:
+
+#         crawl_res = WebSiteInfo(web_info_list=[], status="", total=0, completed=0)
+#         crawl_res.status = "processing"
+#         yield self.create_crawl_message(crawl_res)
+
+#         crawl_res.status = "completed"
+#         crawl_res.web_info_list = [
+#             WebSiteInfoDetail(
+#                 title="",
+#                 source_url="",
+#                 description="",
+#                 content="",
+#             )
+#         ]
+#         crawl_res.total = 1
+#         crawl_res.completed = 1
+
+#         yield self.create_crawl_message(crawl_res)
+
+
+# For online drive, you can use the following code:
+# from collections.abc import Generator
+
+# from dify_plugin.entities.datasource import (
+#     DatasourceMessage,
+#     OnlineDriveBrowseFilesRequest,
+#     OnlineDriveBrowseFilesResponse,
+#     OnlineDriveDownloadFileRequest,
+#     OnlineDriveFile,
+#     OnlineDriveFileBucket,
+# )
+# from dify_plugin.interfaces.datasource.online_drive import OnlineDriveDatasource
+
+
+# class {{ .PluginName | SnakeToCamel }}DataSource(OnlineDriveDatasource):
+
+#     def _browse_files(
+#         self, request: OnlineDriveBrowseFilesRequest
+#     ) -> OnlineDriveBrowseFilesResponse:
+
+#         credentials = self.runtime.credentials
+#         bucket_name = request.bucket
+#         prefix = request.prefix or ""  # Allow empty prefix for root folder; When you browse the folder, the prefix is the folder id
+#         max_keys = request.max_keys or 10
+#         next_page_parameters = request.next_page_parameters or {}
+
+#         files = []
+#         files.append(OnlineDriveFile(
+#             id="", 
+#             name="", 
+#             size=0, 
+#             type="folder" # or "file"
+#         ))
+
+#         return OnlineDriveBrowseFilesResponse(result=[
+#             OnlineDriveFileBucket(
+#                 bucket="", 
+#                 files=files, 
+#                 is_truncated=False, 
+#                 next_page_parameters={}
+#             )
+#         ])
+
+#     # if file.type is "file", the plugin will download the file content
+#     def _download_file(self, request: OnlineDriveDownloadFileRequest) -> Generator[DatasourceMessage, None, None]:
+#         credentials = self.runtime.credentials
+#         file_id = request.id
+
+#         file_content = bytes()
+#         file_name = ""
+
+#         mime_type = self._get_mime_type_from_filename(file_name)
+        
+#         yield self.create_blob_message(file_content, meta={
+#             "file_name": file_name,
+#             "mime_type": mime_type
+#         })
+
+#     def _get_mime_type_from_filename(self, filename: str) -> str:
+#         """Determine MIME type from file extension."""
+#         import mimetypes
+#         mime_type, _ = mimetypes.guess_type(filename)
+#         return mime_type or "application/octet-stream"

cmd/commandline/plugin/templates/python/datasource.yaml

@@ -0,0 +1,72 @@
+identity:
+  name: "{{ .PluginName }}"
+  author: "{{ .Author }}"
+  label:
+    en_US: "{{ .PluginName }}"
+    zh_Hans: "{{ .PluginName }}"
+    pt_BR: "{{ .PluginName }}"
+description:
+  human:
+    en_US: "{{ .PluginDescription }}"
+    zh_Hans: "{{ .PluginDescription }}"
+    pt_BR: "{{ .PluginDescription }}"
+  llm: "{{ .PluginDescription }}"
+
+# For website crawl, you can use the following code:
+parameters:
+  - name: query
+    type: string
+    required: true
+    label:
+      en_US: Query string
+      zh_Hans: 查询语句
+      pt_BR: Query string
+    human_description:
+      en_US: "{{ .PluginDescription }}"
+      zh_Hans: "{{ .PluginDescription }}"
+      pt_BR: "{{ .PluginDescription }}"
+    llm_description: "{{ .PluginDescription }}"
+    form: llm
+output_schema:
+    type: object
+    properties:
+      source_url:
+        type: string
+        description: the source url of the website
+      content:
+        type: string
+        description: the content from the website
+      title:
+        type: string
+        description: the title of the website
+      "description":
+        type: string
+        description: the description of the website
+
+# For online document, you can use the following code:
+parameters:
+output_schema:
+    type: object
+    properties:
+      workspace_id:
+        type: string
+        description: workspace id
+      page_id:
+        type: string
+        description: page id
+      content:
+        type: string
+        description: page content
+
+
+# For online document, you can use the following code:
+parameters:
+output_schema:
+    type: object
+    properties:
+      file:
+        $ref: "https://dify.ai/schemas/v1/file.json"
+
+extra:
+  python:
+    source: datasources/{{ .PluginName }}.py

cmd/commandline/plugin/templates/python/datasource_provider.py

@@ -0,0 +1,68 @@
+from typing import Any, Mapping
+import secrets
+import urllib.parse
+
+from dify_plugin.errors.tool import ToolProviderCredentialValidationError, DatasourceOAuthError
+from dify_plugin.interfaces.datasource import DatasourceProvider, DatasourceOAuthCredentials
+
+
+
+class {{ .PluginName | SnakeToCamel }}Provider(DatasourceProvider):
+
+    def _validate_credentials(self, credentials: Mapping[str, Any]) -> None:
+        try:
+            """
+            IMPLEMENT YOUR VALIDATION HERE
+            """
+        except Exception as e:
+            raise ToolProviderCredentialValidationError(str(e))
+
+
+    #########################################################################################
+    # If OAuth is supported, uncomment the following functions.
+    # Warning: please make sure that the sdk version is 0.5.0 or higher.
+    #########################################################################################
+    # def _oauth_get_authorization_url(self, redirect_uri: str, system_credentials: Mapping[str, Any]) -> str:
+    #     """
+    #     Generate the authorization URL for {{ .PluginName }} OAuth.
+    #     """
+    #     try:
+    #         """
+    #         IMPLEMENT YOUR AUTHORIZATION URL GENERATION HERE
+    #         """
+    #     except Exception as e:
+    #         raise DatasourceOAuthError(str(e))
+    #     return ""
+
+    # def _oauth_get_credentials(
+    #     self, redirect_uri: str, system_credentials: Mapping[str, Any], request: Request
+    # ) -> DatasourceOAuthCredentials:
+    #     """
+    #     Exchange code for access_token.
+    #     """
+    #     try:
+    #         """
+    #         IMPLEMENT YOUR CREDENTIALS EXCHANGE HERE
+    #         """
+    #     except Exception as e:
+    #         raise DatasourceOAuthError(str(e))
+    #     return DatasourceOAuthCredentials(
+    #         name="",
+    #         avatar_url="",
+    #         expires_at=-1,
+    #         credentials={},
+    #     )
+
+    # def _oauth_refresh_credentials(
+    #     self, redirect_uri: str, system_credentials: Mapping[str, Any], credentials: Mapping[str, Any]
+    # ) -> DatasourceOAuthCredentials:
+    #     """
+    #     Refresh the credentials
+    #     """
+    #     return DatasourceOAuthCredentials(
+    #         name="",
+    #         avatar_url="",
+    #         expires_at=-1,
+    #         credentials={},
+    #     )
+