文章
文章用户商铺文档快讯圈子网址导航供求信息

Dify v1.4.0中的Multi-Modal LLM Output：基本操作和原理


本文使用dify 1.4.0版本，该版本除了浅色和深色模式之间切换功能外，还有一个重要功能，即多模态大语言模型输出，允许大语言模型（LLMs）在 LLMNode 中同时处理文本和图像。这一功能提升了数据交互的丰富性，并增强了模型的能力。

Dify v1.4.0中的Multi-Modal LLM Output：基本操作和原理

一.Gemini 2.0 Flash Exp模型

1.测试Chatflow流程

现在可使用 Gemini 2.0 Flash Exp 模型尝试这些新功能，这一集成确保了 AI 能够无缝处理和响应文本与图像数据。其中，Gemini 是 Google 推出的一系列多模态 AI 模型，能够处理和生成多种类型的数据，包括文本、图像、音频和视频。

Dify v1.4.0中的Multi-Modal LLM Output：基本操作和原理
Dify v1.4.0中的Multi-Modal LLM Output：基本操作和原理

2.Gemini的API key验证

由于Dify在做Gemini的API key验证的时候，使用的google库，所以基本上直接是网络原因导致验证失败，可以使用https://cloud.dify.ai/或者其它代理设置方式进行操作。

Dify v1.4.0中的Multi-Modal LLM Output：基本操作和原理

3.流程输出结果 

data: {"event": "workflow_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sequence_number": 2, "inputs": {"sys.query": "u63cfu8ff0u56feu50cf", "sys.files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=82184cad92dcb7aea431765ce35105d3&sign=K2C69sZxG4Z1U-WSe41xbLKpz1JHbh-JAjv2rAl4NiI="}], "sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "sys.dialogue_count": 0, "sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d", "sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"}, "created_at": 1748335614}}

data: {"event": "node_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "2800f85c-907b-4bcc-aab0-273a4b87779e", "node_id": "1748333062401", "node_type": "start", "title": "u5f00u59cb", "index": 1, "predecessor_node_id": null, "inputs": null, "created_at": 1748335614, "extras": {}, "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null, "parallel_run_id": null, "Agent_strategy": null}}

data: {"event": "node_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "2800f85c-907b-4bcc-aab0-273a4b87779e", "node_id": "1748333062401", "node_type": "start", "title": "u5f00u59cb", "index": 1, "predecessor_node_id": null, "inputs": {"sys.query": "u63cfu8ff0u56feu50cf", "sys.files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=67dfea243007a69609a2b16e6b14bbcd&sign=AA4HmgJKtcdPkztPsadtLzr8LypyaNUAeWmEiuqpugo="}], "sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "sys.dialogue_count": 0, "sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d", "sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"}, "process_data": null, "outputs": {"sys.query": "u63cfu8ff0u56feu50cf", "sys.files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=a83024f2e70dd080937d147963747f64&sign=9bhuGRAnH4YKORSG5l8I9FWhtKaJqiE9gUJ-vYBxwa0="}], "sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "sys.dialogue_count": 0, "sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d", "sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"}, "status": "succeeded", "error": null, "elapsed_time": 0.031084, "execution_metadata": null, "created_at": 1748335614, "finished_at": 1748335614, "files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=a83024f2e70dd080937d147963747f64&sign=9bhuGRAnH4YKORSG5l8I9FWhtKaJqiE9gUJ-vYBxwa0="}], "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null}}

data: {"event": "node_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ed004c58-4c89-4432-ab91-29888e10f3d6", "node_id": "llm", "node_type": "llm", "title": "LLM", "index": 2, "predecessor_node_id": "1748333062401", "inputs": null, "created_at": 1748335614, "extras": {}, "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null, "parallel_run_id": null, "agent_strategy": null}}

data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u8fd9", "from_variable_selector": ["llm", "text"]}

data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9", "from_variable_selector": ["llm", "text"]}

data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5fae", "from_variable_selector": ["llm", "text"]}

data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540e", "from_variable_selector": ["llm", "text"]}

data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002", "from_variable_selector": ["llm", "text"]}

data: {"event": "node_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ed004c58-4c89-4432-ab91-29888e10f3d6", "node_id": "llm", "node_type": "llm", "title": "LLM", "index": 2, "predecessor_node_id": "1748333062401", "inputs": null, "process_data": {"model_mode": "chat", "prompts": [{"role": "user", "text": "u63cfu8ff0u56feu50cfn", "files": []}, {"role": "user", "text": "", "files": [{"type": "image", "data": "data:image...[TRUNCATED]...lFTkSuQmCC", "detail": "high"}]}], "model_provider": "langgenius/gemini/google", "model_name": "gemini-2.0-flash-exp"}, "outputs": {"text": "u8fd9u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5faeu7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540eu65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002", "usage": {"prompt_tokens": 0, "prompt_unit_price": "0.0", "prompt_price_unit": "0.0", "prompt_price": "0.0", "completion_tokens": 0, "completion_unit_price": "0.0", "completion_price_unit": "0.0", "completion_price": "0.0", "total_tokens": 0, "total_price": "0.0", "currency": "USD", "latency": 0.0}, "finish_reason": null, "files": []}, "status": "succeeded", "error": null, "elapsed_time": 1.894678, "execution_metadata": {"total_tokens": 0, "total_price": "0.0", "currency": "USD"}, "created_at": 1748335614, "finished_at": 1748335616, "files": [], "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null}}

data: {"event": "node_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ca84c578-56b7-4f9b-be57-299f9ffc5f08", "node_id": "answer", "node_type": "answer", "title": "u76f4u63a5u56deu590d", "index": 3, "predecessor_node_id": "llm", "inputs": null, "created_at": 1748335616, "extras": {}, "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null, "parallel_run_id": null, "agent_strategy": null}}

data: {"event": "node_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ca84c578-56b7-4f9b-be57-299f9ffc5f08", "node_id": "answer", "node_type": "answer", "title": "u76f4u63a5u56deu590d", "index": 3, "predecessor_node_id": "llm", "inputs": null, "process_data": null, "outputs": {"answer": "u8fd9u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5faeu7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540eu65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002", "files": []}, "status": "succeeded", "error": null, "elapsed_time": 0.026377, "execution_metadata": null, "created_at": 1748335616, "finished_at": 1748335616, "files": [], "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null}}

data: {"event": "workflow_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sequence_number": 2, "status": "succeeded", "outputs": {"answer": "u8fd9u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5faeu7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540eu65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002"}, "error": null, "elapsed_time": 1.9400834180414677, "total_tokens": 0, "total_steps": 3, "created_by": {"id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "name": "HISu7cfbu7edf", "email": "913292836@qq.com"}, "created_at": 1748335614, "finished_at": 1748335616, "exceptions_count": 0, "files": []}}

data: {"event": "message_end", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "metadata": {"usage": {"prompt_tokens": 0, "prompt_unit_price": "0.0", "prompt_price_unit": "0.0", "prompt_price": "0.0", "completion_tokens": 0, "completion_unit_price": "0.0", "completion_price_unit": "0.0", "completion_price": "0.0", "total_tokens": 0, "total_price": "0.0", "currency": "USD", "latency": 0.0}}, "files": []}

（1）node_started

{
"event" : "node_started",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"workflow_run_id" : "0e2f2237-05dc-4272-b536-3e97bf02691c",
"data" : {
"id" : "ed004c58-4c89-4432-ab91-29888e10f3d6",
"node_id" : "llm",
"node_type" : "llm",
"title" : "LLM",
"index" : 2,
"predecessor_node_id" : "1748333062401",
"inputs" : null,
"created_at" : 1748335614,
"extras" : {
        },
"parallel_id" : null,
"parallel_start_node_id" : null,
"parent_parallel_id" : null,
"parent_parallel_start_node_id" : null,
"iteration_id" : null,
"loop_id" : null,
"parallel_run_id" : null,
"agent_strategy" : null
    }
}

（2）message1

{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "这",
"from_variable_selector" : [
"llm",
"text"
    ]
}

（3）message2

{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "幅照片是一个特写镜头，拍摄了一只德国牧羊犬。这",
"from_variable_selector" : [
"llm",
"text"
    ]
}

（4）message3

{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "只狗有着传统的黑褐色毛皮，竖立着一对警惕的耳朵。它的眼睛是温暖的棕色，直视前方。它的嘴巴微微张开，露出粉红色的舌头，显示出轻微",
"from_variable_selector" : [
"llm",
"text"
    ]
}

（5）message4

{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "的喘息。它戴着一个金属链条项圈，上面挂着一个身份牌。背景是柔和模糊的绿色和棕色，暗示着户外环境，可能是草地或树林，阳光似乎从后",
"from_variable_selector" : [
"llm",
"text"
    ]
}

（6）message5

{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "方照射，照亮了狗狗的轮廓。",
"from_variable_selector" : [
"llm",
"text"
    ]
}

（7）node_finished

{
"event" : "node_finished",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"workflow_run_id" : "0e2f2237-05dc-4272-b536-3e97bf02691c",
"data" : {
"id" : "ed004c58-4c89-4432-ab91-29888e10f3d6",
"node_id" : "llm",
"node_type" : "llm",
"title" : "LLM",
"index" : 2,
"predecessor_node_id" : "1748333062401",
"inputs" : null,
"process_data" : {
"model_mode" : "chat",
"prompts" : [
                {
"role" : "user",
"text" : "描述图像
",
"files" : [
                    ]
                },
                {
"role" : "user",
"text" : "",
"files" : [
                        {
"type" : "image",
"data" : "data:image...[TRUNCATED]...lFTkSuQmCC",
"detail" : "high"
                        }
                    ]
                }
            ],
"model_provider" : "langgenius/gemini/google",
"model_name" : "gemini-2.0-flash-exp"
        },
"outputs" : {
"text" : "这幅照片是一个特写镜头，拍摄了一只德国牧羊犬。这只狗有着传统的黑褐色毛皮，竖立着一对警惕的耳朵。它的眼睛是温暖的棕色，直视前方。它的嘴巴微微张开，露出粉红色的舌头，显示出轻微的喘息。它戴着一个金属链条项圈，上面挂着一个身份牌。背景是柔和模糊的绿色和棕色，暗示着户外环境，可能是草地或树林，阳光似乎从后方照射，照亮了狗狗的轮廓。",
"usage" : {
"prompt_tokens" : 0,
"prompt_unit_price" : "0.0",
"prompt_price_unit" : "0.0",
"prompt_price" : "0.0",
"completion_tokens" : 0,
"completion_unit_price" : "0.0",
"completion_price_unit" : "0.0",
"completion_price" : "0.0",
"total_tokens" : 0,
"total_price" : "0.0",
"currency" : "USD",
"latency" : 0.0
            },
"finish_reason" : null,
"files" : [
            ]
        },
"status" : "succeeded",
"error" : null,
"elapsed_time" : 1.894678,
"execution_metadata" : {
"total_tokens" : 0,
"total_price" : "0.0",
"currency" : "USD"
        },
"created_at" : 1748335614,
"finished_at" : 1748335616,
"files" : [
        ],
"parallel_id" : null,
"parallel_start_node_id" : null,
"parent_parallel_id" : null,
"parent_parallel_start_node_id" : null,
"iteration_id" : null,
"loop_id" : null
    }
}

二.开始节点

1.输入 

{
"sys.query": "描述图像",
"sys.files": [
    {
"dify_model_identity": "__dify__file__",
"id": null,
"tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae",
"type": "image",
"transfer_method": "local_file",
"remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=",
"related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1",
"filename": "dog.png",
"extension": ".png",
"mime_type": "image/png",
"size": 368313,
"url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=67dfea243007a69609a2b16e6b14bbcd&sign=AA4HmgJKtcdPkztPsadtLzr8LypyaNUAeWmEiuqpugo="
    }
  ],
"sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a",
"sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377",
"sys.dialogue_count": 0,
"sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d",
"sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46",
"sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"
}

2.输出 

{
"sys.query": "描述图像",
"sys.files": [
    {
"dify_model_identity": "__dify__file__",
"id": null,
"tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae",
"type": "image",
"transfer_method": "local_file",
"remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=",
"related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1",
"filename": "dog.png",
"extension": ".png",
"mime_type": "image/png",
"size": 368313,
"url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=a83024f2e70dd080937d147963747f64&sign=9bhuGRAnH4YKORSG5l8I9FWhtKaJqiE9gUJ-vYBxwa0="
    }
  ],
"sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a",
"sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377",
"sys.dialogue_count": 0,
"sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d",
"sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46",
"sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"
}

三.LLM节点

1.数据处理 

{
"model_mode": "chat",
"prompts": [
    {
"role": "user",
"text": "描述图像n",
"files": []
    },
    {
"role": "user",
"text": "",
"files": [
        {
"type": "image",
"data": "data:image...[TRUNCATED]...lFTkSuQmCC",
"detail": "high"
        }
      ]
    }
  ],
"model_provider": "langgenius/gemini/google",
"model_name": "gemini-2.0-flash-exp"
}

2.输出 

{
"text": "这幅照片是一个特写镜头，拍摄了一只德国牧羊犬。这只狗有着传统的黑褐色毛皮，竖立着一对警惕的耳朵。它的眼睛是温暖的棕色，直视前方。它的嘴巴微微张开，露出粉红色的舌头，显示出轻微的喘息。它戴着一个金属链条项圈，上面挂着一个身份牌。背景是柔和模糊的绿色和棕色，暗示着户外环境，可能是草地或树林，阳光似乎从后方照射，照亮了狗狗的轮廓。",
"usage": {
"prompt_tokens": 0,
"prompt_unit_price": "0.0",
"prompt_price_unit": "0.0",
"prompt_price": "0.0",
"completion_tokens": 0,
"completion_unit_price": "0.0",
"completion_price_unit": "0.0",
"completion_price": "0.0",
"total_tokens": 0,
"total_price": "0.0",
"currency": "USD",
"latency": 0
  },
"finish_reason": null,
"files": []
}

四.直接回复节点

1.输出 

{
"answer": "这幅照片是一个特写镜头，拍摄了一只德国牧羊犬。这只狗有着传统的黑褐色毛皮，竖立着一对警惕的耳朵。它的眼睛是温暖的棕色，直视前方。它的嘴巴微微张开，露出粉红色的舌头，显示出轻微的喘息。它戴着一个金属链条项圈，上面挂着一个身份牌。背景是柔和模糊的绿色和棕色，暗示着户外环境，可能是草地或树林，阳光似乎从后方照射，照亮了狗狗的轮廓。",
"files": []
}

五.LLM节点-数据处理

这个数据是LLM节点处理过程中生成的process_data对象，描述了一个多模态AI模型调用的请求配置。它是这样得到的：

1.模型配置信息

  • model_mode: “chat”表示使用聊天模式，这来自model_config.mode

  • model_provider: “langgenius/gemini/google”表示使用Google的Gemini模型

  • model_name: “gemini-2.0-flash-exp”是具体使用的模型版本

2.提示消息构建

通过_fetch_prompt_messages方法生成，包含两个用户消息：（1）第一个消息包含文本提示”描述图像”（2）第二个消息包含图像文件，使用base64编码存储

3.图像处理

  • 图像通过_fetch_files方法获取

  • detail: "high"表示要求模型进行高分辨率的图像分析，对应代码中的vision_detail_config参数

4.数据流程

  • 当工作流执行到LLM节点时，系统提取用户配置的提示和上传的图像

  • 组装成规范化的请求格式

  • 记录在process_data中，用于后续调用AI模型和记录执行过程

这个数据随后会被传递给_invoke_llm方法，向Gemini模型发送请求来获取对图像的描述结果。

参考文献

[1] https://github.com/langgenius/dify/releases/tag/1.4.0

[2] https://ai.google.dev/

[3] https://aistudio.google.com/

[4] Dify v1.4.0中的Multi-Modal LLM Output：基本操作和原理：https://z0yrmerhgi8.feishu.cn/wiki/C3enwZFXci7SS0kKGhQcgv9Mnnh

知识星球服务内容：Dify源码剖析及答疑，Dify对话系统源码，NLP电子书籍报告下载，公众号所有付费资料。加微信buxingtianxia21进NLP工程化资料群。Dify v1.4.0中的Multi-Modal LLM Output：基本操作和原理

RAG技术前沿技术新闻资讯

Chonkie：开源、轻量、极速的 RAG 分块神器 🦛

2025-6-23 16:24:41

RAG技术前沿技术新闻资讯

Chonkie：开源、轻量、极速的 RAG 分块神器 🦛

2025-6-23 17:29:08

0 条回复 A文章作者 M管理员
    暂无讨论，说说你的看法吧
❯
购物车
优惠劵
搜索

  • 扫码打开当前页

  • 联系我们

返回顶部