本文使用dify 1.4.0版本,该版本除了浅色和深色模式之间切换功能外,还有一个重要功能,即多模态大语言模型输出,允许大语言模型(LLMs)在 LLMNode 中同时处理文本和图像。这一功能提升了数据交互的丰富性,并增强了模型的能力。

一.Gemini 2.0 Flash Exp模型
1.测试Chatflow流程
现在可使用 Gemini 2.0 Flash Exp 模型尝试这些新功能,这一集成确保了 AI 能够无缝处理和响应文本与图像数据。其中,Gemini 是 Google 推出的一系列多模态 AI 模型,能够处理和生成多种类型的数据,包括文本、图像、音频和视频。


2.Gemini的API key验证
由于Dify在做Gemini的API key验证的时候,使用的google库,所以基本上直接是网络原因导致验证失败,可以使用https://cloud.dify.ai/或者其它代理设置方式进行操作。

3.流程输出结果
data: {"event": "workflow_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sequence_number": 2, "inputs": {"sys.query": "u63cfu8ff0u56feu50cf", "sys.files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=82184cad92dcb7aea431765ce35105d3&sign=K2C69sZxG4Z1U-WSe41xbLKpz1JHbh-JAjv2rAl4NiI="}], "sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "sys.dialogue_count": 0, "sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d", "sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"}, "created_at": 1748335614}}
data: {"event": "node_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "2800f85c-907b-4bcc-aab0-273a4b87779e", "node_id": "1748333062401", "node_type": "start", "title": "u5f00u59cb", "index": 1, "predecessor_node_id": null, "inputs": null, "created_at": 1748335614, "extras": {}, "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null, "parallel_run_id": null, "Agent_strategy": null}}
data: {"event": "node_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "2800f85c-907b-4bcc-aab0-273a4b87779e", "node_id": "1748333062401", "node_type": "start", "title": "u5f00u59cb", "index": 1, "predecessor_node_id": null, "inputs": {"sys.query": "u63cfu8ff0u56feu50cf", "sys.files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=67dfea243007a69609a2b16e6b14bbcd&sign=AA4HmgJKtcdPkztPsadtLzr8LypyaNUAeWmEiuqpugo="}], "sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "sys.dialogue_count": 0, "sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d", "sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"}, "process_data": null, "outputs": {"sys.query": "u63cfu8ff0u56feu50cf", "sys.files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=a83024f2e70dd080937d147963747f64&sign=9bhuGRAnH4YKORSG5l8I9FWhtKaJqiE9gUJ-vYBxwa0="}], "sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "sys.dialogue_count": 0, "sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d", "sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"}, "status": "succeeded", "error": null, "elapsed_time": 0.031084, "execution_metadata": null, "created_at": 1748335614, "finished_at": 1748335614, "files": [{"dify_model_identity": "__dify__file__", "id": null, "tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae", "type": "image", "transfer_method": "local_file", "remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=", "related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1", "filename": "dog.png", "extension": ".png", "mime_type": "image/png", "size": 368313, "url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=a83024f2e70dd080937d147963747f64&sign=9bhuGRAnH4YKORSG5l8I9FWhtKaJqiE9gUJ-vYBxwa0="}], "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null}}
data: {"event": "node_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ed004c58-4c89-4432-ab91-29888e10f3d6", "node_id": "llm", "node_type": "llm", "title": "LLM", "index": 2, "predecessor_node_id": "1748333062401", "inputs": null, "created_at": 1748335614, "extras": {}, "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null, "parallel_run_id": null, "agent_strategy": null}}
data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u8fd9", "from_variable_selector": ["llm", "text"]}
data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9", "from_variable_selector": ["llm", "text"]}
data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5fae", "from_variable_selector": ["llm", "text"]}
data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540e", "from_variable_selector": ["llm", "text"]}
data: {"event": "message", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "answer": "u65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002", "from_variable_selector": ["llm", "text"]}
data: {"event": "node_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ed004c58-4c89-4432-ab91-29888e10f3d6", "node_id": "llm", "node_type": "llm", "title": "LLM", "index": 2, "predecessor_node_id": "1748333062401", "inputs": null, "process_data": {"model_mode": "chat", "prompts": [{"role": "user", "text": "u63cfu8ff0u56feu50cfn", "files": []}, {"role": "user", "text": "", "files": [{"type": "image", "data": "data:image...[TRUNCATED]...lFTkSuQmCC", "detail": "high"}]}], "model_provider": "langgenius/gemini/google", "model_name": "gemini-2.0-flash-exp"}, "outputs": {"text": "u8fd9u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5faeu7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540eu65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002", "usage": {"prompt_tokens": 0, "prompt_unit_price": "0.0", "prompt_price_unit": "0.0", "prompt_price": "0.0", "completion_tokens": 0, "completion_unit_price": "0.0", "completion_price_unit": "0.0", "completion_price": "0.0", "total_tokens": 0, "total_price": "0.0", "currency": "USD", "latency": 0.0}, "finish_reason": null, "files": []}, "status": "succeeded", "error": null, "elapsed_time": 1.894678, "execution_metadata": {"total_tokens": 0, "total_price": "0.0", "currency": "USD"}, "created_at": 1748335614, "finished_at": 1748335616, "files": [], "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null}}
data: {"event": "node_started", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ca84c578-56b7-4f9b-be57-299f9ffc5f08", "node_id": "answer", "node_type": "answer", "title": "u76f4u63a5u56deu590d", "index": 3, "predecessor_node_id": "llm", "inputs": null, "created_at": 1748335616, "extras": {}, "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null, "parallel_run_id": null, "agent_strategy": null}}
data: {"event": "node_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "ca84c578-56b7-4f9b-be57-299f9ffc5f08", "node_id": "answer", "node_type": "answer", "title": "u76f4u63a5u56deu590d", "index": 3, "predecessor_node_id": "llm", "inputs": null, "process_data": null, "outputs": {"answer": "u8fd9u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5faeu7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540eu65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002", "files": []}, "status": "succeeded", "error": null, "elapsed_time": 0.026377, "execution_metadata": null, "created_at": 1748335616, "finished_at": 1748335616, "files": [], "parallel_id": null, "parallel_start_node_id": null, "parent_parallel_id": null, "parent_parallel_start_node_id": null, "iteration_id": null, "loop_id": null}}
data: {"event": "workflow_finished", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "data": {"id": "0e2f2237-05dc-4272-b536-3e97bf02691c", "workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46", "sequence_number": 2, "status": "succeeded", "outputs": {"answer": "u8fd9u5e45u7167u7247u662fu4e00u4e2au7279u5199u955cu5934uff0cu62cdu6444u4e86u4e00u53eau5fb7u56fdu7267u7f8au72acu3002u8fd9u53eau72d7u6709u7740u4f20u7edfu7684u9ed1u8910u8272u6bdbu76aeuff0cu7ad6u7acbu7740u4e00u5bf9u8b66u60d5u7684u8033u6735u3002u5b83u7684u773cu775bu662fu6e29u6696u7684u68d5u8272uff0cu76f4u89c6u524du65b9u3002u5b83u7684u5634u5df4u5faeu5faeu5f20u5f00uff0cu9732u51fau7c89u7ea2u8272u7684u820cu5934uff0cu663eu793au51fau8f7bu5faeu7684u5598u606fu3002u5b83u6234u7740u4e00u4e2au91d1u5c5eu94feu6761u9879u5708uff0cu4e0au9762u6302u7740u4e00u4e2au8eabu4efdu724cu3002u80ccu666fu662fu67d4u548cu6a21u7ccau7684u7effu8272u548cu68d5u8272uff0cu6697u793au7740u6237u5916u73afu5883uff0cu53efu80fdu662fu8349u5730u6216u6811u6797uff0cu9633u5149u4f3cu4e4eu4eceu540eu65b9u7167u5c04uff0cu7167u4eaeu4e86u72d7u72d7u7684u8f6eu5ed3u3002"}, "error": null, "elapsed_time": 1.9400834180414677, "total_tokens": 0, "total_steps": 3, "created_by": {"id": "7d8864c3-c456-4588-9b0a-9368c94ca377", "name": "HISu7cfbu7edf", "email": "913292836@qq.com"}, "created_at": 1748335614, "finished_at": 1748335616, "exceptions_count": 0, "files": []}}
data: {"event": "message_end", "conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a", "message_id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "created_at": 1748335614, "task_id": "4cc59763-acb2-47e3-bce4-0700c0c2753a", "id": "af0c19aa-5cc6-4002-bc5a-517d4cc83922", "metadata": {"usage": {"prompt_tokens": 0, "prompt_unit_price": "0.0", "prompt_price_unit": "0.0", "prompt_price": "0.0", "completion_tokens": 0, "completion_unit_price": "0.0", "completion_price_unit": "0.0", "completion_price": "0.0", "total_tokens": 0, "total_price": "0.0", "currency": "USD", "latency": 0.0}}, "files": []}
(1)node_started
{
"event" : "node_started",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"workflow_run_id" : "0e2f2237-05dc-4272-b536-3e97bf02691c",
"data" : {
"id" : "ed004c58-4c89-4432-ab91-29888e10f3d6",
"node_id" : "llm",
"node_type" : "llm",
"title" : "LLM",
"index" : 2,
"predecessor_node_id" : "1748333062401",
"inputs" : null,
"created_at" : 1748335614,
"extras" : {
},
"parallel_id" : null,
"parallel_start_node_id" : null,
"parent_parallel_id" : null,
"parent_parallel_start_node_id" : null,
"iteration_id" : null,
"loop_id" : null,
"parallel_run_id" : null,
"agent_strategy" : null
}
}
(2)message1
{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "这",
"from_variable_selector" : [
"llm",
"text"
]
}
(3)message2
{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "幅照片是一个特写镜头,拍摄了一只德国牧羊犬。这",
"from_variable_selector" : [
"llm",
"text"
]
}
(4)message3
{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "只狗有着传统的黑褐色毛皮,竖立着一对警惕的耳朵。它的眼睛是温暖的棕色,直视前方。它的嘴巴微微张开,露出粉红色的舌头,显示出轻微",
"from_variable_selector" : [
"llm",
"text"
]
}
(5)message4
{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "的喘息。它戴着一个金属链条项圈,上面挂着一个身份牌。背景是柔和模糊的绿色和棕色,暗示着户外环境,可能是草地或树林,阳光似乎从后",
"from_variable_selector" : [
"llm",
"text"
]
}
(6)message5
{
"event" : "message",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"answer" : "方照射,照亮了狗狗的轮廓。",
"from_variable_selector" : [
"llm",
"text"
]
}
(7)node_finished
{
"event" : "node_finished",
"conversation_id" : "30a1c40b-536d-4bf8-954f-afb27a29040a",
"message_id" : "af0c19aa-5cc6-4002-bc5a-517d4cc83922",
"created_at" : 1748335614,
"task_id" : "4cc59763-acb2-47e3-bce4-0700c0c2753a",
"workflow_run_id" : "0e2f2237-05dc-4272-b536-3e97bf02691c",
"data" : {
"id" : "ed004c58-4c89-4432-ab91-29888e10f3d6",
"node_id" : "llm",
"node_type" : "llm",
"title" : "LLM",
"index" : 2,
"predecessor_node_id" : "1748333062401",
"inputs" : null,
"process_data" : {
"model_mode" : "chat",
"prompts" : [
{
"role" : "user",
"text" : "描述图像
",
"files" : [
]
},
{
"role" : "user",
"text" : "",
"files" : [
{
"type" : "image",
"data" : "data:image...[TRUNCATED]...lFTkSuQmCC",
"detail" : "high"
}
]
}
],
"model_provider" : "langgenius/gemini/google",
"model_name" : "gemini-2.0-flash-exp"
},
"outputs" : {
"text" : "这幅照片是一个特写镜头,拍摄了一只德国牧羊犬。这只狗有着传统的黑褐色毛皮,竖立着一对警惕的耳朵。它的眼睛是温暖的棕色,直视前方。它的嘴巴微微张开,露出粉红色的舌头,显示出轻微的喘息。它戴着一个金属链条项圈,上面挂着一个身份牌。背景是柔和模糊的绿色和棕色,暗示着户外环境,可能是草地或树林,阳光似乎从后方照射,照亮了狗狗的轮廓。",
"usage" : {
"prompt_tokens" : 0,
"prompt_unit_price" : "0.0",
"prompt_price_unit" : "0.0",
"prompt_price" : "0.0",
"completion_tokens" : 0,
"completion_unit_price" : "0.0",
"completion_price_unit" : "0.0",
"completion_price" : "0.0",
"total_tokens" : 0,
"total_price" : "0.0",
"currency" : "USD",
"latency" : 0.0
},
"finish_reason" : null,
"files" : [
]
},
"status" : "succeeded",
"error" : null,
"elapsed_time" : 1.894678,
"execution_metadata" : {
"total_tokens" : 0,
"total_price" : "0.0",
"currency" : "USD"
},
"created_at" : 1748335614,
"finished_at" : 1748335616,
"files" : [
],
"parallel_id" : null,
"parallel_start_node_id" : null,
"parent_parallel_id" : null,
"parent_parallel_start_node_id" : null,
"iteration_id" : null,
"loop_id" : null
}
}
二.开始节点
1.输入
{
"sys.query": "描述图像",
"sys.files": [
{
"dify_model_identity": "__dify__file__",
"id": null,
"tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae",
"type": "image",
"transfer_method": "local_file",
"remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=",
"related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1",
"filename": "dog.png",
"extension": ".png",
"mime_type": "image/png",
"size": 368313,
"url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=67dfea243007a69609a2b16e6b14bbcd&sign=AA4HmgJKtcdPkztPsadtLzr8LypyaNUAeWmEiuqpugo="
}
],
"sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a",
"sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377",
"sys.dialogue_count": 0,
"sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d",
"sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46",
"sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"
}
2.输出
{
"sys.query": "描述图像",
"sys.files": [
{
"dify_model_identity": "__dify__file__",
"id": null,
"tenant_id": "796610e5-ff95-42ed-8f0e-f5d3fa8c40ae",
"type": "image",
"transfer_method": "local_file",
"remote_url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335608&nonce=c3a2116a1019aeceeab65bae841da65a&sign=4nJFZvggRLT73fGLnWskMOlMe3qQbe374LbZwrSwrSM=",
"related_id": "3e063508-298c-4188-ad53-1b7cdb9421b1",
"filename": "dog.png",
"extension": ".png",
"mime_type": "image/png",
"size": 368313,
"url": "https://upload.dify.ai/files/3e063508-298c-4188-ad53-1b7cdb9421b1/file-preview?timestamp=1748335614&nonce=a83024f2e70dd080937d147963747f64&sign=9bhuGRAnH4YKORSG5l8I9FWhtKaJqiE9gUJ-vYBxwa0="
}
],
"sys.conversation_id": "30a1c40b-536d-4bf8-954f-afb27a29040a",
"sys.user_id": "7d8864c3-c456-4588-9b0a-9368c94ca377",
"sys.dialogue_count": 0,
"sys.app_id": "22d976e3-4842-4769-a5b0-73ca84894e8d",
"sys.workflow_id": "c600204e-1bdd-4f28-86db-e46777890c46",
"sys.workflow_run_id": "0e2f2237-05dc-4272-b536-3e97bf02691c"
}
三.LLM节点
1.数据处理
{
"model_mode": "chat",
"prompts": [
{
"role": "user",
"text": "描述图像n",
"files": []
},
{
"role": "user",
"text": "",
"files": [
{
"type": "image",
"data": "data:image...[TRUNCATED]...lFTkSuQmCC",
"detail": "high"
}
]
}
],
"model_provider": "langgenius/gemini/google",
"model_name": "gemini-2.0-flash-exp"
}
2.输出
{
"text": "这幅照片是一个特写镜头,拍摄了一只德国牧羊犬。这只狗有着传统的黑褐色毛皮,竖立着一对警惕的耳朵。它的眼睛是温暖的棕色,直视前方。它的嘴巴微微张开,露出粉红色的舌头,显示出轻微的喘息。它戴着一个金属链条项圈,上面挂着一个身份牌。背景是柔和模糊的绿色和棕色,暗示着户外环境,可能是草地或树林,阳光似乎从后方照射,照亮了狗狗的轮廓。",
"usage": {
"prompt_tokens": 0,
"prompt_unit_price": "0.0",
"prompt_price_unit": "0.0",
"prompt_price": "0.0",
"completion_tokens": 0,
"completion_unit_price": "0.0",
"completion_price_unit": "0.0",
"completion_price": "0.0",
"total_tokens": 0,
"total_price": "0.0",
"currency": "USD",
"latency": 0
},
"finish_reason": null,
"files": []
}
四.直接回复节点
1.输出
{
"answer": "这幅照片是一个特写镜头,拍摄了一只德国牧羊犬。这只狗有着传统的黑褐色毛皮,竖立着一对警惕的耳朵。它的眼睛是温暖的棕色,直视前方。它的嘴巴微微张开,露出粉红色的舌头,显示出轻微的喘息。它戴着一个金属链条项圈,上面挂着一个身份牌。背景是柔和模糊的绿色和棕色,暗示着户外环境,可能是草地或树林,阳光似乎从后方照射,照亮了狗狗的轮廓。",
"files": []
}
五.LLM节点-数据处理
这个数据是LLM节点处理过程中生成的process_data
对象,描述了一个多模态AI模型调用的请求配置。它是这样得到的:
1.模型配置信息
-
model_mode
: “chat”表示使用聊天模式,这来自model_config.mode
-
model_provider
: “langgenius/gemini/google”表示使用Google的Gemini模型 -
model_name
: “gemini-2.0-flash-exp”是具体使用的模型版本
2.提示消息构建
通过_fetch_prompt_messages
方法生成,包含两个用户消息:(1)第一个消息包含文本提示”描述图像”(2)第二个消息包含图像文件,使用base64编码存储
3.图像处理
-
图像通过
_fetch_files
方法获取 -
detail: "high"
表示要求模型进行高分辨率的图像分析,对应代码中的vision_detail_config
参数
4.数据流程
-
当工作流执行到LLM节点时,系统提取用户配置的提示和上传的图像
-
组装成规范化的请求格式
-
记录在
process_data
中,用于后续调用AI模型和记录执行过程
这个数据随后会被传递给_invoke_llm
方法,向Gemini模型发送请求来获取对图像的描述结果。
参考文献
[1] https://github.com/langgenius/dify/releases/tag/1.4.0
[2] https://ai.google.dev/
[3] https://aistudio.google.com/
[4] Dify v1.4.0中的Multi-Modal LLM Output:基本操作和原理:https://z0yrmerhgi8.feishu.cn/wiki/C3enwZFXci7SS0kKGhQcgv9Mnnh
知识星球服务内容:Dify源码剖析及答疑,Dify对话系统源码,NLP电子书籍报告下载,公众号所有付费资料。加微信buxingtianxia21进NLP工程化资料群。