From 407d254cf7031e7dd5c7788c619e23f56e591fc2 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Thu, 18 Sep 2025 17:46:47 -0300 Subject: [PATCH] refactor(chatbot): streamline media message handling across chatbot services - Removed redundant instance name references in EvolutionStartupService to enhance data consistency. - Updated media message processing in various chatbot services to utilize base64 and mediaUrl more effectively, ensuring better handling of image messages. - Improved overall code readability and maintainability by simplifying media handling logic. --- .../evolution/evolution.channel.service.ts | 7 -- .../chatbot/chatbot.controller.ts | 14 +-- .../chatbot/dify/services/dify.service.ts | 106 +++++++++++++----- .../chatbot/evoai/services/evoai.service.ts | 49 +++++--- .../services/evolutionBot.service.ts | 29 +++-- .../flowise/services/flowise.service.ts | 32 ++++-- .../chatbot/openai/services/openai.service.ts | 74 ++++++++---- 7 files changed, 215 insertions(+), 96 deletions(-) diff --git a/src/api/integrations/channel/evolution/evolution.channel.service.ts b/src/api/integrations/channel/evolution/evolution.channel.service.ts index b5e12ca3..be88421a 100644 --- a/src/api/integrations/channel/evolution/evolution.channel.service.ts +++ b/src/api/integrations/channel/evolution/evolution.channel.service.ts @@ -331,7 +331,6 @@ export class EvolutionStartupService extends ChannelStartupService { messageTimestamp: Math.round(new Date().getTime() / 1000), webhookUrl, source: 'unknown', - instanceName: this.instance.name, instanceId: this.instanceId, }; } else if (message?.mediaType === 'video') { @@ -346,7 +345,6 @@ export class EvolutionStartupService extends ChannelStartupService { messageTimestamp: Math.round(new Date().getTime() / 1000), webhookUrl, source: 'unknown', - instanceName: this.instance.name, instanceId: this.instanceId, }; } else if (message?.mediaType === 'audio') { @@ -361,7 +359,6 @@ export class EvolutionStartupService extends ChannelStartupService { messageTimestamp: Math.round(new Date().getTime() / 1000), webhookUrl, source: 'unknown', - instanceName: this.instance.name, instanceId: this.instanceId, }; @@ -383,7 +380,6 @@ export class EvolutionStartupService extends ChannelStartupService { messageTimestamp: Math.round(new Date().getTime() / 1000), webhookUrl, source: 'unknown', - instanceName: this.instance.name, instanceId: this.instanceId, }; } else if (message.buttonMessage) { @@ -400,7 +396,6 @@ export class EvolutionStartupService extends ChannelStartupService { messageTimestamp: Math.round(new Date().getTime() / 1000), webhookUrl, source: 'unknown', - instanceName: this.instance.name, instanceId: this.instanceId, }; } else if (message.listMessage) { @@ -414,7 +409,6 @@ export class EvolutionStartupService extends ChannelStartupService { messageTimestamp: Math.round(new Date().getTime() / 1000), webhookUrl, source: 'unknown', - instanceName: this.instance.name, instanceId: this.instanceId, }; } else { @@ -428,7 +422,6 @@ export class EvolutionStartupService extends ChannelStartupService { messageTimestamp: Math.round(new Date().getTime() / 1000), webhookUrl, source: 'unknown', - instanceName: this.instance.name, instanceId: this.instanceId, }; } diff --git a/src/api/integrations/chatbot/chatbot.controller.ts b/src/api/integrations/chatbot/chatbot.controller.ts index a2312f6e..74f1427a 100644 --- a/src/api/integrations/chatbot/chatbot.controller.ts +++ b/src/api/integrations/chatbot/chatbot.controller.ts @@ -91,19 +91,19 @@ export class ChatbotController { pushName, isIntegration, }; - await evolutionBotController.emit(emitData); + evolutionBotController.emit(emitData); - await typebotController.emit(emitData); + typebotController.emit(emitData); - await openaiController.emit(emitData); + openaiController.emit(emitData); - await difyController.emit(emitData); + difyController.emit(emitData); - await n8nController.emit(emitData); + n8nController.emit(emitData); - await evoaiController.emit(emitData); + evoaiController.emit(emitData); - await flowiseController.emit(emitData); + flowiseController.emit(emitData); } public processDebounce( diff --git a/src/api/integrations/chatbot/dify/services/dify.service.ts b/src/api/integrations/chatbot/dify/services/dify.service.ts index 773efe49..a0c2283e 100644 --- a/src/api/integrations/chatbot/dify/services/dify.service.ts +++ b/src/api/integrations/chatbot/dify/services/dify.service.ts @@ -4,6 +4,7 @@ import { Integration } from '@api/types/wa.types'; import { ConfigService, HttpServer } from '@config/env.config'; import { Dify, DifySetting, IntegrationSession } from '@prisma/client'; import axios from 'axios'; +import { isURL } from 'class-validator'; import { BaseChatbotService } from '../../base-chatbot.service'; import { OpenaiService } from '../../openai/services/openai.service'; @@ -78,15 +79,35 @@ export class DifyService extends BaseChatbotService { // Handle image messages if (this.isImageMessage(content)) { - const contentSplit = content.split('|'); - payload.files = [ - { - type: 'image', - transfer_method: 'remote_url', - url: contentSplit[1].split('?')[0], - }, - ]; - payload.query = contentSplit[2] || content; + const media = content.split('|'); + + if (msg.message.mediaUrl || msg.message.base64) { + let mediaBase64 = msg.message.base64 || null; + + if (msg.message.mediaUrl && isURL(msg.message.mediaUrl)) { + const result = await axios.get(msg.message.mediaUrl, { responseType: 'arraybuffer' }); + mediaBase64 = Buffer.from(result.data).toString('base64'); + } + + if (mediaBase64) { + payload.files = [ + { + type: 'image', + transfer_method: 'remote_url', + url: mediaBase64, + }, + ]; + } + } else { + payload.files = [ + { + type: 'image', + transfer_method: 'remote_url', + url: media[1].split('?')[0], + }, + ]; + } + payload.query = media[2] || content; } if (instance.integration === Integration.WHATSAPP_BAILEYS) { @@ -140,15 +161,35 @@ export class DifyService extends BaseChatbotService { // Handle image messages if (this.isImageMessage(content)) { - const contentSplit = content.split('|'); - payload.files = [ - { - type: 'image', - transfer_method: 'remote_url', - url: contentSplit[1].split('?')[0], - }, - ]; - payload.inputs.query = contentSplit[2] || content; + const media = content.split('|'); + + if (msg.message.mediaUrl || msg.message.base64) { + let mediaBase64 = msg.message.base64 || null; + + if (msg.message.mediaUrl && isURL(msg.message.mediaUrl)) { + const result = await axios.get(msg.message.mediaUrl, { responseType: 'arraybuffer' }); + mediaBase64 = Buffer.from(result.data).toString('base64'); + } + + if (mediaBase64) { + payload.files = [ + { + type: 'image', + transfer_method: 'remote_url', + url: mediaBase64, + }, + ]; + } + } else { + payload.files = [ + { + type: 'image', + transfer_method: 'remote_url', + url: media[1].split('?')[0], + }, + ]; + payload.inputs.query = media[2] || content; + } } if (instance.integration === Integration.WHATSAPP_BAILEYS) { @@ -202,15 +243,26 @@ export class DifyService extends BaseChatbotService { // Handle image messages if (this.isImageMessage(content)) { - const contentSplit = content.split('|'); - payload.files = [ - { - type: 'image', - transfer_method: 'remote_url', - url: contentSplit[1].split('?')[0], - }, - ]; - payload.query = contentSplit[2] || content; + const media = content.split('|'); + + if (msg.message.mediaUrl || msg.message.base64) { + payload.files = [ + { + type: 'image', + transfer_method: 'remote_url', + url: msg.message.mediaUrl || msg.message.base64, + }, + ]; + } else { + payload.files = [ + { + type: 'image', + transfer_method: 'remote_url', + url: media[1].split('?')[0], + }, + ]; + payload.query = media[2] || content; + } } if (instance.integration === Integration.WHATSAPP_BAILEYS) { diff --git a/src/api/integrations/chatbot/evoai/services/evoai.service.ts b/src/api/integrations/chatbot/evoai/services/evoai.service.ts index 173ebe34..4d04b85c 100644 --- a/src/api/integrations/chatbot/evoai/services/evoai.service.ts +++ b/src/api/integrations/chatbot/evoai/services/evoai.service.ts @@ -5,6 +5,7 @@ import { ConfigService, HttpServer } from '@config/env.config'; import { Evoai, EvoaiSetting, IntegrationSession } from '@prisma/client'; import axios from 'axios'; import { downloadMediaMessage } from 'baileys'; +import { isURL } from 'class-validator'; import { v4 as uuidv4 } from 'uuid'; import { BaseChatbotService } from '../../base-chatbot.service'; @@ -82,23 +83,43 @@ export class EvoaiService extends BaseChatbotService { // Handle image message if present if (this.isImageMessage(content) && msg) { - const contentSplit = content.split('|'); - parts[0].text = contentSplit[2] || content; + const media = content.split('|'); + parts[0].text = media[2] || content; try { - // Download the image - const mediaBuffer = await downloadMediaMessage(msg, 'buffer', {}); - const fileContent = Buffer.from(mediaBuffer).toString('base64'); - const fileName = contentSplit[2] || `${msg.key?.id || 'image'}.jpg`; + if (msg.message.mediaUrl || msg.message.base64) { + let mediaBase64 = msg.message.base64 || null; - parts.push({ - type: 'file', - file: { - name: fileName, - mimeType: 'image/jpeg', - bytes: fileContent, - }, - } as any); + if (msg.message.mediaUrl && isURL(msg.message.mediaUrl)) { + const result = await axios.get(msg.message.mediaUrl, { responseType: 'arraybuffer' }); + mediaBase64 = Buffer.from(result.data).toString('base64'); + } + + if (mediaBase64) { + parts.push({ + type: 'file', + file: { + name: msg.key.id + '.jpeg', + mimeType: 'image/jpeg', + bytes: mediaBase64, + }, + } as any); + } + } else { + // Download the image + const mediaBuffer = await downloadMediaMessage(msg, 'buffer', {}); + const fileContent = Buffer.from(mediaBuffer).toString('base64'); + const fileName = media[2] || `${msg.key?.id || 'image'}.jpg`; + + parts.push({ + type: 'file', + file: { + name: fileName, + mimeType: 'image/jpeg', + bytes: fileContent, + }, + } as any); + } } catch (fileErr) { this.logger.error(`[EvoAI] Failed to process image: ${fileErr}`); } diff --git a/src/api/integrations/chatbot/evolutionBot/services/evolutionBot.service.ts b/src/api/integrations/chatbot/evolutionBot/services/evolutionBot.service.ts index b82e8fe1..2a0c547c 100644 --- a/src/api/integrations/chatbot/evolutionBot/services/evolutionBot.service.ts +++ b/src/api/integrations/chatbot/evolutionBot/services/evolutionBot.service.ts @@ -6,6 +6,7 @@ import { ConfigService, HttpServer } from '@config/env.config'; import { EvolutionBot, EvolutionBotSetting, IntegrationSession } from '@prisma/client'; import { sendTelemetry } from '@utils/sendTelemetry'; import axios from 'axios'; +import { isURL } from 'class-validator'; import { BaseChatbotService } from '../../base-chatbot.service'; import { OpenaiService } from '../../openai/services/openai.service'; @@ -71,16 +72,26 @@ export class EvolutionBotService extends BaseChatbotService { } if (this.isImageMessage(content)) { - const contentSplit = content.split('|'); + const media = content.split('|'); - payload.uploads = [ - { - data: contentSplit[1].split('?')[0], - type: 'url', - name: 'Flowise.png', - mime: 'image/png', - }, - ]; - payload.question = contentSplit[2] || content; + if (msg.message.mediaUrl || msg.message.base64) { + payload.uploads = [ + { + data: msg.message.base64 || msg.message.mediaUrl, + type: 'url', + name: 'Flowise.png', + mime: 'image/png', + }, + ]; + } else { + payload.uploads = [ + { + data: media[1].split('?')[0], + type: 'url', + name: 'Flowise.png', + mime: 'image/png', + }, + ]; + payload.question = media[2] || content; + } } if (instance.integration === Integration.WHATSAPP_BAILEYS) { diff --git a/src/api/integrations/chatbot/openai/services/openai.service.ts b/src/api/integrations/chatbot/openai/services/openai.service.ts index dd00b04c..036f8259 100644 --- a/src/api/integrations/chatbot/openai/services/openai.service.ts +++ b/src/api/integrations/chatbot/openai/services/openai.service.ts @@ -6,6 +6,7 @@ import { IntegrationSession, OpenaiBot, OpenaiSetting } from '@prisma/client'; import { sendTelemetry } from '@utils/sendTelemetry'; import axios from 'axios'; import { downloadMediaMessage } from 'baileys'; +import { isURL } from 'class-validator'; import FormData from 'form-data'; import OpenAI from 'openai'; import P from 'pino'; @@ -173,7 +174,7 @@ export class OpenaiService extends BaseChatbotService } // Process with the appropriate API based on bot type - await this.sendMessageToBot(instance, session, settings, openaiBot, remoteJid, pushName || '', content); + await this.sendMessageToBot(instance, session, settings, openaiBot, remoteJid, pushName || '', content, msg); } catch (error) { this.logger.error(`Error in process: ${error.message || JSON.stringify(error)}`); return; @@ -191,6 +192,7 @@ export class OpenaiService extends BaseChatbotService remoteJid: string, pushName: string, content: string, + msg?: any, ): Promise { this.logger.log(`Sending message to bot for remoteJid: ${remoteJid}, bot type: ${openaiBot.botType}`); @@ -222,10 +224,11 @@ export class OpenaiService extends BaseChatbotService pushName, false, // Not fromMe content, + msg, ); } else { this.logger.log('Processing with ChatCompletion API'); - message = await this.processChatCompletionMessage(instance, openaiBot, remoteJid, content); + message = await this.processChatCompletionMessage(instance, openaiBot, remoteJid, content, msg); } this.logger.log(`Got response from OpenAI: ${message?.substring(0, 50)}${message?.length > 50 ? '...' : ''}`); @@ -268,6 +271,7 @@ export class OpenaiService extends BaseChatbotService pushName: string, fromMe: boolean, content: string, + msg?: any, ): Promise { const messageData: any = { role: fromMe ? 'assistant' : 'user', @@ -276,18 +280,35 @@ export class OpenaiService extends BaseChatbotService // Handle image messages if (this.isImageMessage(content)) { - const contentSplit = content.split('|'); - const url = contentSplit[1].split('?')[0]; + const media = content.split('|'); - messageData.content = [ - { type: 'text', text: contentSplit[2] || content }, - { - type: 'image_url', - image_url: { - url: url, + if (msg.message.mediaUrl || msg.message.base64) { + let mediaBase64 = msg.message.base64 || null; + + if (msg.message.mediaUrl && isURL(msg.message.mediaUrl)) { + const result = await axios.get(msg.message.mediaUrl, { responseType: 'arraybuffer' }); + mediaBase64 = Buffer.from(result.data).toString('base64'); + } + + if (mediaBase64) { + messageData.content = [ + { type: 'text', text: media[2] || content }, + { type: 'image_url', image_url: { url: mediaBase64 } }, + ]; + } + } else { + const url = media[1].split('?')[0]; + + messageData.content = [ + { type: 'text', text: media[2] || content }, + { + type: 'image_url', + image_url: { + url: url, + }, }, - }, - ]; + ]; + } } // Get thread ID from session or create new thread @@ -376,6 +397,7 @@ export class OpenaiService extends BaseChatbotService openaiBot: OpenaiBot, remoteJid: string, content: string, + msg?: any, ): Promise { this.logger.log('Starting processChatCompletionMessage'); @@ -468,18 +490,26 @@ export class OpenaiService extends BaseChatbotService // Handle image messages if (this.isImageMessage(content)) { this.logger.log('Found image message'); - const contentSplit = content.split('|'); - const url = contentSplit[1].split('?')[0]; + const media = content.split('|'); - messageData.content = [ - { type: 'text', text: contentSplit[2] || content }, - { - type: 'image_url', - image_url: { - url: url, + if (msg.message.mediaUrl || msg.message.base64) { + messageData.content = [ + { type: 'text', text: media[2] || content }, + { type: 'image_url', image_url: { url: msg.message.base64 || msg.message.mediaUrl } }, + ]; + } else { + const url = media[1].split('?')[0]; + + messageData.content = [ + { type: 'text', text: media[2] || content }, + { + type: 'image_url', + image_url: { + url: url, + }, }, - }, - ]; + ]; + } } // Combine all messages: system messages, pre-defined messages, conversation history, and current message