NeoAnd/evolution-api
1
0
Fork 0
mirror of https://github.com/EvolutionAPI/evolution-api.git synced 2025-07-15 11:42:53 -06:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: openai now identifies images

Browse Source
This commit is contained in:
Davidson Gomes 2024-08-15 12:29:08 -03:00
parent 061696e140
commit 785a72cbc8
3 changed files with 121 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -4,6 +4,7 @@
* Added ignoreJids in chatwoot settings * Added ignoreJids in chatwoot settings
* Dify now identifies images * Dify now identifies images
* Openai now identifies images
### Fixed ### Fixed
@ -16,6 +17,7 @@
* Deprecate buttons and list in new Baileys version * Deprecate buttons and list in new Baileys version
* Changed labels to be unique on the same instance * Changed labels to be unique on the same instance
* Remove instance from redis even if using database * Remove instance from redis even if using database
* Unified integration session system so they don't overlap
# 2.0.9-rc (2024-08-09 18:00) # 2.0.9-rc (2024-08-09 18:00)

16
src/api/integrations/dify/services/dify.service.ts
View File

@ -1101,7 +1101,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.query = contentSplit[2]; payload.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1169,7 +1169,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.inputs.query = contentSplit[2]; payload.inputs.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1237,7 +1237,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.query = contentSplit[2]; payload.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1329,7 +1329,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.inputs.query = contentSplit[2]; payload.inputs.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1506,7 +1506,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.query = contentSplit[2]; payload.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1574,7 +1574,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.inputs.query = contentSplit[2]; payload.inputs.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1642,7 +1642,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.query = contentSplit[2]; payload.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1741,7 +1741,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0], url: contentSplit[1].split('?')[0],
}, },
]; ];
payload.inputs.query = contentSplit[2]; payload.inputs.query = contentSplit[2] || content;
} }
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);

140
src/api/integrations/openai/services/openai.service.ts
View File

@ -874,11 +874,27 @@ export class OpenaiService {
: msg?.message?.audioMessage : msg?.message?.audioMessage
? `audioMessage|${mediaId}` ? `audioMessage|${mediaId}`
: undefined, : undefined,
imageMessage: msg?.message?.imageMessage ? `imageMessage|${mediaId}` : undefined, imageMessage: msg?.message?.imageMessage
videoMessage: msg?.message?.videoMessage ? `videoMessage|${mediaId}` : undefined, ? `imageMessage|${mediaId}${
documentMessage: msg?.message?.documentMessage ? `documentMessage|${mediaId}` : undefined, msg?.message?.imageMessage?.caption ? `|${msg?.message?.imageMessage?.caption}` : ''
documentWithCaptionMessage: msg?.message?.auddocumentWithCaptionMessageioMessage }`
? `documentWithCaptionMessage|${mediaId}` : undefined,
videoMessage: msg?.message?.videoMessage
? `videoMessage|${mediaId}${
msg?.message?.videoMessage?.caption ? `|${msg?.message?.videoMessage?.caption}` : ''
}`
: undefined,
documentMessage: msg?.message?.documentMessage
? `documentMessage|${mediaId}${
msg?.message?.documentMessage?.caption ? `|${msg?.message?.documentMessage?.caption}` : ''
}`
: undefined,
documentWithCaptionMessage: msg?.message?.documentWithCaptionMessage?.message?.documentMessage
? `documentWithCaptionMessage|${mediaId}${
msg?.message?.documentWithCaptionMessage?.message?.documentMessage?.caption
? `|${msg?.message?.documentWithCaptionMessage?.message?.documentMessage?.caption}`
: ''
}`
: undefined, : undefined,
}; };
@ -1303,10 +1319,28 @@ export class OpenaiService {
session = data.session; session = data.session;
} }
await this.client.beta.threads.messages.create(data.session.sessionId, { const messageData: any = {
role: 'user', role: 'user',
content, content: [{ type: 'text', text: content }],
}); };
if (this.isImageMessage(content)) {
const contentSplit = content.split('|');
const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
await this.client.beta.threads.messages.create(data.session.sessionId, messageData);
const runAssistant = await this.client.beta.threads.runs.create(data.session.sessionId, { const runAssistant = await this.client.beta.threads.runs.create(data.session.sessionId, {
assistant_id: openaiBot.assistantId, assistant_id: openaiBot.assistantId,
@ -1420,6 +1454,10 @@ export class OpenaiService {
} }
} }
private isImageMessage(content: string) {
return content.includes('imageMessage');
}
private async processOpenaiAssistant( private async processOpenaiAssistant(
instance: any, instance: any,
remoteJid: string, remoteJid: string,
@ -1531,10 +1569,28 @@ export class OpenaiService {
const threadId = session.sessionId; const threadId = session.sessionId;
await this.client.beta.threads.messages.create(threadId, { const messageData: any = {
role: 'user', role: 'user',
content, content: [{ type: 'text', text: content }],
}); };
if (this.isImageMessage(content)) {
const contentSplit = content.split('|');
const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
await this.client.beta.threads.messages.create(threadId, messageData);
const runAssistant = await this.client.beta.threads.runs.create(threadId, { const runAssistant = await this.client.beta.threads.runs.create(threadId, {
assistant_id: openaiBot.assistantId, assistant_id: openaiBot.assistantId,
@ -1654,15 +1710,28 @@ export class OpenaiService {
}; };
}); });
const messages: any[] = [ const messageData: any = {
...messagesSystem, role: 'user',
...messagesAssistant, content: [{ type: 'text', text: content }],
...messagesUser, };
{
role: 'user', if (this.isImageMessage(content)) {
content: content, const contentSplit = content.split('|');
},
]; const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
const messages: any[] = [...messagesSystem, ...messagesAssistant, ...messagesUser, messageData];
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);
@ -1838,15 +1907,28 @@ export class OpenaiService {
}; };
}); });
const messages: any[] = [ const messageData: any = {
...messagesSystem, role: 'user',
...messagesAssistant, content: [{ type: 'text', text: content }],
...messagesUser, };
{
role: 'user', if (this.isImageMessage(content)) {
content: content, const contentSplit = content.split('|');
},
]; const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
const messages: any[] = [...messagesSystem, ...messagesAssistant, ...messagesUser, messageData];
await instance.client.presenceSubscribe(remoteJid); await instance.client.presenceSubscribe(remoteJid);