NeoAnd/evolution-api
1
0
Fork 0
mirror of https://github.com/EvolutionAPI/evolution-api.git synced 2025-07-14 01:41:24 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: openai now identifies images

Browse Source
This commit is contained in:
Davidson Gomes 2024-08-15 12:29:08 -03:00
parent 061696e140
commit 785a72cbc8
3 changed files with 121 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -4,6 +4,7 @@
* Added ignoreJids in chatwoot settings
* Dify now identifies images
* Openai now identifies images
### Fixed
@ -16,6 +17,7 @@
* Deprecate buttons and list in new Baileys version
* Changed labels to be unique on the same instance
* Remove instance from redis even if using database
* Unified integration session system so they don't overlap
# 2.0.9-rc (2024-08-09 18:00)

16
src/api/integrations/dify/services/dify.service.ts
View File

@ -1101,7 +1101,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.query = contentSplit[2];
payload.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);
@ -1169,7 +1169,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.inputs.query = contentSplit[2];
payload.inputs.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);
@ -1237,7 +1237,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.query = contentSplit[2];
payload.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);
@ -1329,7 +1329,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.inputs.query = contentSplit[2];
payload.inputs.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);
@ -1506,7 +1506,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.query = contentSplit[2];
payload.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);
@ -1574,7 +1574,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.inputs.query = contentSplit[2];
payload.inputs.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);
@ -1642,7 +1642,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.query = contentSplit[2];
payload.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);
@ -1741,7 +1741,7 @@ export class DifyService {
url: contentSplit[1].split('?')[0],
},
];
payload.inputs.query = contentSplit[2];
payload.inputs.query = contentSplit[2] || content;
}
await instance.client.presenceSubscribe(remoteJid);

140
src/api/integrations/openai/services/openai.service.ts
View File

@ -874,11 +874,27 @@ export class OpenaiService {
: msg?.message?.audioMessage
? `audioMessage|${mediaId}`
: undefined,
imageMessage: msg?.message?.imageMessage ? `imageMessage|${mediaId}` : undefined,
videoMessage: msg?.message?.videoMessage ? `videoMessage|${mediaId}` : undefined,
documentMessage: msg?.message?.documentMessage ? `documentMessage|${mediaId}` : undefined,
documentWithCaptionMessage: msg?.message?.auddocumentWithCaptionMessageioMessage
? `documentWithCaptionMessage|${mediaId}`
imageMessage: msg?.message?.imageMessage
? `imageMessage|${mediaId}${
msg?.message?.imageMessage?.caption ? `|${msg?.message?.imageMessage?.caption}` : ''
}`
: undefined,
videoMessage: msg?.message?.videoMessage
? `videoMessage|${mediaId}${
msg?.message?.videoMessage?.caption ? `|${msg?.message?.videoMessage?.caption}` : ''
}`
: undefined,
documentMessage: msg?.message?.documentMessage
? `documentMessage|${mediaId}${
msg?.message?.documentMessage?.caption ? `|${msg?.message?.documentMessage?.caption}` : ''
}`
: undefined,
documentWithCaptionMessage: msg?.message?.documentWithCaptionMessage?.message?.documentMessage
? `documentWithCaptionMessage|${mediaId}${
msg?.message?.documentWithCaptionMessage?.message?.documentMessage?.caption
? `|${msg?.message?.documentWithCaptionMessage?.message?.documentMessage?.caption}`
: ''
}`
: undefined,
};
@ -1303,10 +1319,28 @@ export class OpenaiService {
session = data.session;
}
await this.client.beta.threads.messages.create(data.session.sessionId, {
const messageData: any = {
role: 'user',
content,
});
content: [{ type: 'text', text: content }],
};
if (this.isImageMessage(content)) {
const contentSplit = content.split('|');
const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
await this.client.beta.threads.messages.create(data.session.sessionId, messageData);
const runAssistant = await this.client.beta.threads.runs.create(data.session.sessionId, {
assistant_id: openaiBot.assistantId,
@ -1420,6 +1454,10 @@ export class OpenaiService {
}
}
private isImageMessage(content: string) {
return content.includes('imageMessage');
}
private async processOpenaiAssistant(
instance: any,
remoteJid: string,
@ -1531,10 +1569,28 @@ export class OpenaiService {
const threadId = session.sessionId;
await this.client.beta.threads.messages.create(threadId, {
const messageData: any = {
role: 'user',
content,
});
content: [{ type: 'text', text: content }],
};
if (this.isImageMessage(content)) {
const contentSplit = content.split('|');
const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
await this.client.beta.threads.messages.create(threadId, messageData);
const runAssistant = await this.client.beta.threads.runs.create(threadId, {
assistant_id: openaiBot.assistantId,
@ -1654,15 +1710,28 @@ export class OpenaiService {
};
});
const messages: any[] = [
...messagesSystem,
...messagesAssistant,
...messagesUser,
{
role: 'user',
content: content,
},
];
const messageData: any = {
role: 'user',
content: [{ type: 'text', text: content }],
};
if (this.isImageMessage(content)) {
const contentSplit = content.split('|');
const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
const messages: any[] = [...messagesSystem, ...messagesAssistant, ...messagesUser, messageData];
await instance.client.presenceSubscribe(remoteJid);
@ -1838,15 +1907,28 @@ export class OpenaiService {
};
});
const messages: any[] = [
...messagesSystem,
...messagesAssistant,
...messagesUser,
{
role: 'user',
content: content,
},
];
const messageData: any = {
role: 'user',
content: [{ type: 'text', text: content }],
};
if (this.isImageMessage(content)) {
const contentSplit = content.split('|');
const url = contentSplit[1].split('?')[0];
messageData.content = [
{ type: 'text', text: contentSplit[2] || content },
{
type: 'image_url',
image_url: {
url: url,
},
},
];
}
const messages: any[] = [...messagesSystem, ...messagesAssistant, ...messagesUser, messageData];
await instance.client.presenceSubscribe(remoteJid);