Merge pull request #395 from jaison-x/import-messages-chatwoot

feat(chatwoot): import history messages to chatwoot on whatsapp connection

package.json

@@ -76,6 +76,7 @@
     "node-mime-types": "^1.1.0",
     "node-windows": "^1.0.0-beta.8",
     "parse-bmfont-xml": "^1.1.4",
+    "pg": "^8.11.3",
     "pino": "^8.11.0",
     "qrcode": "^1.5.1",
     "qrcode-terminal": "^0.12.0",
@@ -112,4 +113,4 @@
     "ts-node-dev": "^2.0.0",
     "typescript": "^4.9.5"
   }
-}
+}

src/config/env.config.ts

@@ -149,7 +149,18 @@ export type Webhook = { GLOBAL?: GlobalWebhook; EVENTS: EventsWebhook };
 export type ConfigSessionPhone = { CLIENT: string; NAME: string };
 export type QrCode = { LIMIT: number; COLOR: string };
 export type Typebot = { API_VERSION: string; KEEP_OPEN: boolean };
-export type ChatWoot = { MESSAGE_DELETE: boolean };
+export type Chatwoot = {
+  MESSAGE_DELETE: boolean;
+  IMPORT: {
+    DATABASE: {
+      CONNECTION: {
+        URI: string;
+      };
+    };
+    PLACEHOLDER_MEDIA_MESSAGE: boolean;
+  };
+};
+
 export type CacheConf = { REDIS: CacheConfRedis; LOCAL: CacheConfLocal };
 export type Production = boolean;
 
@@ -171,7 +182,7 @@ export interface Env {
   CONFIG_SESSION_PHONE: ConfigSessionPhone;
   QRCODE: QrCode;
   TYPEBOT: Typebot;
-  CHATWOOT: ChatWoot;
+  CHATWOOT: Chatwoot;
   CACHE: CacheConf;
   AUTHENTICATION: Auth;
   PRODUCTION?: Production;
@@ -338,6 +349,14 @@ export class ConfigService {
       },
       CHATWOOT: {
         MESSAGE_DELETE: process.env.CHATWOOT_MESSAGE_DELETE === 'false',
+        IMPORT: {
+          DATABASE: {
+            CONNECTION: {
+              URI: process.env.CHATWOOT_DATABASE_CONNECTION_URI || '',
+            },
+          },
+          PLACEHOLDER_MEDIA_MESSAGE: process.env?.CHATWOOT_IMPORT_PLACEHOLDER_MEDIA_MESSAGE === 'true',
+        },
       },
       CACHE: {
         REDIS: {

src/dev-env.yml

@@ -153,10 +153,16 @@ TYPEBOT:
   API_VERSION: 'old' # old | latest
   KEEP_OPEN: false
 
-# If you leave this option as false, when deleting the message for everyone on WhatsApp, it will not be deleted on Chatwoot.
 CHATWOOT:
+  # If you leave this option as false, when deleting the message for everyone on WhatsApp, it will not be deleted on Chatwoot.
   MESSAGE_DELETE: true # false | true 
- 
+  IMPORT:
+    # This db connection is used to import messages from whatsapp to chatwoot database
+    DATABASE:
+      CONNECTION:
+        URI: "postgres://user:password@hostname:port/dbname"
+    PLACEHOLDER_MEDIA_MESSAGE: true
+
 # Cache to optimize application performance
 CACHE:
   REDIS:

src/docs/swagger.yaml

@@ -2076,6 +2076,9 @@ paths:
                 read_status:
                   type: boolean
                   description: "Indicates whether to mark status messages as read."
+                sync_full_history:
+                  type: boolean
+                  description: "Indicates whether to request a full history messages sync on connect."
       parameters:
         - name: instanceName
           in: path
@@ -2141,6 +2144,15 @@ paths:
                 conversation_pending:
                   type: boolean
                   description: "Indicates whether to mark conversations as pending."
+                import_contacts:
+                  type: boolean
+                  description: "Indicates whether to import contacts from phone to Chatwoot when connecting."
+                import_messages:
+                  type: boolean
+                  description: "Indicates whether to import messages from phone to Chatwoot when connecting."
+                days_limit_import_messages:
+                  type: number
+                  description: "Indicates number of days to limit messages imported to Chatwoot."
       parameters:
         - name: instanceName
           in: path

src/libs/postgres.client.ts

@@ -0,0 +1,49 @@
+import postgresql from 'pg';
+
+import { Chatwoot, configService } from '../config/env.config';
+import { Logger } from '../config/logger.config';
+
+const { Pool } = postgresql;
+
+class Postgres {
+  private logger = new Logger(Postgres.name);
+  private pool;
+  private connected = false;
+
+  getConnection(connectionString: string) {
+    if (this.connected) {
+      return this.pool;
+    } else {
+      this.pool = new Pool({
+        connectionString,
+        ssl: {
+          rejectUnauthorized: false,
+        },
+      });
+
+      this.pool.on('error', () => {
+        this.logger.error('postgres disconnected');
+        this.connected = false;
+      });
+
+      try {
+        this.logger.verbose('connecting new postgres');
+        this.connected = true;
+      } catch (e) {
+        this.connected = false;
+        this.logger.error('postgres connect exception caught: ' + e);
+        return null;
+      }
+
+      return this.pool;
+    }
+  }
+
+  getChatwootConnection() {
+    const uri = configService.get<Chatwoot>('CHATWOOT').IMPORT.DATABASE.CONNECTION.URI;
+
+    return this.getConnection(uri);
+  }
+}
+
+export const postgresClient = new Postgres();

src/utils/chatwoot-import-helper.ts

@@ -0,0 +1,472 @@
+import { inbox } from '@figuro/chatwoot-sdk';
+import { proto } from '@whiskeysockets/baileys';
+
+import { Chatwoot, configService } from '../config/env.config';
+import { Logger } from '../config/logger.config';
+import { postgresClient } from '../libs/postgres.client';
+import { InstanceDto } from '../whatsapp/dto/instance.dto';
+import { ChatwootRaw, ContactRaw, MessageRaw } from '../whatsapp/models';
+import { ChatwootService } from '../whatsapp/services/chatwoot.service';
+
+type ChatwootUser = {
+  user_type: string;
+  user_id: number;
+};
+
+type FksChatwoot = {
+  phone_number: string;
+  contact_id: string;
+  conversation_id: string;
+};
+
+type firstLastTimestamp = {
+  first: number;
+  last: number;
+};
+
+type IWebMessageInfo = Omit<proto.IWebMessageInfo, 'key'> & Partial<Pick<proto.IWebMessageInfo, 'key'>>;
+
+class ChatwootImport {
+  private logger = new Logger(ChatwootImport.name);
+  private repositoryMessagesCache = new Map<string, Set<string>>();
+  private historyMessages = new Map<string, MessageRaw[]>();
+  private historyContacts = new Map<string, ContactRaw[]>();
+
+  public getRepositoryMessagesCache(instance: InstanceDto) {
+    return this.repositoryMessagesCache.has(instance.instanceName)
+      ? this.repositoryMessagesCache.get(instance.instanceName)
+      : null;
+  }
+
+  public setRepositoryMessagesCache(instance: InstanceDto, repositoryMessagesCache: Set<string>) {
+    this.repositoryMessagesCache.set(instance.instanceName, repositoryMessagesCache);
+  }
+
+  public deleteRepositoryMessagesCache(instance: InstanceDto) {
+    this.repositoryMessagesCache.delete(instance.instanceName);
+  }
+
+  public addHistoryMessages(instance: InstanceDto, messagesRaw: MessageRaw[]) {
+    const actualValue = this.historyMessages.has(instance.instanceName)
+      ? this.historyMessages.get(instance.instanceName)
+      : [];
+    this.historyMessages.set(instance.instanceName, actualValue.concat(messagesRaw));
+  }
+
+  public addHistoryContacts(instance: InstanceDto, contactsRaw: ContactRaw[]) {
+    const actualValue = this.historyContacts.has(instance.instanceName)
+      ? this.historyContacts.get(instance.instanceName)
+      : [];
+    this.historyContacts.set(instance.instanceName, actualValue.concat(contactsRaw));
+  }
+
+  public deleteHistoryMessages(instance: InstanceDto) {
+    this.historyMessages.delete(instance.instanceName);
+  }
+
+  public deleteHistoryContacts(instance: InstanceDto) {
+    this.historyContacts.delete(instance.instanceName);
+  }
+
+  public clearAll(instance: InstanceDto) {
+    this.deleteRepositoryMessagesCache(instance);
+    this.deleteHistoryMessages(instance);
+    this.deleteHistoryContacts(instance);
+  }
+
+  public getHistoryMessagesLenght(instance: InstanceDto) {
+    return this.historyMessages.get(instance.instanceName)?.length ?? 0;
+  }
+
+  public async importHistoryContacts(instance: InstanceDto, provider: ChatwootRaw) {
+    try {
+      if (this.getHistoryMessagesLenght(instance) > 0) {
+        return;
+      }
+
+      const pgClient = postgresClient.getChatwootConnection();
+
+      let totalContactsImported = 0;
+
+      const contacts = this.historyContacts.get(instance.instanceName) || [];
+      if (contacts.length === 0) {
+        return 0;
+      }
+
+      let contactsChunk: ContactRaw[] = this.sliceIntoChunks(contacts, 3000);
+      while (contactsChunk.length > 0) {
+        // inserting contacts in chatwoot db
+        let sqlInsert = `INSERT INTO contacts
+          (name, phone_number, account_id, identifier, created_at, updated_at) VALUES `;
+        const bindInsert = [provider.account_id];
+
+        for (const contact of contactsChunk) {
+          bindInsert.push(contact.pushName);
+          const bindName = `$${bindInsert.length}`;
+
+          bindInsert.push(`+${contact.id.split('@')[0]}`);
+          const bindPhoneNumber = `$${bindInsert.length}`;
+
+          bindInsert.push(contact.id);
+          const bindIdentifier = `$${bindInsert.length}`;
+
+          sqlInsert += `(${bindName}, ${bindPhoneNumber}, $1, ${bindIdentifier}, NOW(), NOW()),`;
+        }
+        if (sqlInsert.slice(-1) === ',') {
+          sqlInsert = sqlInsert.slice(0, -1);
+        }
+        sqlInsert += ` ON CONFLICT (identifier, account_id)
+                       DO UPDATE SET
+                        name = EXCLUDED.name,
+                        phone_number = EXCLUDED.phone_number,
+                        identifier = EXCLUDED.identifier`;
+
+        totalContactsImported += (await pgClient.query(sqlInsert, bindInsert))?.rowCount ?? 0;
+        contactsChunk = this.sliceIntoChunks(contacts, 3000);
+      }
+
+      this.deleteHistoryContacts(instance);
+
+      return totalContactsImported;
+    } catch (error) {
+      this.logger.error(`Error on import history contacts: ${error.toString()}`);
+    }
+  }
+
+  public async importHistoryMessages(
+    instance: InstanceDto,
+    chatwootService: ChatwootService,
+    inbox: inbox,
+    provider: ChatwootRaw,
+  ) {
+    try {
+      const pgClient = postgresClient.getChatwootConnection();
+
+      const chatwootUser = await this.getChatwootUser(provider);
+      if (!chatwootUser) {
+        throw new Error('User not found to import messages.');
+      }
+
+      let totalMessagesImported = 0;
+
+      const messagesOrdered = this.historyMessages.get(instance.instanceName) || [];
+      if (messagesOrdered.length === 0) {
+        return 0;
+      }
+
+      // ordering messages by number and timestamp asc
+      messagesOrdered.sort((a, b) => {
+        return (
+          parseInt(a.key.remoteJid) - parseInt(b.key.remoteJid) ||
+          (a.messageTimestamp as number) - (b.messageTimestamp as number)
+        );
+      });
+
+      const allMessagesMappedByPhoneNumber = this.createMessagesMapByPhoneNumber(messagesOrdered);
+      // Map structure: +552199999999 => { first message timestamp from number, last message timestamp from number}
+      const phoneNumbersWithTimestamp = new Map<string, firstLastTimestamp>();
+      allMessagesMappedByPhoneNumber.forEach((messages: MessageRaw[], phoneNumber: string) => {
+        phoneNumbersWithTimestamp.set(phoneNumber, {
+          first: messages[0]?.messageTimestamp as number,
+          last: messages[messages.length - 1]?.messageTimestamp as number,
+        });
+      });
+
+      // processing messages in batch
+      const batchSize = 4000;
+      let messagesChunk: MessageRaw[] = this.sliceIntoChunks(messagesOrdered, batchSize);
+      while (messagesChunk.length > 0) {
+        // Map structure: +552199999999 => MessageRaw[]
+        const messagesByPhoneNumber = this.createMessagesMapByPhoneNumber(messagesChunk);
+
+        if (messagesByPhoneNumber.size > 0) {
+          const fksByNumber = await this.selectOrCreateFksFromChatwoot(
+            provider,
+            inbox,
+            phoneNumbersWithTimestamp,
+            messagesByPhoneNumber,
+          );
+
+          // inserting messages in chatwoot db
+          let sqlInsertMsg = `INSERT INTO messages
+            (content, account_id, inbox_id, conversation_id, message_type, private, content_type,
+            sender_type, sender_id, created_at, updated_at) VALUES `;
+          const bindInsertMsg = [provider.account_id, inbox.id];
+
+          messagesByPhoneNumber.forEach((messages: MessageRaw[], phoneNumber: string) => {
+            const fksChatwoot = fksByNumber.get(phoneNumber);
+
+            messages.forEach((message) => {
+              if (!message.message) {
+                return;
+              }
+
+              if (!fksChatwoot?.conversation_id || !fksChatwoot?.contact_id) {
+                return;
+              }
+
+              const contentMessage = this.getContentMessage(chatwootService, message);
+              if (!contentMessage) {
+                return;
+              }
+
+              bindInsertMsg.push(contentMessage);
+              const bindContent = `$${bindInsertMsg.length}`;
+
+              bindInsertMsg.push(fksChatwoot.conversation_id);
+              const bindConversationId = `$${bindInsertMsg.length}`;
+
+              bindInsertMsg.push(message.key.fromMe ? '1' : '0');
+              const bindMessageType = `$${bindInsertMsg.length}`;
+
+              bindInsertMsg.push(message.key.fromMe ? chatwootUser.user_type : 'Contact');
+              const bindSenderType = `$${bindInsertMsg.length}`;
+
+              bindInsertMsg.push(message.key.fromMe ? chatwootUser.user_id : fksChatwoot.contact_id);
+              const bindSenderId = `$${bindInsertMsg.length}`;
+
+              bindInsertMsg.push(message.messageTimestamp as number);
+              const bindmessageTimestamp = `$${bindInsertMsg.length}`;
+
+              sqlInsertMsg += `(${bindContent}, $1, $2, ${bindConversationId}, ${bindMessageType}, FALSE, 0,
+                  ${bindSenderType},${bindSenderId}, to_timestamp(${bindmessageTimestamp}), to_timestamp(${bindmessageTimestamp})),`;
+            });
+          });
+          if (bindInsertMsg.length > 2) {
+            if (sqlInsertMsg.slice(-1) === ',') {
+              sqlInsertMsg = sqlInsertMsg.slice(0, -1);
+            }
+            totalMessagesImported += (await pgClient.query(sqlInsertMsg, bindInsertMsg))?.rowCount ?? 0;
+          }
+        }
+        messagesChunk = this.sliceIntoChunks(messagesOrdered, batchSize);
+      }
+
+      this.deleteHistoryMessages(instance);
+      this.deleteRepositoryMessagesCache(instance);
+
+      this.importHistoryContacts(instance, provider);
+
+      return totalMessagesImported;
+    } catch (error) {
+      this.logger.error(`Error on import history messages: ${error.toString()}`);
+
+      this.deleteHistoryMessages(instance);
+      this.deleteRepositoryMessagesCache(instance);
+    }
+  }
+
+  public async selectOrCreateFksFromChatwoot(
+    provider: ChatwootRaw,
+    inbox: inbox,
+    phoneNumbersWithTimestamp: Map<string, firstLastTimestamp>,
+    messagesByPhoneNumber: Map<string, MessageRaw[]>,
+  ): Promise<Map<string, FksChatwoot>> {
+    const pgClient = postgresClient.getChatwootConnection();
+
+    const bindValues = [provider.account_id, inbox.id];
+    const phoneNumberBind = Array.from(messagesByPhoneNumber.keys())
+      .map((phoneNumber) => {
+        const phoneNumberTimestamp = phoneNumbersWithTimestamp.get(phoneNumber);
+
+        if (phoneNumberTimestamp) {
+          bindValues.push(phoneNumber);
+          let bindStr = `($${bindValues.length},`;
+
+          bindValues.push(phoneNumberTimestamp.first);
+          bindStr += `$${bindValues.length},`;
+
+          bindValues.push(phoneNumberTimestamp.last);
+          return `${bindStr}$${bindValues.length})`;
+        }
+      })
+      .join(',');
+
+    // select (or insert when necessary) data from tables contacts, contact_inboxes, conversations from chatwoot db
+    const sqlFromChatwoot = `WITH
+              phone_number AS (
+                SELECT phone_number, created_at::INTEGER, last_activity_at::INTEGER FROM (
+                  VALUES 
+                   ${phoneNumberBind}
+                 ) as t (phone_number, created_at, last_activity_at)
+              ),
+
+              only_new_phone_number AS (
+                SELECT * FROM phone_number
+                WHERE phone_number NOT IN (
+                  SELECT phone_number
+                  FROM contacts
+                    JOIN contact_inboxes ci ON ci.contact_id = contacts.id AND ci.inbox_id = $2
+                    JOIN conversations con ON con.contact_inbox_id = ci.id 
+                      AND con.account_id = $1
+                      AND con.inbox_id = $2
+                      AND con.contact_id = contacts.id
+                  WHERE contacts.account_id = $1
+                )
+              ),
+
+              new_contact AS (
+                INSERT INTO contacts (name, phone_number, account_id, identifier, created_at, updated_at)
+                SELECT REPLACE(p.phone_number, '+', ''), p.phone_number, $1, CONCAT(REPLACE(p.phone_number, '+', ''),
+                  '@s.whatsapp.net'), to_timestamp(p.created_at), to_timestamp(p.last_activity_at)
+                FROM only_new_phone_number AS p
+                ON CONFLICT(identifier, account_id) DO UPDATE SET updated_at = EXCLUDED.updated_at
+                RETURNING id, phone_number, created_at, updated_at
+              ),
+
+              new_contact_inbox AS (
+                INSERT INTO contact_inboxes (contact_id, inbox_id, source_id, created_at, updated_at)
+                SELECT new_contact.id, $2, gen_random_uuid(), new_contact.created_at, new_contact.updated_at
+                FROM new_contact 
+                RETURNING id, contact_id, created_at, updated_at
+              ),
+
+              new_conversation AS (
+                INSERT INTO conversations (account_id, inbox_id, status, contact_id,
+                  contact_inbox_id, uuid, last_activity_at, created_at, updated_at)
+                SELECT $1, $2, 0, new_contact_inbox.contact_id, new_contact_inbox.id, gen_random_uuid(),
+                  new_contact_inbox.updated_at, new_contact_inbox.created_at, new_contact_inbox.updated_at
+                FROM new_contact_inbox
+                RETURNING id, contact_id
+              )
+
+              SELECT new_contact.phone_number, new_conversation.contact_id, new_conversation.id AS conversation_id
+              FROM new_conversation 
+              JOIN new_contact ON new_conversation.contact_id = new_contact.id
+
+              UNION
+
+              SELECT p.phone_number, c.id contact_id, con.id conversation_id
+                FROM phone_number p
+              JOIN contacts c ON c.phone_number = p.phone_number
+              JOIN contact_inboxes ci ON ci.contact_id = c.id AND ci.inbox_id = $2
+              JOIN conversations con ON con.contact_inbox_id = ci.id AND con.account_id = $1
+                AND con.inbox_id = $2 AND con.contact_id = c.id`;
+
+    const fksFromChatwoot = await pgClient.query(sqlFromChatwoot, bindValues);
+
+    return new Map(fksFromChatwoot.rows.map((item: FksChatwoot) => [item.phone_number, item]));
+  }
+
+  public async getChatwootUser(provider: ChatwootRaw): Promise<ChatwootUser> {
+    try {
+      const pgClient = postgresClient.getChatwootConnection();
+
+      const sqlUser = `SELECT owner_type AS user_type, owner_id AS user_id
+                         FROM access_tokens
+                       WHERE token = $1`;
+
+      return (await pgClient.query(sqlUser, [provider.token]))?.rows[0] || false;
+    } catch (error) {
+      this.logger.error(`Error on getChatwootUser: ${error.toString()}`);
+    }
+  }
+
+  public createMessagesMapByPhoneNumber(messages: MessageRaw[]): Map<string, MessageRaw[]> {
+    return messages.reduce((acc: Map<string, MessageRaw[]>, message: MessageRaw) => {
+      if (!this.isIgnorePhoneNumber(message?.key?.remoteJid)) {
+        const phoneNumber = message?.key?.remoteJid?.split('@')[0];
+        if (phoneNumber) {
+          const phoneNumberPlus = `+${phoneNumber}`;
+          const messages = acc.has(phoneNumberPlus) ? acc.get(phoneNumberPlus) : [];
+          messages.push(message);
+          acc.set(phoneNumberPlus, messages);
+        }
+      }
+
+      return acc;
+    }, new Map());
+  }
+
+  public async getContactsOrderByRecentConversations(
+    inbox: inbox,
+    provider: ChatwootRaw,
+    limit = 50,
+  ): Promise<{ id: number; phone_number: string; identifier: string }[]> {
+    try {
+      const pgClient = postgresClient.getChatwootConnection();
+
+      const sql = `SELECT contacts.id, contacts.identifier, contacts.phone_number
+                     FROM conversations
+                   JOIN contacts ON contacts.id = conversations.contact_id
+                   WHERE conversations.account_id = $1
+                     AND inbox_id = $2
+                   ORDER BY conversations.last_activity_at DESC
+                   LIMIT $3`;
+
+      return (await pgClient.query(sql, [provider.account_id, inbox.id, limit]))?.rows;
+    } catch (error) {
+      this.logger.error(`Error on get recent conversations: ${error.toString()}`);
+    }
+  }
+
+  public getContentMessage(chatwootService: ChatwootService, msg: IWebMessageInfo) {
+    const contentMessage = chatwootService.getConversationMessage(msg.message);
+    if (contentMessage) {
+      return contentMessage;
+    }
+
+    if (!configService.get<Chatwoot>('CHATWOOT').IMPORT.PLACEHOLDER_MEDIA_MESSAGE) {
+      return '';
+    }
+
+    const types = {
+      documentMessage: msg.message.documentMessage,
+      documentWithCaptionMessage: msg.message.documentWithCaptionMessage?.message?.documentMessage,
+      imageMessage: msg.message.imageMessage,
+      videoMessage: msg.message.videoMessage,
+      audioMessage: msg.message.audioMessage,
+      stickerMessage: msg.message.stickerMessage,
+      templateMessage: msg.message.templateMessage?.hydratedTemplate?.hydratedContentText,
+    };
+    const typeKey = Object.keys(types).find((key) => types[key] !== undefined);
+
+    switch (typeKey) {
+      case 'documentMessage':
+        return `_<File: ${msg.message.documentMessage.fileName}${
+          msg.message.documentMessage.caption ? ` ${msg.message.documentMessage.caption}` : ''
+        }>_`;
+
+      case 'documentWithCaptionMessage':
+        return `_<File: ${msg.message.documentWithCaptionMessage.message.documentMessage.fileName}${
+          msg.message.documentWithCaptionMessage.message.documentMessage.caption
+            ? ` ${msg.message.documentWithCaptionMessage.message.documentMessage.caption}`
+            : ''
+        }>_`;
+
+      case 'templateMessage':
+        return msg.message.templateMessage.hydratedTemplate.hydratedTitleText
+          ? `*${msg.message.templateMessage.hydratedTemplate.hydratedTitleText}*\\n`
+          : '' + msg.message.templateMessage.hydratedTemplate.hydratedContentText;
+
+      case 'imageMessage':
+        return '_<Image Message>_';
+
+      case 'videoMessage':
+        return '_<Video Message>_';
+
+      case 'audioMessage':
+        return '_<Audio Message>_';
+
+      case 'stickerMessage':
+        return '_<Sticker Message>_';
+
+      default:
+        return '';
+    }
+  }
+
+  public sliceIntoChunks(arr: any[], chunkSize: number) {
+    return arr.splice(0, chunkSize);
+  }
+
+  public isGroup(remoteJid: string) {
+    return remoteJid.includes('@g.us');
+  }
+
+  public isIgnorePhoneNumber(remoteJid: string) {
+    return this.isGroup(remoteJid) || remoteJid === 'status@broadcast' || remoteJid === '0@s.whatsapp.net';
+  }
+}
+
+export const chatwootImport = new ChatwootImport();

src/validate/validate.schema.ts

@@ -923,6 +923,9 @@ export const chatwootSchema: JSONSchema7 = {
     reopen_conversation: { type: 'boolean', enum: [true, false] },
     conversation_pending: { type: 'boolean', enum: [true, false] },
     auto_create: { type: 'boolean', enum: [true, false] },
+    import_contacts: { type: 'boolean', enum: [true, false] },
+    import_messages: { type: 'boolean', enum: [true, false] },
+    days_limit_import_messages: { type: 'number' },
   },
   required: ['enabled', 'account_id', 'token', 'url', 'sign_msg', 'reopen_conversation', 'conversation_pending'],
   ...isNotEmpty('account_id', 'token', 'url', 'sign_msg', 'reopen_conversation', 'conversation_pending'),
@@ -938,9 +941,10 @@ export const settingsSchema: JSONSchema7 = {
     always_online: { type: 'boolean', enum: [true, false] },
     read_messages: { type: 'boolean', enum: [true, false] },
     read_status: { type: 'boolean', enum: [true, false] },
+    sync_full_history: { type: 'boolean', enum: [true, false] },
   },
-  required: ['reject_call', 'groups_ignore', 'always_online', 'read_messages', 'read_status'],
+  required: ['reject_call', 'groups_ignore', 'always_online', 'read_messages', 'read_status', 'sync_full_history'],
-  ...isNotEmpty('reject_call', 'groups_ignore', 'always_online', 'read_messages', 'read_status'),
+  ...isNotEmpty('reject_call', 'groups_ignore', 'always_online', 'read_messages', 'read_status', 'sync_full_history'),
 };
 
 export const websocketSchema: JSONSchema7 = {

src/whatsapp/controllers/chatwoot.controller.ts

@@ -51,6 +51,9 @@ export class ChatwootController {
       data.sign_delimiter = null;
       data.reopen_conversation = false;
       data.conversation_pending = false;
+      data.import_contacts = false;
+      data.import_messages = false;
+      data.days_limit_import_messages = 0;
       data.auto_create = false;
     }
 

src/whatsapp/controllers/instance.controller.ts

@@ -57,12 +57,16 @@ export class InstanceController {
     chatwoot_sign_msg,
     chatwoot_reopen_conversation,
     chatwoot_conversation_pending,
+    chatwoot_import_contacts,
+    chatwoot_import_messages,
+    chatwoot_days_limit_import_messages,
     reject_call,
     msg_call,
     groups_ignore,
     always_online,
     read_messages,
     read_status,
+    sync_full_history,
     websocket_enabled,
     websocket_events,
     rabbitmq_enabled,
@@ -342,6 +346,7 @@ export class InstanceController {
         always_online: always_online || false,
         read_messages: read_messages || false,
         read_status: read_status || false,
+        sync_full_history: sync_full_history ?? false,
       };
 
       this.logger.verbose('settings: ' + JSON.stringify(settings));
@@ -444,6 +449,9 @@ export class InstanceController {
           number,
           reopen_conversation: chatwoot_reopen_conversation || false,
           conversation_pending: chatwoot_conversation_pending || false,
+          import_contacts: chatwoot_import_contacts ?? true,
+          import_messages: chatwoot_import_messages ?? true,
+          days_limit_import_messages: chatwoot_days_limit_import_messages ?? 60,
           auto_create: true,
         });
       } catch (error) {
@@ -494,6 +502,9 @@ export class InstanceController {
           sign_msg: chatwoot_sign_msg || false,
           reopen_conversation: chatwoot_reopen_conversation || false,
           conversation_pending: chatwoot_conversation_pending || false,
+          import_contacts: chatwoot_import_contacts ?? true,
+          import_messages: chatwoot_import_messages ?? true,
+          days_limit_import_messages: chatwoot_days_limit_import_messages || 60,
           number,
           name_inbox: instance.instanceName,
           webhook_url: `${urlServer}/chatwoot/webhook/${encodeURIComponent(instance.instanceName)}`,

src/whatsapp/dto/chatwoot.dto.ts

@@ -9,5 +9,8 @@ export class ChatwootDto {
   number?: string;
   reopen_conversation?: boolean;
   conversation_pending?: boolean;
+  import_contacts?: boolean;
+  import_messages?: boolean;
+  days_limit_import_messages?: number;
   auto_create?: boolean;
 }

src/whatsapp/dto/instance.dto.ts

@@ -14,12 +14,16 @@ export class InstanceDto {
   always_online?: boolean;
   read_messages?: boolean;
   read_status?: boolean;
+  sync_full_history?: boolean;
   chatwoot_account_id?: string;
   chatwoot_token?: string;
   chatwoot_url?: string;
   chatwoot_sign_msg?: boolean;
   chatwoot_reopen_conversation?: boolean;
   chatwoot_conversation_pending?: boolean;
+  chatwoot_import_contacts?: boolean;
+  chatwoot_import_messages?: boolean;
+  chatwoot_days_limit_import_messages?: number;
   websocket_enabled?: boolean;
   websocket_events?: string[];
   rabbitmq_enabled?: boolean;

src/whatsapp/dto/settings.dto.ts

@@ -5,4 +5,5 @@ export class SettingsDto {
   always_online?: boolean;
   read_messages?: boolean;
   read_status?: boolean;
+  sync_full_history?: boolean;
 }

src/whatsapp/models/chat.model.ts

@@ -9,6 +9,11 @@ export class ChatRaw {
   lastMsgTimestamp?: number;
 }
 
+type ChatRawBoolean<T> = {
+  [P in keyof T]?: 0 | 1;
+};
+export type ChatRawSelect = ChatRawBoolean<ChatRaw>;
+
 const chatSchema = new Schema<ChatRaw>({
   _id: { type: String, _id: true },
   id: { type: String, required: true, minlength: 1 },

src/whatsapp/models/chatwoot.model.ts

@@ -14,6 +14,9 @@ export class ChatwootRaw {
   number?: string;
   reopen_conversation?: boolean;
   conversation_pending?: boolean;
+  import_contacts?: boolean;
+  import_messages?: boolean;
+  days_limit_import_messages?: number;
 }
 
 const chatwootSchema = new Schema<ChatwootRaw>({
@@ -28,6 +31,9 @@ const chatwootSchema = new Schema<ChatwootRaw>({
   number: { type: String, required: true },
   reopen_conversation: { type: Boolean, required: true },
   conversation_pending: { type: Boolean, required: true },
+  import_contacts: { type: Boolean, required: true },
+  import_messages: { type: Boolean, required: true },
+  days_limit_import_messages: { type: Number, required: true },
 });
 
 export const ChatwootModel = dbserver?.model(ChatwootRaw.name, chatwootSchema, 'chatwoot');

src/whatsapp/models/contact.model.ts

@@ -10,6 +10,11 @@ export class ContactRaw {
   owner: string;
 }
 
+type ContactRawBoolean<T> = {
+  [P in keyof T]?: 0 | 1;
+};
+export type ContactRawSelect = ContactRawBoolean<ContactRaw>;
+
 const contactSchema = new Schema<ContactRaw>({
   _id: { type: String, _id: true },
   pushName: { type: String, minlength: 1 },

src/whatsapp/models/message.model.ts

@@ -33,6 +33,13 @@ export class MessageRaw {
   contextInfo?: any;
 }
 
+type MessageRawBoolean<T> = {
+  [P in keyof T]?: 0 | 1;
+};
+export type MessageRawSelect = Omit<MessageRawBoolean<MessageRaw>, 'key'> & {
+  key?: MessageRawBoolean<Key>;
+};
+
 const messageSchema = new Schema<MessageRaw>({
   _id: { type: String, _id: true },
   key: {

src/whatsapp/models/settings.model.ts

@@ -10,6 +10,7 @@ export class SettingsRaw {
   always_online?: boolean;
   read_messages?: boolean;
   read_status?: boolean;
+  sync_full_history?: boolean;
 }
 
 const settingsSchema = new Schema<SettingsRaw>({
@@ -20,6 +21,7 @@ const settingsSchema = new Schema<SettingsRaw>({
   always_online: { type: Boolean, required: true },
   read_messages: { type: Boolean, required: true },
   read_status: { type: Boolean, required: true },
+  sync_full_history: { type: Boolean, required: true },
 });
 
 export const SettingsModel = dbserver?.model(SettingsRaw.name, settingsSchema, 'settings');

src/whatsapp/repository/chat.repository.ts

@@ -4,9 +4,10 @@ import { join } from 'path';
 import { ConfigService, StoreConf } from '../../config/env.config';
 import { Logger } from '../../config/logger.config';
 import { IInsert, Repository } from '../abstract/abstract.repository';
-import { ChatRaw, IChatModel } from '../models';
+import { ChatRaw, ChatRawSelect, IChatModel } from '../models';
 
 export class ChatQuery {
+  select?: ChatRawSelect;
   where: ChatRaw;
 }
 
@@ -69,7 +70,7 @@ export class ChatRepository extends Repository {
       this.logger.verbose('finding chats');
       if (this.dbSettings.ENABLED) {
         this.logger.verbose('finding chats in db');
-        return await this.chatModel.find({ owner: query.where.owner });
+        return await this.chatModel.find({ owner: query.where.owner }).select(query.select ?? {});
       }
 
       this.logger.verbose('finding chats in store');

src/whatsapp/repository/contact.repository.ts

@@ -4,9 +4,10 @@ import { join } from 'path';
 import { ConfigService, StoreConf } from '../../config/env.config';
 import { Logger } from '../../config/logger.config';
 import { IInsert, Repository } from '../abstract/abstract.repository';
-import { ContactRaw, IContactModel } from '../models';
+import { ContactRaw, ContactRawSelect, IContactModel } from '../models';
 
 export class ContactQuery {
+  select?: ContactRawSelect;
   where: ContactRaw;
 }
 
@@ -129,7 +130,7 @@ export class ContactRepository extends Repository {
       this.logger.verbose('finding contacts');
       if (this.dbSettings.ENABLED) {
         this.logger.verbose('finding contacts in db');
-        return await this.contactModel.find({ ...query.where });
+        return await this.contactModel.find({ ...query.where }).select(query.select ?? {});
       }
 
       this.logger.verbose('finding contacts in store');

src/whatsapp/repository/message.repository.ts

@@ -4,9 +4,10 @@ import { join } from 'path';
 import { ConfigService, StoreConf } from '../../config/env.config';
 import { Logger } from '../../config/logger.config';
 import { IInsert, Repository } from '../abstract/abstract.repository';
-import { IMessageModel, MessageRaw } from '../models';
+import { IMessageModel, MessageRaw, MessageRawSelect } from '../models';
 
 export class MessageQuery {
+  select?: MessageRawSelect;
   where: MessageRaw;
   limit?: number;
 }
@@ -19,7 +20,7 @@ export class MessageRepository extends Repository {
   private readonly logger = new Logger('MessageRepository');
 
   public buildQuery(query: MessageQuery): MessageQuery {
-    for (const [o, p] of Object.entries(query?.where)) {
+    for (const [o, p] of Object.entries(query?.where || {})) {
       if (typeof p === 'object' && p !== null && !Array.isArray(p)) {
         for (const [k, v] of Object.entries(p)) {
           query.where[`${o}.${k}`] = v;
@@ -28,6 +29,15 @@ export class MessageRepository extends Repository {
       }
     }
 
+    for (const [o, p] of Object.entries(query?.select || {})) {
+      if (typeof p === 'object' && p !== null && !Array.isArray(p)) {
+        for (const [k, v] of Object.entries(p)) {
+          query.select[`${o}.${k}`] = v;
+        }
+        delete query.select[o];
+      }
+    }
+
     return query;
   }
 
@@ -149,6 +159,7 @@ export class MessageRepository extends Repository {
         })
         .splice(0, query?.limit ?? messages.length);
     } catch (error) {
+      this.logger.error(`error on message find: ${error.toString()}`);
       return [];
     }
   }

src/whatsapp/services/chatwoot.service.ts

@@ -1,4 +1,4 @@
-import ChatwootClient, { ChatwootAPIConfig, contact, conversation, inbox } from '@figuro/chatwoot-sdk';
+import ChatwootClient, { ChatwootAPIConfig, contact, conversation, generic_id, inbox } from '@figuro/chatwoot-sdk';
 import { request as chatwootRequest } from '@figuro/chatwoot-sdk/dist/core/request';
 import axios from 'axios';
 import FormData from 'form-data';
@@ -7,14 +7,15 @@ import Jimp from 'jimp';
 import mimeTypes from 'mime-types';
 import path from 'path';
 
-import { ChatWoot, ConfigService, HttpServer } from '../../config/env.config';
+import { Chatwoot, ConfigService, HttpServer } from '../../config/env.config';
 import { Logger } from '../../config/logger.config';
+import { chatwootImport } from '../../utils/chatwoot-import-helper';
 import i18next from '../../utils/i18n';
 import { ICache } from '../abstract/abstract.cache';
 import { ChatwootDto } from '../dto/chatwoot.dto';
 import { InstanceDto } from '../dto/instance.dto';
 import { Options, Quoted, SendAudioDto, SendMediaDto, SendTextDto } from '../dto/sendMessage.dto';
-import { ChatwootRaw, MessageRaw } from '../models';
+import { ChatwootRaw, ContactRaw, MessageRaw } from '../models';
 import { RepositoryBroker } from '../repository/repository.manager';
 import { Events } from '../types/wa.types';
 import { WAMonitoringService } from './monitor.service';
@@ -773,6 +774,43 @@ export class ChatwootService {
     return message;
   }
 
+  public async getOpenConversationByContact(
+    instance: InstanceDto,
+    inbox: inbox,
+    contact: generic_id & contact,
+  ): Promise<conversation> {
+    this.logger.verbose('find conversation in chatwoot');
+
+    const client = await this.clientCw(instance);
+
+    if (!client) {
+      this.logger.warn('client not found');
+      return null;
+    }
+
+    const payload = [
+      ['inbox_id', inbox.id.toString()],
+      ['contact_id', contact.id.toString()],
+      ['status', 'open'],
+    ];
+
+    return (
+      (
+        (await client.conversations.filter({
+          accountId: this.provider.account_id,
+          payload: payload.map((item, i, payload) => {
+            return {
+              attribute_key: item[0],
+              filter_operator: 'equal_to',
+              values: [item[1]],
+              query_operator: i < payload.length - 1 ? 'AND' : null,
+            };
+          }),
+        })) as { payload: conversation[] }
+      ).payload[0] || undefined
+    );
+  }
+
   public async createBotMessage(
     instance: InstanceDto,
     content: string,
@@ -808,21 +846,7 @@ export class ChatwootService {
       return null;
     }
 
-    this.logger.verbose('find conversation in chatwoot');
+    const conversation = await this.getOpenConversationByContact(instance, filterInbox, contact);
-    const findConversation = await client.conversations.list({
-      accountId: this.provider.account_id,
-      inboxId: filterInbox.id,
-    });
-
-    if (!findConversation) {
-      this.logger.warn('conversation not found');
-      return null;
-    }
-
-    this.logger.verbose('find conversation by contact id');
-    const conversation = findConversation.data.payload.find(
-      (conversation) => conversation?.meta?.sender?.id === contact.id && conversation.status === 'open',
-    );
 
     if (!conversation) {
       this.logger.warn('conversation not found');
@@ -945,21 +969,7 @@ export class ChatwootService {
       return null;
     }
 
-    this.logger.verbose('find conversation in chatwoot');
+    const conversation = await this.getOpenConversationByContact(instance, filterInbox, contact);
-    const findConversation = await client.conversations.list({
-      accountId: this.provider.account_id,
-      inboxId: filterInbox.id,
-    });
-
-    if (!findConversation) {
-      this.logger.warn('conversation not found');
-      return null;
-    }
-
-    this.logger.verbose('find conversation by contact id');
-    const conversation = findConversation.data.payload.find(
-      (conversation) => conversation?.meta?.sender?.id === contact.id && conversation.status === 'open',
-    );
 
     if (!conversation) {
       this.logger.warn('conversation not found');
@@ -1658,7 +1668,7 @@ export class ChatwootService {
     return result;
   }
 
-  private getConversationMessage(msg: any) {
+  public getConversationMessage(msg: any) {
     this.logger.verbose('get conversation message');
 
     const types = this.getTypeMessage(msg);
@@ -1989,7 +1999,7 @@ export class ChatwootService {
       }
 
       if (event === Events.MESSAGES_DELETE) {
-        const chatwootDelete = this.configService.get<ChatWoot>('CHATWOOT').MESSAGE_DELETE;
+        const chatwootDelete = this.configService.get<Chatwoot>('CHATWOOT').MESSAGE_DELETE;
         if (chatwootDelete === true) {
           this.logger.verbose('deleting message from instance: ' + instance.instanceName);
 
@@ -2084,6 +2094,7 @@ export class ChatwootService {
             this.logger.verbose('send message to chatwoot');
             await this.createBotMessage(instance, msgConnection, 'incoming');
             this.waMonitor.waInstances[instance.instanceName].qrCode.count = 0;
+            chatwootImport.clearAll(instance);
           }
         }
       }
@@ -2134,4 +2145,113 @@ export class ChatwootService {
   public getNumberFromRemoteJid(remoteJid: string) {
     return remoteJid.replace(/:\d+/, '').split('@')[0];
   }
+
+  public startImportHistoryMessages(instance: InstanceDto) {
+    if (!this.isImportHistoryAvailable()) {
+      return;
+    }
+
+    this.createBotMessage(instance, `💬 Starting to import messages. Please wait...`, 'incoming');
+  }
+
+  public isImportHistoryAvailable() {
+    const uri = this.configService.get<Chatwoot>('CHATWOOT').IMPORT.DATABASE.CONNECTION.URI;
+
+    return uri && uri !== 'postgres://user:password@hostname:port/dbname';
+  }
+
+  /* We can't proccess messages exactly in batch because Chatwoot use message id to order
+     messages in frontend and we are receiving the messages mixed between the batches.
+     Because this, we need to put all batches together and order after */
+  public addHistoryMessages(instance: InstanceDto, messagesRaw: MessageRaw[]) {
+    if (!this.isImportHistoryAvailable()) {
+      return;
+    }
+
+    chatwootImport.addHistoryMessages(instance, messagesRaw);
+  }
+
+  public addHistoryContacts(instance: InstanceDto, contactsRaw: ContactRaw[]) {
+    if (!this.isImportHistoryAvailable()) {
+      return;
+    }
+
+    return chatwootImport.addHistoryContacts(instance, contactsRaw);
+  }
+
+  public async importHistoryMessages(instance: InstanceDto) {
+    if (!this.isImportHistoryAvailable()) {
+      return;
+    }
+
+    this.createBotMessage(instance, '💬 Importing messages. More one moment...', 'incoming');
+
+    const totalMessagesImported = await chatwootImport.importHistoryMessages(
+      instance,
+      this,
+      await this.getInbox(instance),
+      this.provider,
+    );
+    this.updateContactAvatarInRecentConversations(instance);
+
+    const msg = Number.isInteger(totalMessagesImported)
+      ? `${totalMessagesImported} messages imported. Refresh page to see the new messages`
+      : `Something went wrong in importing messages`;
+
+    this.createBotMessage(instance, `💬 ${msg}`, 'incoming');
+
+    return totalMessagesImported;
+  }
+
+  public async updateContactAvatarInRecentConversations(instance: InstanceDto, limitContacts = 100) {
+    try {
+      if (!this.isImportHistoryAvailable()) {
+        return;
+      }
+
+      const client = await this.clientCw(instance);
+      if (!client) {
+        this.logger.warn('client not found');
+        return null;
+      }
+
+      const inbox = await this.getInbox(instance);
+      if (!inbox) {
+        this.logger.warn('inbox not found');
+        return null;
+      }
+
+      const recentContacts = await chatwootImport.getContactsOrderByRecentConversations(
+        inbox,
+        this.provider,
+        limitContacts,
+      );
+
+      const contactsWithProfilePicture = (
+        await this.repository.contact.find({
+          where: {
+            owner: instance.instanceName,
+            id: {
+              $in: recentContacts.map((contact) => contact.identifier),
+            },
+            profilePictureUrl: { $ne: null },
+          },
+        } as any)
+      ).reduce((acc: Map<string, ContactRaw>, contact: ContactRaw) => acc.set(contact.id, contact), new Map());
+
+      recentContacts.forEach(async (contact) => {
+        if (contactsWithProfilePicture.has(contact.identifier)) {
+          client.contacts.update({
+            accountId: this.provider.account_id,
+            id: contact.id,
+            data: {
+              avatar_url: contactsWithProfilePicture.get(contact.identifier).profilePictureUrl || null,
+            },
+          });
+        }
+      });
+    } catch (error) {
+      this.logger.error(`Error on update avatar in recent conversations: ${error.toString()}`);
+    }
+  }
 }

src/whatsapp/services/whatsapp.service.ts

@@ -73,6 +73,7 @@ import { dbserver } from '../../libs/db.connect';
 import { RedisCache } from '../../libs/redis.client';
 import { getIO } from '../../libs/socket.server';
 import { getSQS, removeQueues as removeQueuesSQS } from '../../libs/sqs.server';
+import { chatwootImport } from '../../utils/chatwoot-import-helper';
 import { makeProxyAgent } from '../../utils/makeProxyAgent';
 import { useMultiFileAuthStateDb } from '../../utils/use-multi-file-auth-state-db';
 import { useMultiFileAuthStateRedisDb } from '../../utils/use-multi-file-auth-state-redis-db';
@@ -103,6 +104,7 @@ import {
   GroupUpdateParticipantDto,
   GroupUpdateSettingDto,
 } from '../dto/group.dto';
+import { InstanceDto } from '../dto/instance.dto';
 import {
   ContactMessage,
   MediaMessage,
@@ -355,6 +357,15 @@ export class WAStartupService {
     this.localChatwoot.conversation_pending = data?.conversation_pending;
     this.logger.verbose(`Chatwoot conversation pending: ${this.localChatwoot.conversation_pending}`);
 
+    this.localChatwoot.import_contacts = data?.import_contacts;
+    this.logger.verbose(`Chatwoot import contacts: ${this.localChatwoot.import_contacts}`);
+
+    this.localChatwoot.import_messages = data?.import_messages;
+    this.logger.verbose(`Chatwoot import messages: ${this.localChatwoot.import_messages}`);
+
+    this.localChatwoot.days_limit_import_messages = data?.days_limit_import_messages;
+    this.logger.verbose(`Chatwoot days limit import messages: ${this.localChatwoot.days_limit_import_messages}`);
+
     this.logger.verbose('Chatwoot loaded');
   }
 
@@ -369,6 +380,9 @@ export class WAStartupService {
     this.logger.verbose(`Chatwoot sign delimiter: ${data.sign_delimiter}`);
     this.logger.verbose(`Chatwoot reopen conversation: ${data.reopen_conversation}`);
     this.logger.verbose(`Chatwoot conversation pending: ${data.conversation_pending}`);
+    this.logger.verbose(`Chatwoot import contacts: ${data.import_contacts}`);
+    this.logger.verbose(`Chatwoot import messages: ${data.import_messages}`);
+    this.logger.verbose(`Chatwoot days limit import messages: ${data.days_limit_import_messages}`);
 
     Object.assign(this.localChatwoot, { ...data, sign_delimiter: data.sign_msg ? data.sign_delimiter : null });
 
@@ -394,6 +408,9 @@ export class WAStartupService {
     this.logger.verbose(`Chatwoot sign delimiter: ${data.sign_delimiter}`);
     this.logger.verbose(`Chatwoot reopen conversation: ${data.reopen_conversation}`);
     this.logger.verbose(`Chatwoot conversation pending: ${data.conversation_pending}`);
+    this.logger.verbose(`Chatwoot import contacts: ${data.import_contacts}`);
+    this.logger.verbose(`Chatwoot import messages: ${data.import_messages}`);
+    this.logger.verbose(`Chatwoot days limit import messages: ${data.days_limit_import_messages}`);
 
     return {
       enabled: data.enabled,
@@ -405,6 +422,9 @@ export class WAStartupService {
       sign_delimiter: data.sign_delimiter || null,
       reopen_conversation: data.reopen_conversation,
       conversation_pending: data.conversation_pending,
+      import_contacts: data.import_contacts,
+      import_messages: data.import_messages,
+      days_limit_import_messages: data.days_limit_import_messages,
     };
   }
 
@@ -437,6 +457,9 @@ export class WAStartupService {
     this.localSettings.read_status = data?.read_status;
     this.logger.verbose(`Settings read_status: ${this.localSettings.read_status}`);
 
+    this.localSettings.sync_full_history = data?.sync_full_history;
+    this.logger.verbose(`Settings sync_full_history: ${this.localSettings.sync_full_history}`);
+
     this.logger.verbose('Settings loaded');
   }
 
@@ -449,6 +472,7 @@ export class WAStartupService {
     this.logger.verbose(`Settings always_online: ${data.always_online}`);
     this.logger.verbose(`Settings read_messages: ${data.read_messages}`);
     this.logger.verbose(`Settings read_status: ${data.read_status}`);
+    this.logger.verbose(`Settings sync_full_history: ${data.sync_full_history}`);
     Object.assign(this.localSettings, data);
     this.logger.verbose('Settings set');
 
@@ -470,6 +494,7 @@ export class WAStartupService {
     this.logger.verbose(`Settings always_online: ${data.always_online}`);
     this.logger.verbose(`Settings read_messages: ${data.read_messages}`);
     this.logger.verbose(`Settings read_status: ${data.read_status}`);
+    this.logger.verbose(`Settings sync_full_history: ${data.sync_full_history}`);
     return {
       reject_call: data.reject_call,
       msg_call: data.msg_call,
@@ -477,6 +502,7 @@ export class WAStartupService {
       always_online: data.always_online,
       read_messages: data.read_messages,
       read_status: data.read_status,
+      sync_full_history: data.sync_full_history,
     };
   }
 
@@ -1430,7 +1456,10 @@ export class WAStartupService {
         msgRetryCounterCache: this.msgRetryCounterCache,
         getMessage: async (key) => (await this.getMessage(key)) as Promise<proto.IMessage>,
         generateHighQualityLinkPreview: true,
-        syncFullHistory: false,
+        syncFullHistory: this.localSettings.sync_full_history,
+        shouldSyncHistoryMessage: (msg: proto.Message.IHistorySyncNotification) => {
+          return this.historySyncNotification(msg);
+        },
         userDevicesCache: this.userDevicesCache,
         transactionOpts: { maxCommitRetries: 10, delayBetweenTriesMs: 10 },
         patchMessageBeforeSending(message) {
@@ -1517,7 +1546,10 @@ export class WAStartupService {
         msgRetryCounterCache: this.msgRetryCounterCache,
         getMessage: async (key) => (await this.getMessage(key)) as Promise<proto.IMessage>,
         generateHighQualityLinkPreview: true,
-        syncFullHistory: false,
+        syncFullHistory: this.localSettings.sync_full_history,
+        shouldSyncHistoryMessage: (msg: proto.Message.IHistorySyncNotification) => {
+          return this.historySyncNotification(msg);
+        },
         userDevicesCache: this.userDevicesCache,
         transactionOpts: { maxCommitRetries: 10, delayBetweenTriesMs: 10 },
         patchMessageBeforeSending(message) {
@@ -1611,33 +1643,48 @@ export class WAStartupService {
 
   private readonly contactHandle = {
     'contacts.upsert': async (contacts: Contact[], database: Database) => {
-      this.logger.verbose('Event received: contacts.upsert');
+      try {
+        this.logger.verbose('Event received: contacts.upsert');
 
-      this.logger.verbose('Finding contacts in database');
+        this.logger.verbose('Finding contacts in database');
-      const contactsRepository = await this.repository.contact.find({
+        const contactsRepository = new Set(
-        where: { owner: this.instance.name },
+          (
-      });
+            await this.repository.contact.find({
+              select: { id: 1, _id: 0 },
+              where: { owner: this.instance.name },
+            })
+          ).map((contact) => contact.id),
+        );
 
-      this.logger.verbose('Verifying if contacts exists in database to insert');
+        this.logger.verbose('Verifying if contacts exists in database to insert');
-      const contactsRaw: ContactRaw[] = [];
+        const contactsRaw: ContactRaw[] = [];
-      for await (const contact of contacts) {
+
-        if (contactsRepository.find((cr) => cr.id === contact.id)) {
+        for (const contact of contacts) {
-          continue;
+          if (contactsRepository.has(contact.id)) {
+            continue;
+          }
+
+          contactsRaw.push({
+            id: contact.id,
+            pushName: contact?.name || contact?.verifiedName || contact.id.split('@')[0],
+            profilePictureUrl: (await this.profilePicture(contact.id)).profilePictureUrl,
+            owner: this.instance.name,
+          });
         }
 
-        contactsRaw.push({
+        this.logger.verbose('Sending data to webhook in event CONTACTS_UPSERT');
-          id: contact.id,
+        this.sendDataWebhook(Events.CONTACTS_UPSERT, contactsRaw);
-          pushName: contact?.name || contact?.verifiedName,
+
-          profilePictureUrl: (await this.profilePicture(contact.id)).profilePictureUrl,
+        this.logger.verbose('Inserting contacts in database');
-          owner: this.instance.name,
+        this.repository.contact.insert(contactsRaw, this.instance.name, database.SAVE_DATA.CONTACTS);
-        });
+
+        if (this.localChatwoot.enabled && this.localChatwoot.import_contacts && contactsRaw.length) {
+          this.chatwootService.addHistoryContacts({ instanceName: this.instance.name }, contactsRaw);
+          chatwootImport.importHistoryContacts({ instanceName: this.instance.name }, this.localChatwoot);
+        }
+      } catch (error) {
+        this.logger.error(error);
       }
-
-      this.logger.verbose('Sending data to webhook in event CONTACTS_UPSERT');
-      this.sendDataWebhook(Events.CONTACTS_UPSERT, contactsRaw);
-
-      this.logger.verbose('Inserting contacts in database');
-      this.repository.contact.insert(contactsRaw, this.instance.name, database.SAVE_DATA.CONTACTS);
     },
 
     'contacts.update': async (contacts: Partial<Contact>[], database: Database) => {
@@ -1667,7 +1714,6 @@ export class WAStartupService {
       {
         messages,
         chats,
-        isLatest,
       }: {
         chats: Chat[];
         contacts: Contact[];
@@ -1676,55 +1722,115 @@ export class WAStartupService {
       },
       database: Database,
     ) => {
-      this.logger.verbose('Event received: messaging-history.set');
+      try {
-      if (isLatest) {
+        this.logger.verbose('Event received: messaging-history.set');
-        this.logger.verbose('isLatest defined as true');
+
-        const chatsRaw: ChatRaw[] = chats.map((chat) => {
+        const instance: InstanceDto = { instanceName: this.instance.name };
-          return {
+
+        const daysLimitToImport = this.localChatwoot.enabled ? this.localChatwoot.days_limit_import_messages : 1000;
+        this.logger.verbose(`Param days limit import messages is: ${daysLimitToImport}`);
+
+        const date = new Date();
+        const timestampLimitToImport = new Date(date.setDate(date.getDate() - daysLimitToImport)).getTime() / 1000;
+
+        const maxBatchTimestamp = Math.max(...messages.map((message) => message.messageTimestamp as number));
+
+        const processBatch = maxBatchTimestamp >= timestampLimitToImport;
+
+        if (!processBatch) {
+          this.logger.verbose('Batch ignored by maxTimestamp in this batch');
+          return;
+        }
+
+        const chatsRaw: ChatRaw[] = [];
+        const chatsRepository = new Set(
+          (
+            await this.repository.chat.find({
+              select: { id: 1, _id: 0 },
+              where: { owner: this.instance.name },
+            })
+          ).map((chat) => chat.id),
+        );
+
+        for (const chat of chats) {
+          if (chatsRepository.has(chat.id)) {
+            continue;
+          }
+
+          chatsRaw.push({
             id: chat.id,
             owner: this.instance.name,
             lastMsgTimestamp: chat.lastMessageRecvTimestamp,
-          };
+          });
-        });
+        }
 
         this.logger.verbose('Sending data to webhook in event CHATS_SET');
         this.sendDataWebhook(Events.CHATS_SET, chatsRaw);
 
         this.logger.verbose('Inserting chats in database');
         this.repository.chat.insert(chatsRaw, this.instance.name, database.SAVE_DATA.CHATS);
+
+        const messagesRaw: MessageRaw[] = [];
+        const messagesRepository = new Set(
+          chatwootImport.getRepositoryMessagesCache(instance) ??
+            (
+              await this.repository.message.find({
+                select: { key: { id: 1 }, _id: 0 },
+                where: { owner: this.instance.name },
+              })
+            ).map((message) => message.key.id),
+        );
+
+        if (chatwootImport.getRepositoryMessagesCache(instance) === null) {
+          chatwootImport.setRepositoryMessagesCache(instance, messagesRepository);
+        }
+
+        for (const m of messages) {
+          if (!m.message || !m.key || !m.messageTimestamp) {
+            continue;
+          }
+
+          if (Long.isLong(m?.messageTimestamp)) {
+            m.messageTimestamp = m.messageTimestamp?.toNumber();
+          }
+
+          if (m.messageTimestamp <= timestampLimitToImport) {
+            continue;
+          }
+
+          if (messagesRepository.has(m.key.id)) {
+            continue;
+          }
+
+          messagesRaw.push({
+            key: m.key,
+            pushName: m.pushName || m.key.remoteJid.split('@')[0],
+            participant: m.participant,
+            message: { ...m.message },
+            messageType: getContentType(m.message),
+            messageTimestamp: m.messageTimestamp as number,
+            owner: this.instance.name,
+          });
+        }
+
+        this.logger.verbose('Sending data to webhook in event MESSAGES_SET');
+        this.sendDataWebhook(Events.MESSAGES_SET, [...messagesRaw]);
+
+        this.logger.verbose('Inserting messages in database');
+        await this.repository.message.insert(messagesRaw, this.instance.name, database.SAVE_DATA.NEW_MESSAGE);
+
+        if (this.localChatwoot.enabled && this.localChatwoot.import_messages && messagesRaw.length > 0) {
+          this.chatwootService.addHistoryMessages(
+            instance,
+            messagesRaw.filter((msg) => !chatwootImport.isIgnorePhoneNumber(msg.key?.remoteJid)),
+          );
+        }
+
+        messages = undefined;
+        chats = undefined;
+      } catch (error) {
+        this.logger.error(error);
       }
-
-      const messagesRaw: MessageRaw[] = [];
-      const messagesRepository = await this.repository.message.find({
-        where: { owner: this.instance.name },
-      });
-      for await (const [, m] of Object.entries(messages)) {
-        if (!m.message) {
-          continue;
-        }
-        if (messagesRepository.find((mr) => mr.owner === this.instance.name && mr.key.id === m.key.id)) {
-          continue;
-        }
-
-        if (Long.isLong(m?.messageTimestamp)) {
-          m.messageTimestamp = m.messageTimestamp?.toNumber();
-        }
-
-        messagesRaw.push({
-          key: m.key,
-          pushName: m.pushName,
-          participant: m.participant,
-          message: { ...m.message },
-          messageType: getContentType(m.message),
-          messageTimestamp: m.messageTimestamp as number,
-          owner: this.instance.name,
-        });
-      }
-
-      this.logger.verbose('Sending data to webhook in event MESSAGES_SET');
-      this.sendDataWebhook(Events.MESSAGES_SET, [...messagesRaw]);
-
-      messages = undefined;
     },
 
     'messages.upsert': async (
@@ -2215,6 +2321,35 @@ export class WAStartupService {
     }
   }
 
+  private historySyncNotification(msg: proto.Message.IHistorySyncNotification) {
+    const instance: InstanceDto = { instanceName: this.instance.name };
+
+    if (
+      this.localChatwoot.enabled &&
+      this.localChatwoot.import_messages &&
+      this.isSyncNotificationFromUsedSyncType(msg)
+    ) {
+      if (msg.chunkOrder === 1) {
+        this.chatwootService.startImportHistoryMessages(instance);
+      }
+
+      if (msg.progress === 100) {
+        setTimeout(() => {
+          this.chatwootService.importHistoryMessages(instance);
+        }, 10000);
+      }
+    }
+
+    return true;
+  }
+
+  private isSyncNotificationFromUsedSyncType(msg: proto.Message.IHistorySyncNotification) {
+    return (
+      (this.localSettings.sync_full_history && msg?.syncType === 2) ||
+      (!this.localSettings.sync_full_history && msg?.syncType === 3)
+    );
+  }
+
   private createJid(number: string): string {
     this.logger.verbose('Creating jid with number: ' + number);
 

src/whatsapp/types/wa.types.ts

@@ -65,6 +65,9 @@ export declare namespace wa {
     number?: string;
     reopen_conversation?: boolean;
     conversation_pending?: boolean;
+    import_contacts?: boolean;
+    import_messages?: boolean;
+    days_limit_import_messages?: number;
   };
 
   export type LocalSettings = {
@@ -74,6 +77,7 @@ export declare namespace wa {
     always_online?: boolean;
     read_messages?: boolean;
     read_status?: boolean;
+    sync_full_history?: boolean;
   };
 
   export type LocalWebsocket = {