Generate summaries for attachments

Author: exAspArk

backend/src/entities/file-attachment.ts

@@ -23,25 +23,31 @@ export class FileAttachment extends BaseEntity {
   @Property()
   size: number;
 
+  @Property()
+  summary: string;
+
   constructor({
     emailDraft,
     s3Key,
     filename,
     mimeType,
     size,
+    summary,
   }: {
     emailDraft: EmailDraft;
     s3Key: string;
     filename: string;
     mimeType: string;
     size: number;
+    summary: string;
   }) {
     super();
     this.emailDraft = emailDraft;
     this.s3Key = s3Key;
     this.filename = filename;
     this.mimeType = mimeType;
     this.size = size;
+    this.summary = summary;
     this.validate();
   }
 
@@ -65,5 +71,6 @@ export class FileAttachment extends BaseEntity {
     if (!this.mimeType) throw new Error('MIME type is required');
     if (this.size === undefined || this.size === null || this.size < 0)
       throw new Error('Size must be a non-negative number');
+    if (!this.summary) throw new Error('Summary is required');
   }
 }

backend/src/entities/gmail-attachment.ts

@@ -23,6 +23,7 @@ const MIME_TYPE_BY_FILE_EXTENSION: Record<string, string> = {
   docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
   xls: 'application/vnd.ms-excel',
   xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  ics: 'application/ics',
 };
 
 @Entity({ tableName: 'gmail_attachments' })
@@ -45,6 +46,9 @@ export class GmailAttachment extends BaseEntity {
   @Property()
   contentId?: string;
 
+  @Property()
+  summary?: string;
+
   constructor({
     gmailAccount,
     emailMessage,
@@ -73,6 +77,11 @@ export class GmailAttachment extends BaseEntity {
     this.validate();
   }
 
+  setSummary(summary: string) {
+    this.summary = summary;
+    this.validate();
+  }
+
   get derivedMimeType() {
     if (this.mimeType === 'application/octet-stream') {
       const ext = this.filename.toLowerCase().split('.').pop();
@@ -85,6 +94,13 @@ export class GmailAttachment extends BaseEntity {
     return this.mimeType;
   }
 
+  get llmMimeType() {
+    if (this.mimeType === 'application/ics') {
+      return 'text/plain'; // LLMs don't support text/calendar, so we treat it as plain text (e.g. for .ics files)
+    }
+    return this.derivedMimeType;
+  }
+
   static toJson(gmailAttachment: Loaded<GmailAttachment>) {
     return {
       id: gmailAttachment.id,
@@ -107,5 +123,6 @@ export class GmailAttachment extends BaseEntity {
     if (!this.mimeType) throw new Error('MIME type is required');
     if (this.size === undefined || this.size === null || this.size < 0)
       throw new Error('Size must be a non-negative number');
+    if (this.summary === '') throw new Error('Summary cannot be an empty string');
   }
 }

backend/src/migrations/.snapshot-bordly_dev.json

@@ -1019,6 +1019,16 @@
           "nullable": true,
           "length": 255,
           "mappedType": "string"
+        },
+        "summary": {
+          "name": "summary",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "length": 255,
+          "mappedType": "string"
         }
       },
       "name": "gmail_attachments",
@@ -1796,6 +1806,16 @@
           "primary": false,
           "nullable": false,
           "mappedType": "integer"
+        },
+        "summary": {
+          "name": "summary",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "length": 255,
+          "mappedType": "string"
         }
       },
       "name": "file_attachments",

backend/src/migrations/Migration20260304144014_add_summary_to_attachments.ts

@@ -0,0 +1,15 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20260304144014_add_summary_to_attachments extends Migration {
+  override async up(): Promise<void> {
+    this.addSql(`alter table "gmail_attachments" add column "summary" varchar(255) null;`);
+
+    this.addSql(`alter table "file_attachments" add column "summary" varchar(255) not null;`);
+  }
+
+  override async down(): Promise<void> {
+    this.addSql(`alter table "gmail_attachments" drop column "summary";`);
+
+    this.addSql(`alter table "file_attachments" drop column "summary";`);
+  }
+}

backend/src/services/agent.service.ts

@@ -31,7 +31,13 @@ export interface Context {
   userComment: Loaded<Comment>;
 }
 
-const BORDLY_AGENT = {
+const AGENT_ATTACHMENT_SUMMARY = {
+  name: 'Gmail Attachment Summary Agent',
+  instructions: `Analyze the email attachment and provide a concise summary. The summary must be 255 characters or less.`,
+  model: ENV.LLM_FAST_MODEL,
+};
+
+const AGENT_BORDLY = {
   name: 'Bordly',
   instructions: `You are an AI email assistant called Bordly that helps manage email communications within a Trello-like board card using the provided tools.
 
@@ -105,7 +111,7 @@ export class AgentService {
     requestContext.set('boardCard', populatedBoardCard);
     requestContext.set('userComment', userComment);
 
-    const agent = AgentService.createAgent(BORDLY_AGENT);
+    const agent = AgentService.createAgent(AGENT_BORDLY);
     const prompt = userComment.contentText;
     const messages: MessageInput[] = [...systemInstructions, { role: 'user', content: prompt }];
 
@@ -160,4 +166,29 @@ export class AgentService {
 
     return { instructions, boardCard };
   }
+
+  static async generateAttachmentSummary({
+    data,
+    filename,
+    mimeType,
+  }: {
+    data: Buffer;
+    filename: string;
+    mimeType: string;
+  }) {
+    const agent = AgentService.createAgent(AGENT_ATTACHMENT_SUMMARY);
+
+    console.log(`[AGENT] Generating summary for attachment ${filename}...`);
+    const response = await agent.generate([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Summarize the following email attachment' },
+          { type: 'file', filename, mediaType: mimeType, data },
+        ],
+      },
+    ]);
+
+    return response.text.slice(0, 255);
+  }
 }

backend/src/services/email-draft.service.ts

@@ -204,7 +204,7 @@ export class EmailDraftService {
     });
 
     const messageData = await GmailApi.getMessage(gmail, sentMessage.id!);
-    const emailMessage = EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
+    const emailMessage = await EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
     if (domain) {
       emailMessage.domain = domain;
     } else {

backend/src/services/email-message.service.ts

@@ -14,6 +14,7 @@ import { BoardMemberService } from '@/services/board-member.service';
 import { DomainService } from '@/services/domain.service';
 import { EmailDraftService } from '@/services/email-draft.service';
 import { GmailAccountService } from '@/services/gmail-account.service';
+import { GmailAttachmentService } from '@/services/gmail-attachment.service';
 import { htmlToText } from '@/utils/email';
 import { ENV } from '@/utils/env';
 import { reportError } from '@/utils/error-tracking';
@@ -52,6 +53,7 @@ const AGENT_CATEGORIZATION = {
 - {{categories}}
 
 Only output one of the above categories without any explanation.`,
+  model: ENV.LLM_FAST_MODEL,
 };
 
 export class EmailMessageService {
@@ -198,7 +200,10 @@ export class EmailMessageService {
     // Collect EmailMessages and Attachments
     let lastExternalHistoryId: string | undefined;
     // Return type from parseEmailMessage
-    const emailMessagesDescByThreadId: Record<string, ReturnType<typeof EmailMessageService.parseEmailMessage>[]> = {};
+    const emailMessagesDescByThreadId: Record<
+      string,
+      Awaited<ReturnType<typeof EmailMessageService.parseEmailMessage>>[]
+    > = {};
     const domainNames = new Set<string>();
     const processedMessageIds = new Set<string>();
 
@@ -208,7 +213,7 @@ export class EmailMessageService {
       const messageData = await GmailApi.getMessage(gmail, message.id);
       if (messageData.labelIds?.includes(LABEL.DRAFT)) continue; // Skip drafts
 
-      const emailMessage = EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
+      const emailMessage = await EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
       (emailMessagesDescByThreadId[emailMessage.externalThreadId] ??= []).push(emailMessage);
       domainNames.add(emailMessage.loadedDomain.name);
       processedMessageIds.add(message.id);
@@ -232,7 +237,7 @@ export class EmailMessageService {
           if (messageData.labelIds?.includes(LABEL.DRAFT)) continue; // Skip drafts
 
           console.log(`[GMAIL] Processing ${gmailAccount.email} thread message ${messageData.id}...`);
-          const emailMessage = EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
+          const emailMessage = await EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
           (emailMessagesDescByThreadId[emailMessage.externalThreadId] ??= []).push(emailMessage);
           domainNames.add(emailMessage.loadedDomain.name);
           processedMessageIds.add(messageData.id);
@@ -343,7 +348,7 @@ export class EmailMessageService {
     return participant;
   }
 
-  static parseEmailMessage({
+  static async parseEmailMessage({
     gmailAccount,
     messageData,
   }: {
@@ -416,6 +421,17 @@ export class EmailMessageService {
         size: attachmentData.size,
         contentId: attachmentData.contentId,
       });
+
+      if (!labels.includes(LABEL.SPAM) && !labels.includes(LABEL.TRASH)) {
+        const data = await GmailAttachmentService.getAttachmentDataBuffer(attachment);
+        const summary = await AgentService.generateAttachmentSummary({
+          filename: attachment.filename,
+          mimeType: attachment.llmMimeType,
+          data,
+        });
+        attachment.setSummary(summary);
+      }
+
       attachments.push(attachment);
     }
 
@@ -523,7 +539,7 @@ export class EmailMessageService {
           continue; // Skip drafts
         }
 
-        const emailMessage = EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
+        const emailMessage = await EmailMessageService.parseEmailMessage({ gmailAccount, messageData });
         const boardCard = boardCardByThreadId[externalThreadId];
 
         if (!boardCard && !EmailMessageService.boardAccountToSyncWhenNoBoardCard({ emailMessage, gmailAccount })) {
@@ -779,7 +795,7 @@ export class EmailMessageService {
       instructions: renderTemplate(AGENT_CATEGORIZATION.instructionsTemplate, { categories: categories.join('\n- ') }),
     });
 
-    let category = CATEGORIES.OTHER;
+    let category = categories[0]!;
     if (
       !emailMessages.some(
         (emailMessage) => emailMessage.labels.includes(LABEL.SPAM) || emailMessage.labels.includes(LABEL.TRASH),

backend/src/services/file-attachment.service.ts

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import type { Loaded } from '@mikro-orm/postgresql';
 import type { EmailDraft } from '@/entities/email-draft';
 import { FileAttachment } from '@/entities/file-attachment';
+import { AgentService } from '@/services/agent.service';
 import { orm } from '@/utils/orm';
 import { S3Client } from '@/utils/s3-client';
 
@@ -23,12 +24,15 @@ export class FileAttachmentService {
     const s3Key = `${PREFIX_EMAIL_DRAFTS}/${emailDraft.id}/${randomUUID()}-${filename}`;
     await S3Client.uploadFile({ key: s3Key, buffer, contentType: mimeType });
 
+    const summary = await AgentService.generateAttachmentSummary({ data: buffer, filename, mimeType });
+
     const draftAttachment = new FileAttachment({
       emailDraft,
       s3Key,
       filename,
       mimeType,
       size: buffer.length,
+      summary,
     });
 
     orm.em.persist(draftAttachment);

backend/src/tools/gmail-attachment-analyze.tool.ts

@@ -29,6 +29,7 @@ export const gmailAttachmentAnalyzeTool = createTool({
 
     const { externalThreadId } = boardCard;
     if (!externalThreadId) throw new Error('Board card does not have an external thread ID');
+
     const gmailAttachment = await GmailAttachmentService.findByIdAndExternalThreadId(id, {
       externalThreadId,
       populate: ['emailMessage.gmailAccount'],
@@ -44,7 +45,7 @@ export const gmailAttachmentAnalyzeTool = createTool({
           {
             type: 'file',
             filename: gmailAttachment.filename,
-            mediaType: gmailAttachment.derivedMimeType,
+            mediaType: gmailAttachment.llmMimeType,
             data: await GmailAttachmentService.getAttachmentDataBuffer(gmailAttachment),
           },
         ],
@@ -54,7 +55,7 @@ export const gmailAttachmentAnalyzeTool = createTool({
     const result = {
       output: response.text,
       filename: gmailAttachment.filename,
-      mimeType: gmailAttachment.derivedMimeType,
+      mimeType: gmailAttachment.llmMimeType,
       size: gmailAttachment.size,
     };
 

frontend/src/components/board-card/email-message-card.tsx

@@ -194,8 +194,8 @@ export const EmailMessageCard = ({
           </AvatarFallback>
         </Avatar>
         <div className="flex flex-col w-full min-w-0">
-          <div className="flex justify-between items-start gap-2">
-            <div className="flex flex-col">
+          <div className="flex justify-between items-start gap-16">
+            <div className="flex flex-col min-w-0">
               {firstParticipantName === firstParticipant.email ? (
                 <div className="text-sm font-medium">{firstParticipantName}</div>
               ) : (
@@ -263,7 +263,7 @@ export const EmailMessageCard = ({
                 </Popover>
               </div>
             </div>
-            <div className="flex gap-3">
+            <div className="flex gap-3 flex-shrink-0">
               {blockedTrackerDomains.length > 0 && (
                 <Popover>
                   <PopoverTrigger asChild>