Add LLM chat history widget (#3907)

Co-authored-by: github-actions <github-actions@github.com>

src/components/graph/widgets/ChatHistoryWidget.vue

@@ -0,0 +1,135 @@
+<template>
+  <ScrollPanel
+    ref="scrollPanelRef"
+    class="w-full min-h-[400px] rounded-lg px-2 py-2 text-xs"
+    :pt="{ content: { id: 'chat-scroll-content' } }"
+  >
+    <div v-for="(item, i) in parsedHistory" :key="i" class="mb-4">
+      <!-- Prompt (user, right) -->
+      <span
+        :class="{
+          'opacity-40 pointer-events-none': editIndex !== null && i > editIndex
+        }"
+      >
+        <div class="flex justify-end mb-1">
+          <div
+            class="bg-gray-300 dark-theme:bg-gray-800 rounded-xl px-4 py-1 max-w-[80%] text-right"
+          >
+            <div class="break-words text-[12px]">{{ item.prompt }}</div>
+          </div>
+        </div>
+        <div class="flex justify-end mb-2 mr-1">
+          <CopyButton :text="item.prompt" />
+          <Button
+            v-tooltip="
+              editIndex === i ? $t('chatHistory.cancelEditTooltip') : null
+            "
+            text
+            rounded
+            class="!p-1 !h-4 !w-4 text-gray-400 hover:text-gray-600 dark-theme:hover:text-gray-200 transition"
+            pt:icon:class="!text-xs"
+            :icon="editIndex === i ? 'pi pi-times' : 'pi pi-pencil'"
+            :aria-label="
+              editIndex === i ? $t('chatHistory.cancelEdit') : $t('g.edit')
+            "
+            @click="editIndex === i ? handleCancelEdit() : handleEdit(i)"
+          />
+        </div>
+      </span>
+      <!-- Response (LLM, left) -->
+      <ResponseBlurb
+        :text="item.response"
+        :class="{
+          'opacity-25 pointer-events-none': editIndex !== null && i >= editIndex
+        }"
+      >
+        <div v-html="nl2br(linkifyHtml(item.response))" />
+      </ResponseBlurb>
+    </div>
+  </ScrollPanel>
+</template>
+
+<script setup lang="ts">
+import Button from 'primevue/button'
+import ScrollPanel from 'primevue/scrollpanel'
+import { computed, nextTick, ref, watch } from 'vue'
+
+import CopyButton from '@/components/graph/widgets/chatHistory/CopyButton.vue'
+import ResponseBlurb from '@/components/graph/widgets/chatHistory/ResponseBlurb.vue'
+import { ComponentWidget } from '@/scripts/domWidget'
+import { linkifyHtml, nl2br } from '@/utils/formatUtil'
+
+const { widget, history = '[]' } = defineProps<{
+  widget?: ComponentWidget<string>
+  history: string
+}>()
+
+const editIndex = ref<number | null>(null)
+const scrollPanelRef = ref<InstanceType<typeof ScrollPanel> | null>(null)
+
+const parsedHistory = computed(() => JSON.parse(history || '[]'))
+
+const findPromptInput = () =>
+  widget?.node.widgets?.find((w) => w.name === 'prompt')
+let promptInput = findPromptInput()
+const previousPromptInput = ref<string | null>(null)
+
+const getPreviousResponseId = (index: number) =>
+  index > 0 ? parsedHistory.value[index - 1]?.response_id ?? '' : ''
+
+const storePromptInput = () => {
+  promptInput ??= widget?.node.widgets?.find((w) => w.name === 'prompt')
+  if (!promptInput) return
+
+  previousPromptInput.value = String(promptInput.value)
+}
+
+const setPromptInput = (text: string, previousResponseId?: string | null) => {
+  promptInput ??= widget?.node.widgets?.find((w) => w.name === 'prompt')
+  if (!promptInput) return
+
+  if (previousResponseId !== null) {
+    promptInput.value = `<starting_point_id:${previousResponseId}>\n\n${text}`
+  } else {
+    promptInput.value = text
+  }
+}
+
+const handleEdit = (index: number) => {
+  if (!promptInput) return
+
+  editIndex.value = index
+  const prevResponseId = index === 0 ? 'start' : getPreviousResponseId(index)
+  const promptText = parsedHistory.value[index]?.prompt ?? ''
+
+  storePromptInput()
+  setPromptInput(promptText, prevResponseId)
+}
+
+const resetEditingState = () => {
+  editIndex.value = null
+}
+const handleCancelEdit = () => {
+  resetEditingState()
+  if (promptInput) {
+    promptInput.value = previousPromptInput.value ?? ''
+  }
+}
+
+const scrollChatToBottom = () => {
+  const content = document.getElementById('chat-scroll-content')
+  if (content) {
+    content.scrollTo({ top: content.scrollHeight, behavior: 'smooth' })
+  }
+}
+
+const onHistoryChanged = () => {
+  resetEditingState()
+  void nextTick(() => scrollChatToBottom())
+}
+
+watch(() => parsedHistory.value, onHistoryChanged, {
+  immediate: true,
+  deep: true
+})
+</script>

src/components/graph/widgets/DomWidget.vue

@@ -11,6 +11,7 @@
       v-if="isComponentWidget(widget)"
       :model-value="widget.value"
       :widget="widget"
+      v-bind="widget.props"
       @update:model-value="emit('update:widgetValue', $event)"
     />
   </div>

src/components/graph/widgets/chatHistory/CopyButton.vue

@@ -0,0 +1,36 @@
+<template>
+  <Button
+    v-tooltip="
+      copied ? $t('chatHistory.copiedTooltip') : $t('chatHistory.copyTooltip')
+    "
+    text
+    rounded
+    class="!p-1 !h-4 !w-6 text-gray-400 hover:text-gray-600 dark-theme:hover:text-gray-200 transition"
+    pt:icon:class="!text-xs"
+    :icon="copied ? 'pi pi-check' : 'pi pi-copy'"
+    :aria-label="
+      copied ? $t('chatHistory.copiedTooltip') : $t('chatHistory.copyTooltip')
+    "
+    @click="handleCopy"
+  />
+</template>
+
+<script setup lang="ts">
+import Button from 'primevue/button'
+import { ref } from 'vue'
+
+const { text } = defineProps<{
+  text: string
+}>()
+
+const copied = ref(false)
+
+const handleCopy = async () => {
+  if (!text) return
+  await navigator.clipboard.writeText(text)
+  copied.value = true
+  setTimeout(() => {
+    copied.value = false
+  }, 1024)
+}
+</script>

src/components/graph/widgets/chatHistory/ResponseBlurb.vue

@@ -0,0 +1,22 @@
+<template>
+  <span>
+    <div class="flex justify-start mb-1">
+      <div class="rounded-xl px-4 py-1 max-w-[80%]">
+        <div class="break-words text-[12px]">
+          <slot />
+        </div>
+      </div>
+    </div>
+    <div class="flex justify-start ml-1">
+      <CopyButton :text="text" />
+    </div>
+  </span>
+</template>
+
+<script setup lang="ts">
+import CopyButton from '@/components/graph/widgets/chatHistory/CopyButton.vue'
+
+defineProps<{
+  text: string
+}>()
+</script>

src/composables/node/useNodeChatHistory.ts

@@ -0,0 +1,60 @@
+import { LGraphNode } from '@comfyorg/litegraph'
+
+import type ChatHistoryWidget from '@/components/graph/widgets/ChatHistoryWidget.vue'
+import { useChatHistoryWidget } from '@/composables/widgets/useChatHistoryWidget'
+
+const CHAT_HISTORY_WIDGET_NAME = '$$node-chat-history'
+
+/**
+ * Composable for handling node text previews
+ */
+export function useNodeChatHistory(
+  options: {
+    minHeight?: number
+    props?: Omit<InstanceType<typeof ChatHistoryWidget>['$props'], 'widget'>
+  } = {}
+) {
+  const chatHistoryWidget = useChatHistoryWidget(options)
+
+  const findChatHistoryWidget = (node: LGraphNode) =>
+    node.widgets?.find((w) => w.name === CHAT_HISTORY_WIDGET_NAME)
+
+  const addChatHistoryWidget = (node: LGraphNode) =>
+    chatHistoryWidget(node, {
+      name: CHAT_HISTORY_WIDGET_NAME,
+      type: 'chatHistory'
+    })
+
+  /**
+   * Shows chat history for a node
+   * @param node The graph node to show the chat history for
+   */
+  function showChatHistory(node: LGraphNode) {
+    if (!findChatHistoryWidget(node)) {
+      addChatHistoryWidget(node)
+    }
+    node.setDirtyCanvas?.(true)
+  }
+
+  /**
+   * Removes chat history from a node
+   * @param node The graph node to remove the chat history from
+   */
+  function removeChatHistory(node: LGraphNode) {
+    if (!node.widgets) return
+
+    const widgetIdx = node.widgets.findIndex(
+      (w) => w.name === CHAT_HISTORY_WIDGET_NAME
+    )
+
+    if (widgetIdx > -1) {
+      node.widgets[widgetIdx].onRemove?.()
+      node.widgets.splice(widgetIdx, 1)
+    }
+  }
+
+  return {
+    showChatHistory,
+    removeChatHistory
+  }
+}

src/composables/widgets/useChatHistoryWidget.ts

@@ -0,0 +1,43 @@
+import type { LGraphNode } from '@comfyorg/litegraph'
+import { ref } from 'vue'
+
+import ChatHistoryWidget from '@/components/graph/widgets/ChatHistoryWidget.vue'
+import type { InputSpec } from '@/schemas/nodeDef/nodeDefSchemaV2'
+import { ComponentWidgetImpl, addWidget } from '@/scripts/domWidget'
+import type { ComfyWidgetConstructorV2 } from '@/scripts/widgets'
+
+const PADDING = 16
+
+export const useChatHistoryWidget = (
+  options: {
+    props?: Omit<InstanceType<typeof ChatHistoryWidget>['$props'], 'widget'>
+  } = {}
+) => {
+  const widgetConstructor: ComfyWidgetConstructorV2 = (
+    node: LGraphNode,
+    inputSpec: InputSpec
+  ) => {
+    const widgetValue = ref<string>('')
+    const widget = new ComponentWidgetImpl<
+      string | object,
+      InstanceType<typeof ChatHistoryWidget>['$props']
+    >({
+      node,
+      name: inputSpec.name,
+      component: ChatHistoryWidget,
+      props: options.props,
+      inputSpec,
+      options: {
+        getValue: () => widgetValue.value,
+        setValue: (value: string | object) => {
+          widgetValue.value = typeof value === 'string' ? value : String(value)
+        },
+        getMinHeight: () => 400 + PADDING
+      }
+    })
+    addWidget(node, widget)
+    return widget
+  }
+
+  return widgetConstructor
+}

src/composables/widgets/useProgressTextWidget.ts

@@ -8,7 +8,11 @@ import type { ComfyWidgetConstructorV2 } from '@/scripts/widgets'
 
 const PADDING = 16
 
-export const useTextPreviewWidget = () => {
+export const useTextPreviewWidget = (
+  options: {
+    minHeight?: number
+  } = {}
+) => {
   const widgetConstructor: ComfyWidgetConstructorV2 = (
     node: LGraphNode,
     inputSpec: InputSpec
@@ -24,7 +28,7 @@ export const useTextPreviewWidget = () => {
         setValue: (value: string | object) => {
           widgetValue.value = typeof value === 'string' ? value : String(value)
         },
-        getMinHeight: () => 42 + PADDING
+        getMinHeight: () => options.minHeight ?? 42 + PADDING
       }
     })
     addWidget(node, widget)

src/locales/en/main.json

@@ -114,7 +114,9 @@
     "learnMore": "Learn more",
     "amount": "Amount",
     "unknownError": "Unknown error",
-    "title": "Title"
+    "title": "Title",
+    "edit": "Edit",
+    "copy": "Copy"
   },
   "manager": {
     "title": "Custom Nodes Manager",
@@ -1387,5 +1389,12 @@
       "tooltip": "Execute to selected output nodes (Highlighted with orange border)",
       "disabledTooltip": "No output nodes selected"
     }
+  },
+  "chatHistory": {
+    "cancelEdit": "Cancel",
+    "editTooltip": "Edit message",
+    "cancelEditTooltip": "Cancel edit",
+    "copiedTooltip": "Copied",
+    "copyTooltip": "Copy message to clipboard"
   }
 }

src/locales/es/main.json

@@ -82,6 +82,13 @@
       "title": "Crea una cuenta"
     }
   },
+  "chatHistory": {
+    "cancelEdit": "Cancelar",
+    "cancelEditTooltip": "Cancelar edición",
+    "copiedTooltip": "Copiado",
+    "copyTooltip": "Copiar mensaje al portapapeles",
+    "editTooltip": "Editar mensaje"
+  },
   "clipboard": {
     "errorMessage": "Error al copiar al portapapeles",
     "errorNotSupported": "API del portapapeles no soportada en su navegador",
@@ -257,6 +264,7 @@
     "continue": "Continuar",
     "control_after_generate": "control después de generar",
     "control_before_generate": "control antes de generar",
+    "copy": "Copiar",
     "copyToClipboard": "Copiar al portapapeles",
     "currentUser": "Usuario actual",
     "customize": "Personalizar",
@@ -268,6 +276,7 @@
     "disableAll": "Deshabilitar todo",
     "disabling": "Deshabilitando",
     "download": "Descargar",
+    "edit": "Editar",
     "empty": "Vacío",
     "enableAll": "Habilitar todo",
     "enabled": "Habilitado",

src/locales/fr/main.json

@@ -82,6 +82,13 @@
       "title": "Créer un compte"
     }
   },
+  "chatHistory": {
+    "cancelEdit": "Annuler",
+    "cancelEditTooltip": "Annuler la modification",
+    "copiedTooltip": "Copié",
+    "copyTooltip": "Copier le message dans le presse-papiers",
+    "editTooltip": "Modifier le message"
+  },
   "clipboard": {
     "errorMessage": "Échec de la copie dans le presse-papiers",
     "errorNotSupported": "L'API du presse-papiers n'est pas prise en charge par votre navigateur",
@@ -257,6 +264,7 @@
     "continue": "Continuer",
     "control_after_generate": "contrôle après génération",
     "control_before_generate": "contrôle avant génération",
+    "copy": "Copier",
     "copyToClipboard": "Copier dans le presse-papiers",
     "currentUser": "Utilisateur actuel",
     "customize": "Personnaliser",
@@ -268,6 +276,7 @@
     "disableAll": "Désactiver tout",
     "disabling": "Désactivation",
     "download": "Télécharger",
+    "edit": "Modifier",
     "empty": "Vide",
     "enableAll": "Activer tout",
     "enabled": "Activé",

src/locales/ja/main.json

@@ -82,6 +82,13 @@
       "title": "アカウントを作成する"
     }
   },
+  "chatHistory": {
+    "cancelEdit": "キャンセル",
+    "cancelEditTooltip": "編集をキャンセル",
+    "copiedTooltip": "コピーしました",
+    "copyTooltip": "メッセージをクリップボードにコピー",
+    "editTooltip": "メッセージを編集"
+  },
   "clipboard": {
     "errorMessage": "クリップボードへのコピーに失敗しました",
     "errorNotSupported": "お使いのブラウザではクリップボードAPIがサポートされていません",
@@ -257,6 +264,7 @@
     "continue": "続ける",
     "control_after_generate": "生成後の制御",
     "control_before_generate": "生成前の制御",
+    "copy": "コピー",
     "copyToClipboard": "クリップボードにコピー",
     "currentUser": "現在のユーザー",
     "customize": "カスタマイズ",
@@ -268,6 +276,7 @@
     "disableAll": "すべて無効にする",
     "disabling": "無効化",
     "download": "ダウンロード",
+    "edit": "編集",
     "empty": "空",
     "enableAll": "すべて有効にする",
     "enabled": "有効",

src/locales/ko/main.json

@@ -82,6 +82,13 @@
       "title": "계정 생성"
     }
   },
+  "chatHistory": {
+    "cancelEdit": "취소",
+    "cancelEditTooltip": "편집 취소",
+    "copiedTooltip": "복사됨",
+    "copyTooltip": "메시지를 클립보드에 복사",
+    "editTooltip": "메시지 편집"
+  },
   "clipboard": {
     "errorMessage": "클립보드에 복사하지 못했습니다",
     "errorNotSupported": "브라우저가 클립보드 API를 지원하지 않습니다.",
@@ -257,6 +264,7 @@
     "continue": "계속",
     "control_after_generate": "생성 후 제어",
     "control_before_generate": "생성 전 제어",
+    "copy": "복사",
     "copyToClipboard": "클립보드에 복사",
     "currentUser": "현재 사용자",
     "customize": "사용자 정의",
@@ -268,6 +276,7 @@
     "disableAll": "모두 비활성화",
     "disabling": "비활성화 중",
     "download": "다운로드",
+    "edit": "편집",
     "empty": "비어 있음",
     "enableAll": "모두 활성화",
     "enabled": "활성화됨",

src/locales/ru/main.json

@@ -82,6 +82,13 @@
       "title": "Создать аккаунт"
     }
   },
+  "chatHistory": {
+    "cancelEdit": "Отмена",
+    "cancelEditTooltip": "Отменить редактирование",
+    "copiedTooltip": "Скопировано",
+    "copyTooltip": "Скопировать сообщение в буфер",
+    "editTooltip": "Редактировать сообщение"
+  },
   "clipboard": {
     "errorMessage": "Не удалось скопировать в буфер обмена",
     "errorNotSupported": "API буфера обмена не поддерживается в вашем браузере",
@@ -257,6 +264,7 @@
     "continue": "Продолжить",
     "control_after_generate": "управление после генерации",
     "control_before_generate": "управление до генерации",
+    "copy": "Копировать",
     "copyToClipboard": "Скопировать в буфер обмена",
     "currentUser": "Текущий пользователь",
     "customize": "Настроить",
@@ -268,6 +276,7 @@
     "disableAll": "Отключить все",
     "disabling": "Отключение",
     "download": "Скачать",
+    "edit": "Редактировать",
     "empty": "Пусто",
     "enableAll": "Включить все",
     "enabled": "Включено",

src/locales/zh/main.json

@@ -82,6 +82,13 @@
       "title": "创建一个账户"
     }
   },
+  "chatHistory": {
+    "cancelEdit": "取消",
+    "cancelEditTooltip": "取消编辑",
+    "copiedTooltip": "已复制",
+    "copyTooltip": "复制消息到剪贴板",
+    "editTooltip": "编辑消息"
+  },
   "clipboard": {
     "errorMessage": "复制到剪贴板失败",
     "errorNotSupported": "您的浏览器不支持剪贴板API",
@@ -257,6 +264,7 @@
     "continue": "继续",
     "control_after_generate": "生成后控制",
     "control_before_generate": "生成前控制",
+    "copy": "复制",
     "copyToClipboard": "复制到剪贴板",
     "currentUser": "当前用户",
     "customize": "自定义",
@@ -268,6 +276,7 @@
     "disableAll": "禁用全部",
     "disabling": "禁用中",
     "download": "下载",
+    "edit": "编辑",
     "empty": "空",
     "enableAll": "启用全部",
     "enabled": "已启用",

src/schemas/apiSchema.ts

@@ -87,6 +87,12 @@ const zProgressTextWsMessage = z.object({
   text: z.string()
 })
 
+const zDisplayComponentWsMessage = z.object({
+  node_id: zNodeId,
+  component: z.enum(['ChatHistoryWidget']),
+  props: z.record(z.string(), z.any()).optional()
+})
+
 const zTerminalSize = z.object({
   cols: z.number(),
   row: z.number()
@@ -120,6 +126,9 @@ export type ExecutionInterruptedWsMessage = z.infer<
 export type ExecutionErrorWsMessage = z.infer<typeof zExecutionErrorWsMessage>
 export type LogsWsMessage = z.infer<typeof zLogsWsMessage>
 export type ProgressTextWsMessage = z.infer<typeof zProgressTextWsMessage>
+export type DisplayComponentWsMessage = z.infer<
+  typeof zDisplayComponentWsMessage
+>
 // End of ws messages
 
 const zPromptInputItem = z.object({

src/scripts/api.ts

@@ -1,6 +1,7 @@
 import axios from 'axios'
 
 import type {
+  DisplayComponentWsMessage,
   EmbeddingsResponse,
   ExecutedWsMessage,
   ExecutingWsMessage,
@@ -103,6 +104,7 @@ interface BackendApiCalls {
   /** Binary preview/progress data */
   b_preview: Blob
   progress_text: ProgressTextWsMessage
+  display_component: DisplayComponentWsMessage
 }
 
 /** Dictionary of all api calls */

src/scripts/domWidget.ts

@@ -47,10 +47,13 @@ export interface DOMWidget<T extends HTMLElement, V extends object | string>
 /**
  * A DOM widget that wraps a Vue component as a litegraph widget.
  */
-export interface ComponentWidget<V extends object | string>
-  extends BaseDOMWidget<V> {
+export interface ComponentWidget<
+  V extends object | string,
+  P = Record<string, unknown>
+> extends BaseDOMWidget<V> {
   readonly component: Component
   readonly inputSpec: InputSpec
+  readonly props?: P
 }
 
 export interface DOMWidgetOptions<V extends object | string>
@@ -217,18 +220,23 @@ export class DOMWidgetImpl<T extends HTMLElement, V extends object | string>
   }
 }
 
-export class ComponentWidgetImpl<V extends object | string>
+export class ComponentWidgetImpl<
+    V extends object | string,
+    P = Record<string, unknown>
+  >
   extends BaseDOMWidgetImpl<V>
-  implements ComponentWidget<V>
+  implements ComponentWidget<V, P>
 {
   readonly component: Component
   readonly inputSpec: InputSpec
+  readonly props?: P
 
   constructor(obj: {
     node: LGraphNode
     name: string
     component: Component
     inputSpec: InputSpec
+    props?: P
     options: DOMWidgetOptions<V>
   }) {
     super({
@@ -237,6 +245,7 @@ export class ComponentWidgetImpl<V extends object | string>
     })
     this.component = obj.component
     this.inputSpec = obj.inputSpec
+    this.props = obj.props
   }
 
   override computeLayoutSize() {

src/stores/executionStore.ts

@@ -1,8 +1,11 @@
 import { defineStore } from 'pinia'
 import { computed, ref } from 'vue'
 
+import type ChatHistoryWidget from '@/components/graph/widgets/ChatHistoryWidget.vue'
+import { useNodeChatHistory } from '@/composables/node/useNodeChatHistory'
 import { useNodeProgressText } from '@/composables/node/useNodeProgressText'
 import type {
+  DisplayComponentWsMessage,
   ExecutedWsMessage,
   ExecutionCachedWsMessage,
   ExecutionErrorWsMessage,
@@ -107,6 +110,10 @@ export const useExecutionStore = defineStore('execution', () => {
     )
   }
   api.addEventListener('progress_text', handleProgressText as EventListener)
+  api.addEventListener(
+    'display_component',
+    handleDisplayComponent as EventListener
+  )
 
   function unbindExecutionEvents() {
     api.removeEventListener(
@@ -195,6 +202,21 @@ export const useExecutionStore = defineStore('execution', () => {
     useNodeProgressText().showTextPreview(node, text)
   }
 
+  function handleDisplayComponent(e: CustomEvent<DisplayComponentWsMessage>) {
+    const { node_id, component, props = {} } = e.detail
+    const node = app.graph.getNodeById(node_id)
+    if (!node) return
+
+    if (component === 'ChatHistoryWidget') {
+      useNodeChatHistory({
+        props: props as Omit<
+          InstanceType<typeof ChatHistoryWidget>['$props'],
+          'widget'
+        >
+      }).showChatHistory(node)
+    }
+  }
+
   function storePrompt({
     nodes,
     id,