From 2991c074671a683001aaad0bfffdbd066f0bdfd5 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Tue, 9 Apr 2024 21:38:47 +0800 Subject: [PATCH] Fix share page whisper auth (#1161) Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com> --- .../content/docs/development/configuration.md | 2 +- .../development/custom-models/bge-rerank.md | 2 +- .../content/docs/development/upgrading/471.md | 8 +- .../content/docs/workflow/examples/dalle3.md | 2 +- .../docs/workflow/examples/feishu_webhook.md | 2 +- .../docs/workflow/examples/google_search.md | 2 +- docSite/content/docs/workflow/modules/laf.md | 2 - packages/global/common/string/textSplitter.ts | 4 +- packages/global/common/string/tools.ts | 2 + .../core/module/template/system/userGuide.ts | 5 +- packages/global/core/module/template/tip.ts | 1 - packages/global/support/user/team/type.d.ts | 1 + packages/global/support/wallet/bill/type.d.ts | 1 + packages/service/common/file/image/schema.ts | 1 + .../common/vectorStore/controller.d.ts | 11 +- .../common/vectorStore/pg/controller.ts | 34 +++--- .../core/dataset/collection/controller.ts | 67 ++++++++--- packages/service/core/dataset/controller.ts | 28 ++++- .../service/core/dataset/data/controller.ts | 2 - packages/service/core/dataset/data/schema.ts | 9 +- .../service/core/dataset/search/controller.ts | 1 + .../core/dataset/training/controller.ts | 1 - .../service/core/dataset/training/schema.ts | 4 +- .../service/support/user/team/teamSchema.ts | 3 + .../web/components/common/Icon/constants.ts | 1 + .../Icon/icons/core/modules/fixview.svg | 5 + projects/app/public/docs/versionIntro.md | 2 +- projects/app/public/js/iframe.js | 2 +- projects/app/public/locales/en/common.json | 6 +- projects/app/public/locales/zh/common.json | 10 +- .../ChatBox/components/ChatItem.tsx | 2 +- .../components/core/module/Flow/ChatTest.tsx | 21 +++- .../core/module/Flow/FlowProvider.tsx | 2 +- .../Flow/components/nodes/NodeUserGuide.tsx | 28 +++++ .../Flow/components/render/NodeCard.tsx | 2 +- .../src/components/core/module/Flow/index.tsx | 57 ++++++++-- .../support/laf/LafAccountModal.tsx | 5 +- projects/app/src/pages/api/admin/initv471.ts | 29 +++++ .../api/core/dataset/collection/delete.ts | 2 +- .../pages/api/core/dataset/data/insertData.ts | 1 + .../src/pages/api/core/dataset/data/list.ts | 3 +- .../src/pages/api/v1/audio/transcriptions.ts | 18 +-- .../app/detail/components/FlowEdit/Header.tsx | 104 ++++++++---------- .../OutLink/SelectUsingWayModal.tsx | 2 +- .../src/pages/chat/components/ToolMenu.tsx | 2 +- projects/app/src/pages/plugin/edit/Header.tsx | 81 +++++++------- .../app/src/service/common/system/cronTask.ts | 46 +++++--- .../src/service/core/dataset/data/utils.ts | 3 + projects/app/src/web/common/api/lafRequest.ts | 60 +++++++--- .../app/src/web/common/hooks/useSpeech.ts | 6 + projects/app/src/web/core/app/templates.ts | 8 +- projects/app/src/web/core/app/utils.ts | 2 +- projects/app/src/web/styles/reactflow.scss | 6 +- projects/app/src/web/support/laf/api.ts | 2 +- python/bge-rerank/README.md | 2 +- 55 files changed, 465 insertions(+), 250 deletions(-) delete mode 100644 packages/service/core/dataset/data/controller.ts create mode 100644 packages/web/components/common/Icon/icons/core/modules/fixview.svg create mode 100644 projects/app/src/pages/api/admin/initv471.ts diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md index 88604cd3d48..cf700392b6b 100644 --- a/docSite/content/docs/development/configuration.md +++ b/docSite/content/docs/development/configuration.md @@ -20,7 +20,7 @@ llm模型全部合并 ```json { "feConfigs": { - "lafEnv": "https://laf.dev" // laf环境 + "lafEnv": "https://laf.dev" // laf环境。 https://laf.run （杭州阿里云） ,或者私有化的laf环境。如果使用 Laf openapi 功能，需要最新版的 laf 。 }, "systemEnv": { "vectorMaxProcess": 15, diff --git a/docSite/content/docs/development/custom-models/bge-rerank.md b/docSite/content/docs/development/custom-models/bge-rerank.md index ab30356a50f..d68aa048677 100644 --- a/docSite/content/docs/development/custom-models/bge-rerank.md +++ b/docSite/content/docs/development/custom-models/bge-rerank.md @@ -100,7 +100,7 @@ docker run -d --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken --gpus all re version: "3" services: reranker: - image: registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2 + image: registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-base:v0.1 container_name: reranker # GPU运行环境，如果宿主机未安装，将deploy配置隐藏即可 deploy: diff --git a/docSite/content/docs/development/upgrading/471.md b/docSite/content/docs/development/upgrading/471.md index 381a8512531..8544daeaedb 100644 --- a/docSite/content/docs/development/upgrading/471.md +++ b/docSite/content/docs/development/upgrading/471.md @@ -19,10 +19,16 @@ curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \ 该请求会执行脏数据清理（清理无效的文件、清理无效的图片、清理无效的知识库集合、清理无效的向量） + +## 修改配置文件 + +增加了Laf环境配置：[点击查看最新的配置文件](/docs/development/configuration/) + + ## V4.7.1 更新说明 1. 新增 - 语音输入完整配置。支持选择是否打开语音输入（包括分享页面），支持语音输入后自动发送，支持语音输入后自动语音播放（流式）。 -2. 新增 - Pptx 和 xlsx 文件读取。但所有文件读取都放服务端，会消耗更多的服务器资源，以及无法在上传时预览更多内容。 +2. 新增 - pptx 和 xlsx 文件读取。但所有文件读取都放服务端，会消耗更多的服务器资源，以及无法在上传时预览更多内容。 3. 新增 - 集成 Laf 云函数，可以读取 Laf 账号中的云函数作为 HTTP 模块。 4. 新增 - 定时器，清理垃圾数据。（采用小范围清理，会清理最近n个小时的，所以请保证服务持续运行，长时间不允许，可以继续执行 clearInvalidData 的接口进行全量清理。） 5. 商业版新增 - 后台配置系统通知。 diff --git a/docSite/content/docs/workflow/examples/dalle3.md b/docSite/content/docs/workflow/examples/dalle3.md index 89bfa5d290c..f46c6e28c53 100644 --- a/docSite/content/docs/workflow/examples/dalle3.md +++ b/docSite/content/docs/workflow/examples/dalle3.md @@ -88,7 +88,7 @@ Response: [ { "moduleId": "userGuide", - "name": "core.module.template.User guide", + "name": "core.module.template.App system setting", "flowType": "userGuide", "position": { "x": 454.98510354678695, diff --git a/docSite/content/docs/workflow/examples/feishu_webhook.md b/docSite/content/docs/workflow/examples/feishu_webhook.md index 26549d17c36..c9c2b8cb7fd 100644 --- a/docSite/content/docs/workflow/examples/feishu_webhook.md +++ b/docSite/content/docs/workflow/examples/feishu_webhook.md @@ -27,7 +27,7 @@ weight: 404 [ { "moduleId": "userGuide", - "name": "core.module.template.User guide", + "name": "core.module.template.App system setting", "intro": "core.app.tip.userGuideTip", "avatar": "/imgs/module/userGuide.png", "flowType": "userGuide", diff --git a/docSite/content/docs/workflow/examples/google_search.md b/docSite/content/docs/workflow/examples/google_search.md index 18ef2000c21..a19bfec1fd0 100644 --- a/docSite/content/docs/workflow/examples/google_search.md +++ b/docSite/content/docs/workflow/examples/google_search.md @@ -84,7 +84,7 @@ export default async function (ctx: FunctionContext) { [ { "moduleId": "userGuide", - "name": "core.module.template.User guide", + "name": "core.module.template.App system setting", "intro": "core.app.tip.userGuideTip", "avatar": "/imgs/module/userGuide.png", "flowType": "userGuide", diff --git a/docSite/content/docs/workflow/modules/laf.md b/docSite/content/docs/workflow/modules/laf.md index 084d6905a8d..943e7ba4e8d 100644 --- a/docSite/content/docs/workflow/modules/laf.md +++ b/docSite/content/docs/workflow/modules/laf.md @@ -27,8 +27,6 @@ Laf 提供了 PAT(访问凭证) 来实现 Laf 平台外的快捷登录，可以 填入 PAT 验证后，选择需要绑定的应用（应用需要是 Running 状态），即可调用该应用下的云函数。 -> 如果需要解绑则取消绑定后，点击“更新”即可 - ![](/imgs/laf2.webp) ## 编写云函数 diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts index 8ee585721b4..12eaafa2a7d 100644 --- a/packages/global/common/string/textSplitter.ts +++ b/packages/global/common/string/textSplitter.ts @@ -1,5 +1,5 @@ import { getErrText } from '../error/utils'; -import { countPromptTokens } from './tiktoken'; +import { replaceRegChars } from './tools'; /** * text split into chunks @@ -31,7 +31,7 @@ export const splitText2Chunks = (props: { // The larger maxLen is, the next sentence is less likely to trigger splitting const stepReges: { reg: RegExp; maxLen: number }[] = [ ...customReg.map((text) => ({ - reg: new RegExp(`(${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'g'), + reg: new RegExp(`(${replaceRegChars(text)})`, 'g'), maxLen: chunkLen * 1.4 })), { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 }, diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts index a6d1fdaa241..5447297e91f 100644 --- a/packages/global/common/string/tools.ts +++ b/packages/global/common/string/tools.ts @@ -51,3 +51,5 @@ export const replaceSensitiveText = (text: string) => { export const getNanoid = (size = 12) => { return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)(); }; + +export const replaceRegChars = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); diff --git a/packages/global/core/module/template/system/userGuide.ts b/packages/global/core/module/template/system/userGuide.ts index b3b6507fc8a..dc0483c7441 100644 --- a/packages/global/core/module/template/system/userGuide.ts +++ b/packages/global/core/module/template/system/userGuide.ts @@ -1,6 +1,5 @@ import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant'; import { FlowNodeTemplateType } from '../../type.d'; -import { userGuideTip } from '../tip'; import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, @@ -12,8 +11,8 @@ export const UserGuideModule: FlowNodeTemplateType = { templateType: FlowNodeTemplateTypeEnum.userGuide, flowType: FlowNodeTypeEnum.userGuide, avatar: '/imgs/module/userGuide.png', - name: '全局配置', - intro: userGuideTip, + name: '系统配置', + intro: '可以配置应用的系统参数。', inputs: [ { key: ModuleInputKeyEnum.welcomeText, diff --git a/packages/global/core/module/template/tip.ts b/packages/global/core/module/template/tip.ts index 2aa9dd4051c..add805c721a 100644 --- a/packages/global/core/module/template/tip.ts +++ b/packages/global/core/module/template/tip.ts @@ -1,4 +1,3 @@ export const chatNodeSystemPromptTip = 'core.app.tip.chatNodeSystemPromptTip'; -export const userGuideTip = 'core.app.tip.userGuideTip'; export const welcomeTextTip = 'core.app.tip.welcomeTextTip'; export const variableTip = 'core.app.tip.variableTip'; diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts index 97cd5d42b44..d043b6be2f7 100644 --- a/packages/global/support/user/team/type.d.ts +++ b/packages/global/support/user/team/type.d.ts @@ -81,4 +81,5 @@ export type TeamTagItemType = { export type LafAccountType = { token: string; appid: string; + pat: string; }; diff --git a/packages/global/support/wallet/bill/type.d.ts b/packages/global/support/wallet/bill/type.d.ts index 0532630039f..cf808dce575 100644 --- a/packages/global/support/wallet/bill/type.d.ts +++ b/packages/global/support/wallet/bill/type.d.ts @@ -18,6 +18,7 @@ export type BillSchemaType = { month?: number; datasetSize?: number; extraPoints?: number; + invoice: boolean; }; username: string; }; diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index cc43ad03b70..8067ad9da1c 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -35,6 +35,7 @@ try { ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 }); ImageSchema.index({ type: 1 }); ImageSchema.index({ createTime: 1 }); + // delete related img ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 }); } catch (error) { console.log(error); diff --git a/packages/service/common/vectorStore/controller.d.ts b/packages/service/common/vectorStore/controller.d.ts index 7db8a7e07c5..442b9012b97 100644 --- a/packages/service/common/vectorStore/controller.d.ts +++ b/packages/service/common/vectorStore/controller.d.ts @@ -1,10 +1,9 @@ -export type DeleteDatasetVectorProps = { +export type DeleteDatasetVectorProps = ( + | { id: string } + | { datasetIds: string[]; collectionIds?: string[] } + | { idList: string[] } +) & { teamId: string; - - id?: string; - datasetIds?: string[]; - collectionIds?: string[]; - idList?: string[]; }; export type InsertVectorProps = { diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index ad551ef561b..df6dba2c43b 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -26,13 +26,7 @@ export async function initPg() { `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);` ); await PgClient.query( - `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id);` - ); - await PgClient.query( - ` CREATE INDEX CONCURRENTLY IF NOT EXISTS team_collection_index ON ${PgDatasetTableName} USING btree(team_id, collection_id);` - ); - await PgClient.query( - `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_id_index ON ${PgDatasetTableName} USING btree(team_id, id);` + `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id, collection_id);` ); await PgClient.query( `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${PgDatasetTableName} USING btree(createtime);` @@ -83,31 +77,33 @@ export const deleteDatasetDataVector = async ( retry?: number; } ): Promise => { - const { teamId, id, datasetIds, collectionIds, idList, retry = 2 } = props; + const { teamId, retry = 2 } = props; const teamIdWhere = `team_id='${String(teamId)}' AND`; const where = await (() => { - if (id) return `${teamIdWhere} id=${id}`; + if ('id' in props && props.id) return `${teamIdWhere} id=${props.id}`; - if (datasetIds) { - return `${teamIdWhere} dataset_id IN (${datasetIds + if ('datasetIds' in props && props.datasetIds) { + const datasetIdWhere = `dataset_id IN (${props.datasetIds .map((id) => `'${String(id)}'`) .join(',')})`; - } - if (collectionIds) { - return `${teamIdWhere} collection_id IN (${collectionIds - .map((id) => `'${String(id)}'`) - .join(',')})`; + if ('collectionIds' in props && props.collectionIds) { + return `${teamIdWhere} ${datasetIdWhere} AND collection_id IN (${props.collectionIds + .map((id) => `'${String(id)}'`) + .join(',')})`; + } + + return `${teamIdWhere} ${datasetIdWhere}`; } - if (idList) { - return `${teamIdWhere} id IN (${idList.map((id) => `'${String(id)}'`).join(',')})`; + if ('idList' in props && props.idList) { + return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`; } return Promise.reject('deleteDatasetData: no where'); })(); - + console.log(where, '==='); try { await PgClient.delete(PgDatasetTableName, { where: [where] diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index cc25e9a8411..b70d4a852bf 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -118,6 +118,37 @@ export function createDefaultCollection({ ); } +/* delete collection related images/files */ +export const delCollectionRelatedSource = async ({ + collections, + session +}: { + collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[]; + session: ClientSession; +}) => { + if (collections.length === 0) return; + + const teamId = collections[0].teamId; + + if (!teamId) return Promise.reject('teamId is not exist'); + + const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean); + const relatedImageIds = collections + .map((item) => item?.metadata?.relatedImgId || '') + .filter(Boolean); + + // delete images + await delImgByRelatedId({ + teamId, + relateIds: relatedImageIds, + session + }); + // delete files + await delFileByFileIdList({ + bucketName: BucketNameEnum.dataset, + fileIdList + }); +}; /** * delete collection and it related data */ @@ -134,26 +165,32 @@ export async function delCollectionAndRelatedSources({ if (!teamId) return Promise.reject('teamId is not exist'); + const datasetIds = Array.from( + new Set( + collections.map((item) => { + if (typeof item.datasetId === 'string') { + return String(item.datasetId); + } + return String(item.datasetId._id); + }) + ) + ); const collectionIds = collections.map((item) => String(item._id)); - const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean); - const relatedImageIds = collections - .map((item) => item?.metadata?.relatedImgId || '') - .filter(Boolean); + + await delCollectionRelatedSource({ collections, session }); // delete training data await MongoDatasetTraining.deleteMany({ teamId, + datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } }); - // delete dataset.datas - await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }, { session }); - // delete imgs - await delImgByRelatedId({ - teamId, - relateIds: relatedImageIds, - session - }); + await MongoDatasetData.deleteMany( + { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } }, + { session } + ); + // delete collections await MongoDatasetCollection.deleteMany( { @@ -163,9 +200,5 @@ export async function delCollectionAndRelatedSources({ ); // no session delete: delete files, vector data - await deleteDatasetDataVector({ teamId, collectionIds }); - await delFileByFileIdList({ - bucketName: BucketNameEnum.dataset, - fileIdList - }); + await deleteDatasetDataVector({ teamId, datasetIds, collectionIds }); } diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index d0779fbb361..91ab753e73d 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -1,8 +1,11 @@ import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from './collection/schema'; import { MongoDataset } from './schema'; -import { delCollectionAndRelatedSources } from './collection/controller'; +import { delCollectionRelatedSource } from './collection/controller'; import { ClientSession } from '../../common/mongo'; +import { MongoDatasetTraining } from './training/schema'; +import { MongoDatasetData } from './data/schema'; +import { deleteDatasetDataVector } from '../../common/vectorStore/controller'; /* ============= dataset ========== */ /* find all datasetId by top datasetId */ @@ -82,5 +85,26 @@ export async function delDatasetRelevantData({ '_id teamId fileId metadata' ).lean(); - await delCollectionAndRelatedSources({ collections, session }); + // image and file + await delCollectionRelatedSource({ collections, session }); + + // delete training data + await MongoDatasetTraining.deleteMany({ + teamId, + datasetId: { $in: datasetIds } + }); + // delete dataset.datas + await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }, { session }); + + // delete collections + await MongoDatasetCollection.deleteMany( + { + teamId, + datasetId: { $in: datasetIds } + }, + { session } + ); + + // no session delete: delete files, vector data + await deleteDatasetDataVector({ teamId, datasetIds }); } diff --git a/packages/service/core/dataset/data/controller.ts b/packages/service/core/dataset/data/controller.ts deleted file mode 100644 index ebb463c1e05..00000000000 --- a/packages/service/core/dataset/data/controller.ts +++ /dev/null @@ -1,2 +0,0 @@ -import { MongoDatasetData } from './schema'; -import { deleteDatasetDataVector } from '../../../common/vectorStore/controller'; diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index 35f5b05aa11..cf4fba0057f 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -77,17 +77,18 @@ const DatasetDataSchema = new Schema({ }); try { - // list collection and count data; list data + // list collection and count data; list data; delete collection(relate data) DatasetDataSchema.index( { teamId: 1, datasetId: 1, collectionId: 1, chunkIndex: 1, updateTime: -1 }, { background: true } ); - // same data check - DatasetDataSchema.index({ teamId: 1, collectionId: 1, q: 1, a: 1 }, { background: true }); // full text index DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }, { background: true }); // Recall vectors after data matching - DatasetDataSchema.index({ teamId: 1, datasetId: 1, 'indexes.dataId': 1 }, { background: true }); + DatasetDataSchema.index( + { teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 }, + { background: true } + ); DatasetDataSchema.index({ updateTime: 1 }, { background: true }); } catch (error) { console.log(error); diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts index 643a124cb79..0117b6b2324 100644 --- a/packages/service/core/dataset/search/controller.ts +++ b/packages/service/core/dataset/search/controller.ts @@ -93,6 +93,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { { teamId, datasetId: { $in: datasetIds }, + collectionId: { $in: results.map((item) => item.collectionId) }, 'indexes.dataId': { $in: results.map((item) => item.id?.trim()) } }, 'datasetId collectionId q a chunkIndex indexes' diff --git a/packages/service/core/dataset/training/controller.ts b/packages/service/core/dataset/training/controller.ts index 58125f431bb..47ebdd11bb4 100644 --- a/packages/service/core/dataset/training/controller.ts +++ b/packages/service/core/dataset/training/controller.ts @@ -6,7 +6,6 @@ import type { } from '@fastgpt/global/core/dataset/api.d'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { simpleText } from '@fastgpt/global/common/string/tools'; -import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { ClientSession } from '../../../common/mongo'; import { getLLMModel, getVectorModel } from '../../ai/model'; import { addLog } from '../../../common/system/log'; diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index 96610bb7ae4..0ff24ac9c95 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -92,8 +92,8 @@ const TrainingDataSchema = new Schema({ }); try { - // lock training data; delete training data - TrainingDataSchema.index({ teamId: 1, collectionId: 1 }); + // lock training data(teamId); delete training data + TrainingDataSchema.index({ teamId: 1, datasetId: 1 }); // get training data and sort TrainingDataSchema.index({ mode: 1, lockTime: 1, weight: -1 }); TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days diff --git a/packages/service/support/user/team/teamSchema.ts b/packages/service/support/user/team/teamSchema.ts index f1a2b1b4050..8f0f472eb28 100644 --- a/packages/service/support/user/team/teamSchema.ts +++ b/packages/service/support/user/team/teamSchema.ts @@ -42,6 +42,9 @@ const TeamSchema = new Schema({ }, appid: { type: String + }, + pat: { + type: String } } }); diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 706af88201b..5ae123e6c41 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -107,6 +107,7 @@ export const iconPaths = { 'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'), 'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'), 'core/modules/basicNode': () => import('./icons/core/modules/basicNode.svg'), + 'core/modules/fixview': () => import('./icons/core/modules/fixview.svg'), 'core/modules/flowLight': () => import('./icons/core/modules/flowLight.svg'), 'core/modules/previewLight': () => import('./icons/core/modules/previewLight.svg'), 'core/modules/systemPlugin': () => import('./icons/core/modules/systemPlugin.svg'), diff --git a/packages/web/components/common/Icon/icons/core/modules/fixview.svg b/packages/web/components/common/Icon/icons/core/modules/fixview.svg new file mode 100644 index 00000000000..abaeabba516 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/modules/fixview.svg @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/projects/app/public/docs/versionIntro.md b/projects/app/public/docs/versionIntro.md index bf7b3e5853c..45105750aa2 100644 --- a/projects/app/public/docs/versionIntro.md +++ b/projects/app/public/docs/versionIntro.md @@ -1,4 +1,4 @@ -### FastGPT V4.7 +### FastGPT V4.7.1 1. 新增 - 语音输入完整配置。支持选择是否打开语音输入（包括分享页面），支持语音输入后自动发送，支持语音输入后自动语音播放（流式）。 2. 新增 - Pptx 和 xlsx 文件读取。但所有文件读取都放服务端，会消耗更多的服务器资源，以及无法在上传时预览更多内容。 diff --git a/projects/app/public/js/iframe.js b/projects/app/public/js/iframe.js index 84a598c7e9a..bf67dfb148b 100644 --- a/projects/app/public/js/iframe.js +++ b/projects/app/public/js/iframe.js @@ -33,7 +33,7 @@ function embedChatbot() { ChatBtnDiv.draggable = false; const iframe = document.createElement('iframe'); - iframe.allow = 'fullscreen;microphone'; + iframe.allow = '*'; iframe.referrerPolicy = 'no-referrer'; iframe.title = 'FastGPT Chat Window'; iframe.id = chatWindowId; diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 7c23b2c7e91..17b83610f5c 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -230,7 +230,8 @@ "Amount": "{{amount}}{{unit}}" }, "speech": { - "error tip": "Speech Failed" + "error tip": "Speech Failed", + "not support": "Your browser does not support voice input" }, "system": { "Commercial version function": "Commercial version special function", @@ -250,7 +251,7 @@ }, "core": { "Chat": "Chat", - "Chat test": "Chat test", + "Chat test": "Test", "Max Token": "MaxTokens", "Start chat": "Start chat", "Total chars": "Total chars: {{total}}", @@ -423,6 +424,7 @@ "Converting to text": "Converting to text...", "Custom History Title": "Custom history title", "Custom History Title Description": "If set to empty, chat history will be followed automatically.", + "Debug test": "Test", "Exit Chat": "Exit", "Failed to initialize chat": "Failed to initialize chat", "Feedback Failed": "Feedback Failed", diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 0583d2b682f..70cf757b577 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -230,7 +230,8 @@ "Amount": "{{amount}}{{unit}}" }, "speech": { - "error tip": "语音转文字失败" + "error tip": "语音转文字失败", + "not support": "您的浏览器不支持语音输入" }, "system": { "Commercial version function": "商业版特有功能", @@ -250,7 +251,7 @@ }, "core": { "Chat": "对话", - "Chat test": "测试对话", + "Chat test": "测试", "Max Token": "单条数据上限", "Start chat": "立即对话", "Total chars": "总字数: {{total}}", @@ -423,6 +424,7 @@ "Converting to text": "正在转换为文本...", "Custom History Title": "自定义历史记录标题", "Custom History Title Description": "如果设置为空，会自动跟随聊天记录。", + "Debug test": "调试预览", "Exit Chat": "退出聊天", "Failed to initialize chat": "初始化聊天失败", "Feedback Failed": "提交反馈异常", @@ -608,8 +610,7 @@ "success": "开始同步" } }, - "training": { - } + "training": {} }, "data": { "Auxiliary Data": "辅助数据", @@ -996,6 +997,7 @@ "Tool module": "工具", "UnKnow Module": "未知模块", "User guide": "用户引导", + "App system setting": "系统配置", "http body placeholder": "与APIFox相同的语法", "textEditor": "文本加工", "textEditor intro": "可对固定或传入的文本进行加工后输出，非字符串类型数据最终会转成字符串类型。" diff --git a/projects/app/src/components/ChatBox/components/ChatItem.tsx b/projects/app/src/components/ChatBox/components/ChatItem.tsx index 2b14e05db63..79dccc7a6f7 100644 --- a/projects/app/src/components/ChatBox/components/ChatItem.tsx +++ b/projects/app/src/components/ChatBox/components/ChatItem.tsx @@ -94,7 +94,7 @@ const ChatItem = ({ /* AI */ return ( - + {chat.value.map((value, i) => { const key = `${chat.dataId}-ai-${i}`; if (value.text) { diff --git a/projects/app/src/components/core/module/Flow/ChatTest.tsx b/projects/app/src/components/core/module/Flow/ChatTest.tsx index c11eb1e1943..dd4cd2f78b4 100644 --- a/projects/app/src/components/core/module/Flow/ChatTest.tsx +++ b/projects/app/src/components/core/module/Flow/ChatTest.tsx @@ -8,6 +8,7 @@ import React, { useImperativeHandle, ForwardedRef } from 'react'; +import { SmallCloseIcon } from '@chakra-ui/icons'; import { Box, Flex, IconButton } from '@chakra-ui/react'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { streamFetch } from '@/web/common/api/fetch'; @@ -18,6 +19,7 @@ import type { ComponentRef, StartChatFnProps } from '@/components/ChatBox/type.d import { getGuideModule } from '@fastgpt/global/core/module/utils'; import { checkChatSupportSelectFileByModules } from '@/web/core/chat/utils'; import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; +import { useTranslation } from 'next-i18next'; export type ChatTestComponentRef = { resetChatTest: () => void; @@ -35,6 +37,7 @@ const ChatTest = ( }, ref: ForwardedRef ) => { + const { t } = useTranslation(); const ChatBoxRef = useRef(null); const { userInfo } = useUserStore(); const isOpen = useMemo(() => modules && modules.length > 0, [modules]); @@ -100,9 +103,9 @@ const ChatTest = ( > - 调试预览 + {t('core.chat.Debug test')} - + + + } + variant={'grayBase'} + size={'smSquare'} + aria-label={''} + onClick={onClose} + /> + - + /> */} ); }; diff --git a/projects/app/src/components/core/module/Flow/FlowProvider.tsx b/projects/app/src/components/core/module/Flow/FlowProvider.tsx index 18542ec2ec6..15616a57099 100644 --- a/projects/app/src/components/core/module/Flow/FlowProvider.tsx +++ b/projects/app/src/components/core/module/Flow/FlowProvider.tsx @@ -166,7 +166,7 @@ export const FlowProvider = ({ }, [nodes]); const onFixView = useCallback(() => { - const btn = document.querySelector('.react-flow__controls-fitview') as HTMLButtonElement; + const btn = document.querySelector('.custom-workflow-fix_view') as HTMLButtonElement; setTimeout(() => { btn && btn.click(); diff --git a/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx b/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx index b78a1cb385b..03715142b5f 100644 --- a/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx +++ b/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx @@ -15,8 +15,10 @@ import NodeCard from '../render/NodeCard'; import type { VariableItemType } from '@fastgpt/global/core/app/type.d'; import QGSwitch from '@/components/core/app/QGSwitch'; import TTSSelect from '@/components/core/app/TTSSelect'; +import WhisperConfig from '@/components/core/app/WhisperConfig'; import { splitGuideModule } from '@fastgpt/global/core/module/utils'; import { useTranslation } from 'next-i18next'; +import { TTSTypeEnum } from '@/constants/app'; const NodeUserGuide = ({ data, selected }: NodeProps) => { const theme = useTheme(); @@ -31,6 +33,9 @@ const NodeUserGuide = ({ data, selected }: NodeProps) => { + + + @@ -164,3 +169,26 @@ function TTSGuide({ data }: { data: FlowModuleItemType }) { /> ); } + +function WhisperGuide({ data }: { data: FlowModuleItemType }) { + const { inputs, moduleId } = data; + const { ttsConfig, whisperConfig } = splitGuideModule({ inputs } as ModuleItemType); + + return ( + { + onChangeNode({ + moduleId, + key: ModuleInputKeyEnum.whisper, + type: 'updateInput', + value: { + ...inputs.find((item) => item.key === ModuleInputKeyEnum.whisper), + value: e + } + }); + }} + /> + ); +} diff --git a/projects/app/src/components/core/module/Flow/components/render/NodeCard.tsx b/projects/app/src/components/core/module/Flow/components/render/NodeCard.tsx index b1b479c7670..53d01274cdc 100644 --- a/projects/app/src/components/core/module/Flow/components/render/NodeCard.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/NodeCard.tsx @@ -261,7 +261,7 @@ const NodeCard = (props: Props) => { }} > {Header} - {children} + {children} {RenderModal} ); diff --git a/projects/app/src/components/core/module/Flow/index.tsx b/projects/app/src/components/core/module/Flow/index.tsx index d38b56e9a0f..047b02c2bb0 100644 --- a/projects/app/src/components/core/module/Flow/index.tsx +++ b/projects/app/src/components/core/module/Flow/index.tsx @@ -3,8 +3,11 @@ import ReactFlow, { Background, Connection, Controls, + ControlButton, + MiniMap, NodeProps, - ReactFlowProvider + ReactFlowProvider, + useReactFlow } from 'reactflow'; import { Box, Flex, IconButton, useDisclosure } from '@chakra-ui/react'; import { SmallCloseIcon } from '@chakra-ui/icons'; @@ -20,6 +23,8 @@ import 'reactflow/dist/style.css'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { useTranslation } from 'next-i18next'; import { FlowModuleItemType } from '@fastgpt/global/core/module/type'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyTooltip from '@/components/MyTooltip'; const NodeSimple = dynamic(() => import('./components/nodes/NodeSimple')); const nodeTypes: Record<`${FlowNodeTypeEnum}`, any> = { @@ -54,19 +59,10 @@ const edgeTypes = { const Container = React.memo(function Container() { const { toast } = useToast(); const { t } = useTranslation(); + const { reactFlowWrapper, nodes, onNodesChange, edges, onEdgesChange, onConnect } = useFlowProviderStore(); - const memoRenderTools = useMemo( - () => ( - <> - - - - ), - [] - ); - const customOnConnect = useCallback( (connect: Connection) => { if (!connect.sourceHandle || !connect.targetHandle) { @@ -105,7 +101,7 @@ const Container = React.memo(function Container() { onEdgesChange={onEdgesChange} onConnect={customOnConnect} > - {memoRenderTools} + ); }); @@ -168,3 +164,40 @@ const Flow = ({ Header, ...data }: { Header: React.ReactNode }) => { }; export default React.memo(Flow); + +const FlowController = React.memo(function FlowController() { + const { fitView } = useReactFlow(); + return ( + <> + + + + fitView()}> + + + + + + + ); +}); diff --git a/projects/app/src/components/support/laf/LafAccountModal.tsx b/projects/app/src/components/support/laf/LafAccountModal.tsx index c6b6227eb2a..80ab4715b7f 100644 --- a/projects/app/src/components/support/laf/LafAccountModal.tsx +++ b/projects/app/src/components/support/laf/LafAccountModal.tsx @@ -18,7 +18,8 @@ import { getDocPath } from '@/web/common/system/doc'; const LafAccountModal = ({ defaultData = { token: '', - appid: '' + appid: '', + pat: '' }, onClose }: { @@ -140,7 +141,7 @@ const LafAccountModal = ({ onResetForm(); putUpdateTeam({ teamId: userInfo?.team.teamId || '', - lafAccount: { token: '', appid: '' } + lafAccount: { token: '', appid: '', pat: '' } }); }} > diff --git a/projects/app/src/pages/api/admin/initv471.ts b/projects/app/src/pages/api/admin/initv471.ts new file mode 100644 index 00000000000..7dd07a9debb --- /dev/null +++ b/projects/app/src/pages/api/admin/initv471.ts @@ -0,0 +1,29 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; + +/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + await authCert({ req, authRoot: true }); + + // 删除索引 + await PgClient.query(`DROP INDEX IF EXISTS team_dataset_index;`); + await PgClient.query(`DROP INDEX IF EXISTS team_collection_index;`); + await PgClient.query(`DROP INDEX IF EXISTS team_id_index;`); + + jsonRes(res, { + message: 'success' + }); + } catch (error) { + console.log(error); + + jsonRes(res, { + code: 500, + error + }); + } +} diff --git a/projects/app/src/pages/api/core/dataset/collection/delete.ts b/projects/app/src/pages/api/core/dataset/collection/delete.ts index 783d203266a..104917233a1 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delete.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts @@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< teamId, datasetId: collection.datasetId._id, collectionId, - fields: '_id teamId fileId metadata' + fields: '_id teamId datasetId fileId metadata' }); // delete diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts index 35d114ca9ca..77aaa7c1e3e 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertData.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts @@ -70,6 +70,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // Duplicate data check await hasSameValue({ teamId, + datasetId, collectionId, q: formatQ, a: formatA diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index a22eed4d52b..54d1fa27f17 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -6,6 +6,7 @@ import type { GetDatasetDataListProps } from '@/global/core/api/datasetReq'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { PagingData } from '@/types'; +import { replaceRegChars } from '@fastgpt/global/common/string/tools'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -28,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< per: 'r' }); - searchText = searchText.replace(/'/g, ''); + searchText = replaceRegChars(searchText).replace(/'/g, ''); const match = { teamId, diff --git a/projects/app/src/pages/api/v1/audio/transcriptions.ts b/projects/app/src/pages/api/v1/audio/transcriptions.ts index a10f8176ce8..49cfa2c0b96 100644 --- a/projects/app/src/pages/api/v1/audio/transcriptions.ts +++ b/projects/app/src/pages/api/v1/audio/transcriptions.ts @@ -9,6 +9,7 @@ import { pushWhisperUsage } from '@/service/support/wallet/usage/push'; import { authChatCert } from '@/service/support/permission/auth/chat'; import { MongoApp } from '@fastgpt/service/core/app/schema'; import { getGuideModule, splitGuideModule } from '@fastgpt/global/core/module/utils'; +import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat'; const upload = getUploadModel({ maxSize: 2 @@ -20,15 +21,16 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex try { const { file, - data: { appId, duration, teamId: spaceTeamId, teamToken } - } = await upload.doUpload<{ - appId: string; - duration: number; - shareId?: string; - teamId?: string; - teamToken?: string; - }>(req, res); + data: { appId, duration, shareId, outLinkUid, teamId: spaceTeamId, teamToken } + } = await upload.doUpload< + OutLinkChatAuthProps & { + appId: string; + duration: number; + } + >(req, res); + req.body.shareId = shareId; + req.body.outLinkUid = outLinkUid; req.body.teamId = spaceTeamId; req.body.teamToken = teamToken; diff --git a/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx b/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx index 92c849b8b79..785ae823c53 100644 --- a/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx +++ b/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx @@ -1,6 +1,5 @@ import React, { useCallback, useRef, useState } from 'react'; -import { Box, Flex, IconButton, useTheme, useDisclosure } from '@chakra-ui/react'; -import { SmallCloseIcon } from '@chakra-ui/icons'; +import { Box, Flex, IconButton, useTheme, useDisclosure, Button } from '@chakra-ui/react'; import { ModuleItemType } from '@fastgpt/global/core/module/type'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { AppSchema } from '@fastgpt/global/core/app/type.d'; @@ -18,6 +17,7 @@ import { useAppStore } from '@/web/core/app/store/useAppStore'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { getErrText } from '@fastgpt/global/common/error/utils'; +import MyMenu from '@/components/MyMenu'; const ImportSettings = dynamic(() => import('@/components/core/module/Flow/ImportSettings')); @@ -143,73 +143,61 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ {app.name} - - } - variant={'whitePrimary'} - aria-label={'save'} - onClick={onOpenImport} - /> - - - } - size={'smSquare'} - variant={'whitePrimary'} - aria-label={'save'} - onClick={async () => { - const modules = await flow2ModulesAndCheck(); - if (modules) { - copyData(filterExportModules(modules), t('app.Export Config Successful')); - } - }} - /> - - - {testModules ? ( - } - variant={'whitePrimary'} - size={'smSquare'} - aria-label={''} - onClick={() => setTestModules(undefined)} - /> - ) : ( - + } - size={'smSquare'} - aria-label={'save'} + mr={[3, 5]} + icon={} + aria-label={''} + size={'sm'} variant={'whitePrimary'} - onClick={async () => { + /> + } + menuList={[ + { label: t('app.Import Configs'), icon: 'common/importLight', onClick: onOpenImport }, + { + label: t('app.Export Configs'), + icon: 'export', + onClick: async () => { const modules = await flow2ModulesAndCheck(); if (modules) { - setTestModules(modules); + copyData(filterExportModules(modules), t('app.Export Config Successful')); } - }} - /> - - )} + } + } + ]} + /> - - } - size={'smSquare'} - isLoading={isSaving} - aria-label={'save'} + {!testModules && ( + + )} + + {isOpenImport && } ` }, [UsingWayEnum.script]: { diff --git a/projects/app/src/pages/chat/components/ToolMenu.tsx b/projects/app/src/pages/chat/components/ToolMenu.tsx index 77c058ca319..bed7d2a5dd6 100644 --- a/projects/app/src/pages/chat/components/ToolMenu.tsx +++ b/projects/app/src/pages/chat/components/ToolMenu.tsx @@ -1,7 +1,7 @@ import React, { useMemo } from 'react'; import { useChatBox } from '@/components/ChatBox/hooks/useChatBox'; import type { ChatItemType } from '@fastgpt/global/core/chat/type.d'; -import { Menu, MenuButton, MenuList, MenuItem, Box, IconButton } from '@chakra-ui/react'; +import { Box, IconButton } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { useRouter } from 'next/router'; diff --git a/projects/app/src/pages/plugin/edit/Header.tsx b/projects/app/src/pages/plugin/edit/Header.tsx index daef4a312de..03e65cb0ac6 100644 --- a/projects/app/src/pages/plugin/edit/Header.tsx +++ b/projects/app/src/pages/plugin/edit/Header.tsx @@ -1,5 +1,5 @@ import React, { useCallback } from 'react'; -import { Box, Flex, IconButton, useTheme, useDisclosure } from '@chakra-ui/react'; +import { Box, Flex, IconButton, useTheme, useDisclosure, Button } from '@chakra-ui/react'; import { PluginItemSchema } from '@fastgpt/global/core/plugin/type'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useTranslation } from 'next-i18next'; @@ -14,6 +14,7 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; import { ModuleItemType } from '@fastgpt/global/core/module/type'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants'; +import MyMenu from '@/components/MyMenu'; const ImportSettings = dynamic(() => import('@/components/core/module/Flow/ImportSettings')); const PreviewPlugin = dynamic(() => import('./Preview')); @@ -137,38 +138,37 @@ const Header = ({ plugin, onClose }: Props) => { }} /> - + {plugin.name} - - } - variant={'whitePrimary'} - size={'smSquare'} - aria-label={'save'} - onClick={onOpenImport} - /> - - - } - size={'smSquare'} - variant={'whitePrimary'} - aria-label={'save'} - onClick={async () => { - const modules = await flow2ModulesAndCheck(); - if (modules) { - copyData(filterExportModules(modules), t('app.Export Config Successful')); + } + aria-label={''} + size={'sm'} + variant={'whitePrimary'} + /> + } + menuList={[ + { label: t('app.Import Configs'), icon: 'common/importLight', onClick: onOpenImport }, + { + label: t('app.Export Configs'), + icon: 'export', + onClick: async () => { + const modules = await flow2ModulesAndCheck(); + if (modules) { + copyData(filterExportModules(modules), t('app.Export Config Successful')); + } } - }} - /> - + } + ]} + /> } size={'smSquare'} aria-label={'save'} @@ -181,20 +181,19 @@ const Header = ({ plugin, onClose }: Props) => { }} /> - - } - size={'smSquare'} - isLoading={isLoading} - aria-label={'save'} - onClick={async () => { - const modules = await flow2ModulesAndCheck(); - if (modules) { - onclickSave(modules); - } - }} - /> - + {isOpenImport && } {!!previewModules && ( diff --git a/projects/app/src/service/common/system/cronTask.ts b/projects/app/src/service/common/system/cronTask.ts index 2239d97e257..8f97c5d1dd8 100644 --- a/projects/app/src/service/common/system/cronTask.ts +++ b/projects/app/src/service/common/system/cronTask.ts @@ -2,6 +2,7 @@ import { delFileByFileIdList, getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller'; +import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { addLog } from '@fastgpt/service/common/system/log'; import { deleteDatasetDataVector, @@ -72,15 +73,19 @@ export async function checkInvalidDatasetData(start: Date, end: Date) { $lte: end } }, - '_id teamId collectionId' + '_id teamId datasetId collectionId' ).lean(); // 2. 合并所有的collectionId - const map = new Map(); + const map = new Map(); for (const item of rows) { const collectionId = String(item.collectionId); if (!map.has(collectionId)) { - map.set(collectionId, { teamId: item.teamId, collectionId }); + map.set(collectionId, { + teamId: item.teamId, + datasetId: item.datasetId, + collectionId + }); } } const list = Array.from(map.values()); @@ -92,21 +97,28 @@ export async function checkInvalidDatasetData(start: Date, end: Date) { // 3. 查看该collection是否存在，不存在，则删除对应的数据 const collection = await MongoDatasetCollection.findOne({ _id: item.collectionId }); if (!collection) { - const result = await Promise.all([ - MongoDatasetTraining.deleteMany({ - teamId: item.teamId, - collectionId: item.collectionId - }), - MongoDatasetData.deleteMany({ + await mongoSessionRun(async (session) => { + await MongoDatasetTraining.deleteMany( + { + teamId: item.teamId, + collectionId: item.collectionId + }, + { session } + ); + await MongoDatasetData.deleteMany( + { + teamId: item.teamId, + collectionId: item.collectionId + }, + { session } + ); + await deleteDatasetDataVector({ teamId: item.teamId, - collectionId: item.collectionId - }), - deleteDatasetDataVector({ - teamId: item.teamId, - collectionIds: [String(item.collectionId)] - }) - ]); - console.log(result); + datasetIds: [item.datasetId], + collectionIds: [item.collectionId] + }); + }); + console.log('collection is not found', item); continue; } diff --git a/projects/app/src/service/core/dataset/data/utils.ts b/projects/app/src/service/core/dataset/data/utils.ts index f9cbc06ae19..38648a47e0a 100644 --- a/projects/app/src/service/core/dataset/data/utils.ts +++ b/projects/app/src/service/core/dataset/data/utils.ts @@ -5,17 +5,20 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; */ export async function hasSameValue({ teamId, + datasetId, collectionId, q, a = '' }: { teamId: string; + datasetId: string; collectionId: string; q: string; a?: string; }) { const count = await MongoDatasetData.countDocuments({ teamId, + datasetId, collectionId, q, a diff --git a/projects/app/src/web/common/api/lafRequest.ts b/projects/app/src/web/common/api/lafRequest.ts index b337a263059..95eff84170b 100644 --- a/projects/app/src/web/common/api/lafRequest.ts +++ b/projects/app/src/web/common/api/lafRequest.ts @@ -5,6 +5,8 @@ import axios, { AxiosProgressEvent } from 'axios'; import { useUserStore } from '@/web/support/user/useUserStore'; +import { putUpdateTeam } from '@/web/support/user/team/api'; +import { LafAccountType } from '@fastgpt/global/support/user/team/type'; interface ConfigType { headers?: { [key: string]: string }; @@ -72,21 +74,33 @@ function responseSuccess(response: AxiosResponse) { /** * 响应数据检查 */ -function checkRes(data: ResponseDataType) { - if (data === undefined) { - console.log('error->', data, 'data is empty'); +function checkRes( + res: ResponseDataType, + url: string, + data: any, + requestConfig: ConfigType, + method: Method +) { + if (res === undefined) { + console.log('error->', res, 'res is empty'); return Promise.reject('服务器异常'); - } else if (data.error) { - return responseError(data.error); + } else if (res.error) { + return responseError(data.error, url, data, requestConfig, method); } - return data.data; + return res.data; } /** * 响应错误 */ -function responseError(err: any) { +function responseError( + err: any, + url: string, + data: any, + requestConfig: ConfigType, + method: Method +) { console.log('error->', '请求错误', err); if (!err) { @@ -97,6 +111,25 @@ function responseError(err: any) { } if (err?.response?.data) { + const code = err?.response?.data?.statusCode; + if (code === 401) { + return POST(`/v1/auth/pat2token`, { + pat: useUserStore.getState().userInfo?.team?.lafAccount?.pat + }) + .then((res) => { + putUpdateTeam({ + teamId: useUserStore.getState().userInfo?.team.teamId || '', + lafAccount: { + ...useUserStore.getState().userInfo?.team?.lafAccount, + token: res + } as LafAccountType + }); + return request(url, data, requestConfig, method); + }) + .catch((err) => { + return Promise.reject({ message: '登录凭证过期' }); + }); + } return Promise.reject(err?.response?.data); } return Promise.reject(err); @@ -115,12 +148,9 @@ instance.interceptors.request.use(startInterceptors, (err) => Promise.reject(err /* 响应拦截 */ instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err)); -function request( - url: string, - data: any, - { cancelToken, maxQuantity, ...config }: ConfigType, - method: Method -): any { +function request(url: string, data: any, requestConfig: ConfigType, method: Method): any { + const { cancelToken, maxQuantity, ...config } = requestConfig; + /* 去空 */ for (const key in data) { if (data[key] === null || data[key] === undefined) { @@ -140,8 +170,8 @@ function request( signal: cancelToken?.signal, ...config // 用户自定义配置，可以覆盖前面的配置 }) - .then((res) => checkRes(res.data)) - .catch((err) => responseError(err)) + .then((res) => checkRes(res.data, url, data, requestConfig, method)) + .catch((err) => responseError(err, url, data, requestConfig, method)) .finally(() => requestFinish({ url })); } diff --git a/projects/app/src/web/common/hooks/useSpeech.ts b/projects/app/src/web/common/hooks/useSpeech.ts index 83e44d67e23..4bd3886b096 100644 --- a/projects/app/src/web/common/hooks/useSpeech.ts +++ b/projects/app/src/web/common/hooks/useSpeech.ts @@ -51,6 +51,12 @@ export const useSpeech = (props?: OutLinkChatAuthProps & { appId?: string }) => }, []); const startSpeak = async (onFinish: (text: string) => void) => { + if (!navigator.mediaDevices.getUserMedia) { + return toast({ + status: 'warning', + title: t('common.speech.not support') + }); + } try { cancelWhisperSignal.current = false; diff --git a/projects/app/src/web/core/app/templates.ts b/projects/app/src/web/core/app/templates.ts index 482b79ace10..e83eaeb5347 100644 --- a/projects/app/src/web/core/app/templates.ts +++ b/projects/app/src/web/core/app/templates.ts @@ -17,7 +17,7 @@ export const appTemplates: (AppItemType & { modules: [ { moduleId: 'userGuide', - name: 'core.module.template.User guide', + name: 'core.module.template.App system setting', avatar: '/imgs/module/userGuide.png', flowType: 'userGuide', position: { @@ -300,7 +300,7 @@ export const appTemplates: (AppItemType & { modules: [ { moduleId: 'userGuide', - name: 'core.module.template.User guide', + name: 'core.module.template.App system setting', avatar: '/imgs/module/userGuide.png', flowType: 'userGuide', position: { @@ -616,7 +616,7 @@ export const appTemplates: (AppItemType & { modules: [ { moduleId: 'userGuide', - name: 'core.module.template.User guide', + name: 'core.module.template.App system setting', intro: 'core.app.tip.userGuideTip', avatar: '/imgs/module/userGuide.png', flowType: 'userGuide', @@ -1651,7 +1651,7 @@ export const appTemplates: (AppItemType & { }, { moduleId: 'q9equb', - name: 'core.module.template.User guide', + name: 'core.module.template.App system setting', flowType: 'userGuide', position: { x: -272.66416216517086, diff --git a/projects/app/src/web/core/app/utils.ts b/projects/app/src/web/core/app/utils.ts index 0676ba0e62c..e6b0d85a1ab 100644 --- a/projects/app/src/web/core/app/utils.ts +++ b/projects/app/src/web/core/app/utils.ts @@ -14,7 +14,7 @@ export async function postForm2Modules(data: AppSimpleEditFormType) { function userGuideTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { return [ { - name: 'core.module.template.User guide', + name: '系统配置', flowType: FlowNodeTypeEnum.userGuide, inputs: [ { diff --git a/projects/app/src/web/styles/reactflow.scss b/projects/app/src/web/styles/reactflow.scss index 37df1fac9d2..3585787078e 100644 --- a/projects/app/src/web/styles/reactflow.scss +++ b/projects/app/src/web/styles/reactflow.scss @@ -1,6 +1,4 @@ -.react-flow__panel { - display: none; -} .react-flow__panel.react-flow__attribution { - display: none !important; + z-index: 0; + left: 0; } diff --git a/projects/app/src/web/support/laf/api.ts b/projects/app/src/web/support/laf/api.ts index 012db2e8277..78f2727f748 100644 --- a/projects/app/src/web/support/laf/api.ts +++ b/projects/app/src/web/support/laf/api.ts @@ -1,4 +1,4 @@ -import { GET, POST, PUT } from '@/web/common/api/lafRequest'; +import { GET, POST } from '@/web/common/api/lafRequest'; export const postLafPat2Token = (pat: string) => POST(`/v1/auth/pat2token`, { pat }); diff --git a/python/bge-rerank/README.md b/python/bge-rerank/README.md index 51693f2334e..4a5af8fa053 100644 --- a/python/bge-rerank/README.md +++ b/python/bge-rerank/README.md @@ -92,7 +92,7 @@ docker run -d --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken --gpus all re version: "3" services: reranker: - image: registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2 + image: registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-base:v0.1 container_name: reranker # GPU运行环境，如果宿主机未安装，将deploy配置隐藏即可 deploy: