From a92f8cf32c326fa773614796431ca765333b2d22 Mon Sep 17 00:00:00 2001 From: chanzhi82020 Date: Mon, 15 Sep 2025 11:53:00 +0800 Subject: [PATCH] fix: add list evaluation param for frontend --- packages/global/core/evaluation/api.d.ts | 3 + .../service/core/evaluation/metric/schema.ts | 5 +- .../service/core/evaluation/task/index.ts | 115 ++++--- .../service/core/evaluation/task/schema.ts | 3 +- .../service/core/evaluation/utils/index.ts | 88 ++++- .../pages/api/core/evaluation/task/create.ts | 28 +- .../pages/api/core/evaluation/task/list.ts | 7 +- .../pages/api/core/evaluation/task/update.ts | 27 +- .../api/core/evaluation/task/create.test.ts | 16 + .../api/core/evaluation/task/list.test.ts | 103 +++++- .../service/core/evaluation/task.test.ts | 249 +++++++++++++- .../evaluation/utils/validate-dataset.test.ts | 307 ++++++++++++++++++ 12 files changed, 856 insertions(+), 95 deletions(-) create mode 100644 test/cases/service/core/evaluation/utils/validate-dataset.test.ts diff --git a/packages/global/core/evaluation/api.d.ts b/packages/global/core/evaluation/api.d.ts index e15509ef2699..f2dfdb71ef28 100644 --- a/packages/global/core/evaluation/api.d.ts +++ b/packages/global/core/evaluation/api.d.ts @@ -33,6 +33,9 @@ export type DeleteEvaluationResponse = MessageResponse; // List Evaluations export type ListEvaluationsRequest = PaginationProps<{ searchKey?: string; + appName?: string; + appId?: string; + versionId?: string; }>; export type ListEvaluationsResponse = PaginationResponse; diff --git a/packages/service/core/evaluation/metric/schema.ts b/packages/service/core/evaluation/metric/schema.ts index 51b4222b7883..2b0ce254a431 100644 --- a/packages/service/core/evaluation/metric/schema.ts +++ b/packages/service/core/evaluation/metric/schema.ts @@ -23,8 +23,7 @@ const EvalMetricSchema = new Schema({ }, name: { type: String, - required: true, - unique: true + required: true }, description: { type: String, @@ -84,7 +83,7 @@ const EvalMetricSchema = new Schema({ } }); -EvalMetricSchema.index({ teamId: 1, name: 1 }); +EvalMetricSchema.index({ teamId: 1, name: 1 }, { unique: true }); EvalMetricSchema.index({ createTime: -1 }); EvalMetricSchema.pre('save', function (next) { diff --git a/packages/service/core/evaluation/task/index.ts b/packages/service/core/evaluation/task/index.ts index 15577e724cde..ffd7fcbbc489 100644 --- a/packages/service/core/evaluation/task/index.ts +++ b/packages/service/core/evaluation/task/index.ts @@ -169,7 +169,10 @@ export class EvaluationTaskService { searchKey?: string, accessibleIds?: string[], tmbId?: string, - isOwner: boolean = false + isOwner: boolean = false, + appName?: string, + appId?: string, + versionId?: string ): Promise<{ list: EvaluationDisplayType[]; total: number }> { // Build basic filter and pagination const filter: any = { teamId: new Types.ObjectId(teamId) }; @@ -195,43 +198,74 @@ export class EvaluationTaskService { }; } + // Build aggregation pipeline with target filtering + const aggregationPipeline = [ + { $match: finalFilter }, + { + $lookup: { + from: 'eval_dataset_collections', + localField: 'datasetId', + foreignField: '_id', + as: 'dataset' + } + }, + { + $addFields: { + 'target.config.appObjectId': { $toObjectId: '$target.config.appId' } + } + }, + { + $lookup: { + from: 'apps', + localField: 'target.config.appObjectId', + foreignField: '_id', + as: 'app' + } + }, + { + $addFields: { + 'target.config.versionObjectId': { $toObjectId: '$target.config.versionId' } + } + }, + { + $lookup: { + from: 'app_versions', + localField: 'target.config.versionObjectId', + foreignField: '_id', + as: 'appVersion' + } + }, + { + $addFields: { + 'target.config.appName': { $arrayElemAt: ['$app.name', 0] }, + 'target.config.avatar': { $arrayElemAt: ['$app.avatar', 0] }, + 'target.config.versionName': { $arrayElemAt: ['$appVersion.versionName', 0] } + } + } + ]; + + // Add target filtering stage if any target filters are provided + if (appName || appId || versionId) { + const targetFilter: any = {}; + + if (appName) { + targetFilter['target.config.appName'] = { $regex: appName, $options: 'i' }; + } + + if (appId) { + targetFilter['target.config.appId'] = appId; + } + + if (versionId) { + targetFilter['target.config.versionId'] = versionId; + } + + aggregationPipeline.push({ $match: targetFilter }); + } + const [evaluations, total] = await Promise.all([ MongoEvaluation.aggregate([ - { $match: finalFilter }, - { - $lookup: { - from: 'eval_dataset_collections', - localField: 'datasetId', - foreignField: '_id', - as: 'dataset' - } - }, - { - $addFields: { - 'target.config.appObjectId': { $toObjectId: '$target.config.appId' } - } - }, - { - $lookup: { - from: 'apps', - localField: 'target.config.appObjectId', - foreignField: '_id', - as: 'app' - } - }, - { - $addFields: { - 'target.config.versionObjectId': { $toObjectId: '$target.config.versionId' } - } - }, - { - $lookup: { - from: 'app_versions', - localField: 'target.config.versionObjectId', - foreignField: '_id', - as: 'appVersion' - } - }, + ...aggregationPipeline, // Add real-time statistics lookup { $lookup: { @@ -258,10 +292,6 @@ export class EvaluationTaskService { { $addFields: { datasetName: { $arrayElemAt: ['$dataset.name', 0] }, - // Add app name and avatar to target.config - 'target.config.appName': { $arrayElemAt: ['$app.name', 0] }, - 'target.config.avatar': { $arrayElemAt: ['$app.avatar', 0] }, - 'target.config.versionName': { $arrayElemAt: ['$appVersion.versionName', 0] }, metricNames: { $map: { input: '$evaluators', @@ -316,7 +346,10 @@ export class EvaluationTaskService { { $skip: skip }, { $limit: limit } ]), - MongoEvaluation.countDocuments(finalFilter) + // Get total count using the same aggregation pipeline (without pagination) + MongoEvaluation.aggregate([...aggregationPipeline, { $count: 'total' }]).then( + (result) => result[0]?.total || 0 + ) ]); // Return raw data - permissions will be handled in API layer diff --git a/packages/service/core/evaluation/task/schema.ts b/packages/service/core/evaluation/task/schema.ts index 0a7dfb3edf0f..a481e0757a1e 100644 --- a/packages/service/core/evaluation/task/schema.ts +++ b/packages/service/core/evaluation/task/schema.ts @@ -116,7 +116,6 @@ export const EvaluationTaskSchema = new Schema({ name: { type: String, required: true, - unique: true, trim: true, maxlength: 100 }, @@ -170,7 +169,7 @@ export const EvaluationTaskSchema = new Schema({ // Optimized indexes for EvaluationTaskSchema EvaluationTaskSchema.index({ teamId: 1, createTime: -1 }); // Main query: team filtering + time sorting EvaluationTaskSchema.index({ teamId: 1, status: 1, createTime: -1 }); // Status filtering + time sorting -EvaluationTaskSchema.index({ teamId: 1, name: 1 }); // Name uniqueness check +EvaluationTaskSchema.index({ teamId: 1, name: 1 }, { unique: true }); // Name uniqueness check EvaluationTaskSchema.index({ tmbId: 1, createTime: -1 }); // Query by creator EvaluationTaskSchema.index({ status: 1 }); // Status-based queue processing diff --git a/packages/service/core/evaluation/utils/index.ts b/packages/service/core/evaluation/utils/index.ts index 3e52bb8d9b06..d3b476a60153 100644 --- a/packages/service/core/evaluation/utils/index.ts +++ b/packages/service/core/evaluation/utils/index.ts @@ -4,11 +4,59 @@ import type { CreateEvaluationParams } from '@fastgpt/global/core/evaluation/typ import type { ValidationResult } from '@fastgpt/global/core/evaluation/validate'; import { EvaluationErrEnum } from '@fastgpt/global/common/error/code/evaluation'; import { MAX_NAME_LENGTH, MAX_DESCRIPTION_LENGTH } from '@fastgpt/global/core/evaluation/constants'; +import { MongoEvalDatasetCollection } from '../dataset/evalDatasetCollectionSchema'; +import { Types } from 'mongoose'; export type EvaluationValidationParams = Partial; export interface EvaluationValidationOptions { mode?: 'create' | 'update'; // validation mode + teamId?: string; // required for dataset existence validation +} + +/** + * Validate if a dataset exists and is accessible by the team + */ +async function validateDatasetExists( + datasetId: string, + teamId?: string +): Promise { + // Validate datasetId format + if (!Types.ObjectId.isValid(datasetId)) { + return { + isValid: false, + errors: [ + { + code: EvaluationErrEnum.evalDatasetIdRequired, + message: 'Invalid dataset ID format', + field: 'datasetId' + } + ] + }; + } + + // Check if dataset exists + const filter: any = { _id: new Types.ObjectId(datasetId) }; + if (teamId) { + filter.teamId = new Types.ObjectId(teamId); + } + + const dataset = await MongoEvalDatasetCollection.findOne(filter).lean(); + + if (!dataset) { + return { + isValid: false, + errors: [ + { + code: EvaluationErrEnum.datasetCollectionNotFound, + message: 'Dataset not found or access denied', + field: 'datasetId' + } + ] + }; + } + + return { isValid: true, errors: [] }; } export async function validateEvaluationParams( @@ -118,17 +166,25 @@ export async function validateEvaluationParams( }; } - if (datasetId !== undefined && !datasetId) { - return { - isValid: false, - errors: [ - { - code: EvaluationErrEnum.evalDatasetIdRequired, - message: 'Dataset ID is required', - field: 'datasetId' - } - ] - }; + if (datasetId !== undefined) { + if (!datasetId) { + return { + isValid: false, + errors: [ + { + code: EvaluationErrEnum.evalDatasetIdRequired, + message: 'Dataset ID is required', + field: 'datasetId' + } + ] + }; + } + + // Validate dataset exists and is accessible + const datasetValidation = await validateDatasetExists(datasetId, options?.teamId); + if (!datasetValidation.isValid) { + return datasetValidation; + } } if (target !== undefined) { @@ -192,13 +248,15 @@ export async function validateEvaluationParams( } export async function validateEvaluationParamsForCreate( - params: EvaluationValidationParams + params: EvaluationValidationParams, + teamId?: string ): Promise { - return validateEvaluationParams(params, { mode: 'create' }); + return validateEvaluationParams(params, { mode: 'create', teamId }); } export async function validateEvaluationParamsForUpdate( - params: EvaluationValidationParams + params: EvaluationValidationParams, + teamId?: string ): Promise { - return validateEvaluationParams(params, { mode: 'update' }); + return validateEvaluationParams(params, { mode: 'update', teamId }); } diff --git a/projects/app/src/pages/api/core/evaluation/task/create.ts b/projects/app/src/pages/api/core/evaluation/task/create.ts index 25b5406a06b7..c1c5df725e9b 100644 --- a/projects/app/src/pages/api/core/evaluation/task/create.ts +++ b/projects/app/src/pages/api/core/evaluation/task/create.ts @@ -21,24 +21,28 @@ async function handler( ): Promise { const { name, description, datasetId, target, evaluators, autoStart } = req.body; - // Validate all evaluation parameters (includes target validation) - const paramValidation = await validateEvaluationParamsForCreate({ - name, - description, - datasetId, - target, - evaluators - }); - if (!paramValidation.isValid) { - throw ValidationResultUtils.toError(paramValidation); - } - + // First perform auth to get teamId const { teamId, tmbId } = await authEvaluationTaskCreate(target as EvalTarget, { req, authApiKey: true, authToken: true }); + // Now validate all evaluation parameters with teamId (includes target and dataset validation) + const paramValidation = await validateEvaluationParamsForCreate( + { + name, + description, + datasetId, + target, + evaluators + }, + teamId + ); + if (!paramValidation.isValid) { + throw ValidationResultUtils.toError(paramValidation); + } + // Check evaluation task limit await checkTeamEvaluationTaskLimit(teamId); diff --git a/projects/app/src/pages/api/core/evaluation/task/list.ts b/projects/app/src/pages/api/core/evaluation/task/list.ts index 6bb15dfe889c..8b023a63ee72 100644 --- a/projects/app/src/pages/api/core/evaluation/task/list.ts +++ b/projects/app/src/pages/api/core/evaluation/task/list.ts @@ -25,7 +25,7 @@ async function handler( req: ApiRequestProps ): Promise { const { offset, pageSize } = parsePaginationRequest(req); - const { searchKey } = req.body; + const { searchKey, appName, appId, versionId } = req.body; const { teamId, tmbId, isOwner, roleList, myGroupMap, myOrgSet } = await getEvaluationPermissionAggregation({ @@ -49,7 +49,10 @@ async function handler( searchKey?.trim(), accessibleIds, tmbId, - isOwner + isOwner, + appName?.trim(), + appId?.trim(), + versionId?.trim() ); const formatEvaluations = result.list diff --git a/projects/app/src/pages/api/core/evaluation/task/update.ts b/projects/app/src/pages/api/core/evaluation/task/update.ts index 253de3606197..bcfed48c69e4 100644 --- a/projects/app/src/pages/api/core/evaluation/task/update.ts +++ b/projects/app/src/pages/api/core/evaluation/task/update.ts @@ -21,24 +21,27 @@ async function handler( throw new Error(EvaluationErrEnum.evalIdRequired); } - // Validate all evaluation parameters with common validation utility - const paramValidation = await validateEvaluationParamsForUpdate({ - name, - description, - datasetId, - target, - evaluators - }); - if (!paramValidation.isValid) { - throw ValidationResultUtils.toError(paramValidation); - } - const { teamId, tmbId, evaluation } = await authEvaluationTaskWrite(evalId, { req, authApiKey: true, authToken: true }); + // Validate all evaluation parameters with common validation utility + const paramValidation = await validateEvaluationParamsForUpdate( + { + name, + description, + datasetId, + target, + evaluators + }, + teamId + ); + if (!paramValidation.isValid) { + throw ValidationResultUtils.toError(paramValidation); + } + const taskName = name?.trim() || evaluation.name; await EvaluationTaskService.updateEvaluation( diff --git a/test/cases/pages/api/core/evaluation/task/create.test.ts b/test/cases/pages/api/core/evaluation/task/create.test.ts index e171eda90735..0f834a084f1a 100644 --- a/test/cases/pages/api/core/evaluation/task/create.test.ts +++ b/test/cases/pages/api/core/evaluation/task/create.test.ts @@ -8,6 +8,7 @@ import { } from '@fastgpt/service/support/permission/teamLimit'; import { validateTargetConfig } from '@fastgpt/service/core/evaluation/target'; import { EvaluationStatusEnum } from '@fastgpt/global/core/evaluation/constants'; +import { MongoEvalDatasetCollection } from '@fastgpt/service/core/evaluation/dataset/evalDatasetCollectionSchema'; // Mock dependencies vi.mock('@fastgpt/service/core/evaluation/task', () => ({ @@ -47,6 +48,12 @@ vi.mock('@fastgpt/service/common/system/log', () => ({ } })); +vi.mock('@fastgpt/service/core/evaluation/dataset/evalDatasetCollectionSchema', () => ({ + MongoEvalDatasetCollection: { + findOne: vi.fn() + } +})); + describe('Create Evaluation Task API Handler', () => { const mockEvaluation = { _id: new Types.ObjectId(), @@ -82,6 +89,15 @@ describe('Create Evaluation Task API Handler', () => { // Setup mocks vi.mocked(checkTeamAIPoints).mockResolvedValue(undefined); vi.mocked(checkTeamEvaluationTaskLimit).mockResolvedValue(undefined); + + // Mock dataset exists for all tests + (MongoEvalDatasetCollection.findOne as any).mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + _id: new Types.ObjectId(), + teamId: new Types.ObjectId(), + name: 'Test Dataset' + }) + }); }); test('应该成功创建评估任务', async () => { diff --git a/test/cases/pages/api/core/evaluation/task/list.test.ts b/test/cases/pages/api/core/evaluation/task/list.test.ts index db78ed156feb..3a5cf94f476a 100644 --- a/test/cases/pages/api/core/evaluation/task/list.test.ts +++ b/test/cases/pages/api/core/evaluation/task/list.test.ts @@ -77,7 +77,10 @@ describe('List Evaluation Tasks API Handler', () => { undefined, [], 'mock-tmb-id', - true + true, + undefined, + undefined, + undefined ); expect(result).toEqual({ list: mockResult.list.map((item) => ({ @@ -110,7 +113,10 @@ describe('List Evaluation Tasks API Handler', () => { 'test search', [], 'mock-tmb-id', - true + true, + undefined, + undefined, + undefined ); }); @@ -132,7 +138,98 @@ describe('List Evaluation Tasks API Handler', () => { undefined, [], 'mock-tmb-id', - true + true, + undefined, + undefined, + undefined + ); + }); + + test('应该处理target过滤参数', async () => { + const mockReq = { + body: { + pageNum: 1, + pageSize: 10, + appName: 'Test App', + appId: '507f1f77bcf86cd799439011', + versionId: '507f1f77bcf86cd799439012' + } + } as any; + + const mockResult = { list: [], total: 0 }; + (EvaluationTaskService.listEvaluations as any).mockResolvedValue(mockResult); + + await listHandler(mockReq); + + expect(EvaluationTaskService.listEvaluations).toHaveBeenCalledWith( + 'mock-team-id', + 0, + 10, + undefined, + [], + 'mock-tmb-id', + true, + 'Test App', + '507f1f77bcf86cd799439011', + '507f1f77bcf86cd799439012' + ); + }); + + test('应该处理部分target过滤参数', async () => { + const mockReq = { + body: { + pageNum: 1, + pageSize: 10, + appName: 'Partial App' + } + } as any; + + const mockResult = { list: [], total: 0 }; + (EvaluationTaskService.listEvaluations as any).mockResolvedValue(mockResult); + + await listHandler(mockReq); + + expect(EvaluationTaskService.listEvaluations).toHaveBeenCalledWith( + 'mock-team-id', + 0, + 10, + undefined, + [], + 'mock-tmb-id', + true, + 'Partial App', + undefined, + undefined + ); + }); + + test('应该处理包含空白字符的target过滤参数', async () => { + const mockReq = { + body: { + pageNum: 1, + pageSize: 10, + appName: ' Test App ', + appId: ' 507f1f77bcf86cd799439011 ', + versionId: ' 507f1f77bcf86cd799439012 ' + } + } as any; + + const mockResult = { list: [], total: 0 }; + (EvaluationTaskService.listEvaluations as any).mockResolvedValue(mockResult); + + await listHandler(mockReq); + + expect(EvaluationTaskService.listEvaluations).toHaveBeenCalledWith( + 'mock-team-id', + 0, + 10, + undefined, + [], + 'mock-tmb-id', + true, + 'Test App', + '507f1f77bcf86cd799439011', + '507f1f77bcf86cd799439012' ); }); }); diff --git a/test/cases/service/core/evaluation/task.test.ts b/test/cases/service/core/evaluation/task.test.ts index 68bdb3646a32..8d7490b929e8 100644 --- a/test/cases/service/core/evaluation/task.test.ts +++ b/test/cases/service/core/evaluation/task.test.ts @@ -225,11 +225,33 @@ describe('EvaluationTaskService', () => { evaluators: evaluators }; - const evaluation = await EvaluationTaskService.createEvaluation({ - ...params, - teamId: teamId, - tmbId: tmbId - }); + // 添加重试机制处理MongoDB事务冲突 + let evaluation: any; + let retryCount = 0; + const maxRetries = 3; + + while (retryCount < maxRetries) { + try { + evaluation = await EvaluationTaskService.createEvaluation({ + ...params, + teamId: teamId, + tmbId: tmbId + }); + break; // 成功则退出循环 + } catch (error: any) { + retryCount++; + if ( + error?.message?.includes('Collection namespace') && + error?.message?.includes('is already in use') && + retryCount < maxRetries + ) { + // 等待一段时间后重试 + await new Promise((resolve) => setTimeout(resolve, 100 * retryCount)); + continue; + } + throw error; // 非命名空间冲突错误或重试次数用完，直接抛出 + } + } expect(evaluation.name).toBe(params.name); expect(evaluation.description).toBe(params.description); @@ -458,6 +480,223 @@ describe('EvaluationTaskService', () => { expect(Array.isArray(result.list)).toBe(true); expect(result.list.some((evaluation) => evaluation.name.includes('Searchable'))).toBe(true); }); + + test('应该支持按appName过滤', async () => { + // Clean up any leftover evaluation tasks + await MongoEvaluation.deleteMany({ teamId }); + + // 创建使用不同appId的评估任务 + const targetWithAppName = { + type: 'workflow' as const, + config: { + appId: '507f1f77bcf86cd799439015', + versionId: '507f1f77bcf86cd799439016', + chatConfig: {} + } + }; + + const params: CreateEvaluationParams = { + name: 'App Name Filter Test', + description: 'Test evaluation for app name filtering', + datasetId, + target: targetWithAppName, + evaluators: evaluators + }; + + await EvaluationTaskService.createEvaluation({ + ...params, + teamId: teamId, + tmbId: tmbId + }); + + // 模拟有应用名称的情况 - 实际场景中appName会在aggregation阶段从apps collection中lookup得到 + // 这里我们测试带appName参数的调用 + const result = await EvaluationTaskService.listEvaluations( + teamId, + 0, + 10, + undefined, + undefined, + tmbId, + true, + 'Test App Name' + ); + + expect(Array.isArray(result.list)).toBe(true); + expect(typeof result.total).toBe('number'); + }); + + test('应该支持按appId过滤', async () => { + // Clean up any leftover evaluation tasks + await MongoEvaluation.deleteMany({ teamId }); + + const targetWithSpecificAppId = { + type: 'workflow' as const, + config: { + appId: '507f1f77bcf86cd799439020', + versionId: '507f1f77bcf86cd799439021', + chatConfig: {} + } + }; + + const params: CreateEvaluationParams = { + name: 'App ID Filter Test', + description: 'Test evaluation for app ID filtering', + datasetId, + target: targetWithSpecificAppId, + evaluators: evaluators + }; + + const created = await EvaluationTaskService.createEvaluation({ + ...params, + teamId: teamId, + tmbId: tmbId + }); + + const result = await EvaluationTaskService.listEvaluations( + teamId, + 0, + 10, + undefined, + undefined, + tmbId, + true, + undefined, + '507f1f77bcf86cd799439020' + ); + + expect(Array.isArray(result.list)).toBe(true); + expect(typeof result.total).toBe('number'); + // 由于过滤条件匹配，应该能找到创建的评估任务 + const foundEvaluation = result.list.find((e) => e._id.toString() === created._id.toString()); + expect(foundEvaluation).toBeDefined(); + }); + + test('应该支持按versionId过滤', async () => { + // Clean up any leftover evaluation tasks + await MongoEvaluation.deleteMany({ teamId }); + + const targetWithSpecificVersionId = { + type: 'workflow' as const, + config: { + appId: '507f1f77bcf86cd799439025', + versionId: '507f1f77bcf86cd799439026', + chatConfig: {} + } + }; + + const params: CreateEvaluationParams = { + name: 'Version ID Filter Test', + description: 'Test evaluation for version ID filtering', + datasetId, + target: targetWithSpecificVersionId, + evaluators: evaluators + }; + + const created = await EvaluationTaskService.createEvaluation({ + ...params, + teamId: teamId, + tmbId: tmbId + }); + + const result = await EvaluationTaskService.listEvaluations( + teamId, + 0, + 10, + undefined, + undefined, + tmbId, + true, + undefined, + undefined, + '507f1f77bcf86cd799439026' + ); + + expect(Array.isArray(result.list)).toBe(true); + expect(typeof result.total).toBe('number'); + // 由于过滤条件匹配，应该能找到创建的评估任务 + const foundEvaluation = result.list.find((e) => e._id.toString() === created._id.toString()); + expect(foundEvaluation).toBeDefined(); + }); + + test('应该支持组合target过滤条件', async () => { + // Clean up any leftover evaluation tasks + await MongoEvaluation.deleteMany({ teamId }); + + const targetWithMultipleFilters = { + type: 'workflow' as const, + config: { + appId: '507f1f77bcf86cd799439030', + versionId: '507f1f77bcf86cd799439031', + chatConfig: {} + } + }; + + const params: CreateEvaluationParams = { + name: 'Multiple Filters Test', + description: 'Test evaluation for multiple target filtering', + datasetId, + target: targetWithMultipleFilters, + evaluators: evaluators + }; + + await EvaluationTaskService.createEvaluation({ + ...params, + teamId: teamId, + tmbId: tmbId + }); + + const result = await EvaluationTaskService.listEvaluations( + teamId, + 0, + 10, + undefined, + undefined, + tmbId, + true, + 'Test App', // appName filter + '507f1f77bcf86cd799439030', // appId filter + '507f1f77bcf86cd799439031' // versionId filter + ); + + expect(Array.isArray(result.list)).toBe(true); + expect(typeof result.total).toBe('number'); + }); + + test('不匹配的过滤条件应该返回空结果', async () => { + // Clean up any leftover evaluation tasks + await MongoEvaluation.deleteMany({ teamId }); + + const params: CreateEvaluationParams = { + name: 'No Match Test', + description: 'Test evaluation that should not match filters', + datasetId, + target, + evaluators: evaluators + }; + + await EvaluationTaskService.createEvaluation({ + ...params, + teamId: teamId, + tmbId: tmbId + }); + + const result = await EvaluationTaskService.listEvaluations( + teamId, + 0, + 10, + undefined, + undefined, + tmbId, + true, + undefined, + 'non-existent-app-id' + ); + + expect(Array.isArray(result.list)).toBe(true); + expect(result.total).toBe(0); + expect(result.list.length).toBe(0); + }); }); describe('startEvaluation', () => { diff --git a/test/cases/service/core/evaluation/utils/validate-dataset.test.ts b/test/cases/service/core/evaluation/utils/validate-dataset.test.ts new file mode 100644 index 000000000000..fa2cb7e3bbd7 --- /dev/null +++ b/test/cases/service/core/evaluation/utils/validate-dataset.test.ts @@ -0,0 +1,307 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { Types } from '@fastgpt/service/common/mongo'; +import { + validateEvaluationParamsForCreate, + validateEvaluationParamsForUpdate +} from '@fastgpt/service/core/evaluation/utils'; +import { MongoEvalDatasetCollection } from '@fastgpt/service/core/evaluation/dataset/evalDatasetCollectionSchema'; +import { EvaluationErrEnum } from '@fastgpt/global/common/error/code/evaluation'; +import { EvalMetricTypeEnum } from '@fastgpt/global/core/evaluation/metric/constants'; +import type { EvaluatorSchema } from '@fastgpt/global/core/evaluation/type'; + +// Mock the dataset collection +vi.mock('@fastgpt/service/core/evaluation/dataset/evalDatasetCollectionSchema', () => ({ + MongoEvalDatasetCollection: { + findOne: vi.fn() + } +})); + +// Mock target and evaluator validation to always return valid +vi.mock('@fastgpt/service/core/evaluation/target', () => ({ + validateTargetConfig: vi.fn().mockResolvedValue({ isValid: true, errors: [] }) +})); + +vi.mock('@fastgpt/service/core/evaluation/evaluator', () => ({ + validateEvaluatorConfig: vi.fn().mockResolvedValue({ isValid: true, errors: [] }) +})); + +describe('Dataset Existence Validation', () => { + const teamId = new Types.ObjectId().toString(); + const validDatasetId = new Types.ObjectId().toString(); + + const mockDataset = { + _id: new Types.ObjectId(validDatasetId), + teamId: new Types.ObjectId(teamId), + name: 'Test Dataset', + dataItems: [{ userInput: 'test input', expectedOutput: 'test output' }] + }; + + const validEvaluator: EvaluatorSchema = { + metric: { + _id: new Types.ObjectId().toString(), + teamId: new Types.ObjectId().toString(), + tmbId: new Types.ObjectId().toString(), + name: 'Test Metric', + type: EvalMetricTypeEnum.Custom, + prompt: 'Test prompt', + userInputRequired: true, + actualOutputRequired: true, + expectedOutputRequired: true, + contextRequired: false, + retrievalContextRequired: false, + embeddingRequired: false, + llmRequired: true, + createTime: new Date(), + updateTime: new Date() + }, + runtimeConfig: {}, + weight: 1 + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('Dataset ID Format Validation', () => { + it('should reject invalid dataset ID format', async () => { + const result = await validateEvaluationParamsForCreate( + { + name: 'Test Evaluation', + datasetId: 'invalid-id-format', + target: { + type: 'workflow', + config: { + appId: new Types.ObjectId().toString(), + versionId: new Types.ObjectId().toString() + } + }, + evaluators: [validEvaluator] + }, + teamId + ); + + expect(result.isValid).toBe(false); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toMatchObject({ + code: EvaluationErrEnum.evalDatasetIdRequired, + message: 'Invalid dataset ID format', + field: 'datasetId' + }); + }); + + it('should accept valid dataset ID format when dataset exists', async () => { + // Mock dataset found + const mockFindOne = MongoEvalDatasetCollection.findOne as any; + mockFindOne.mockReturnValue({ + lean: vi.fn().mockResolvedValue(mockDataset) + }); + + const result = await validateEvaluationParamsForCreate( + { + name: 'Test Evaluation', + datasetId: validDatasetId, + target: { + type: 'workflow', + config: { + appId: new Types.ObjectId().toString(), + versionId: new Types.ObjectId().toString() + } + }, + evaluators: [validEvaluator] + }, + teamId + ); + + expect(result.isValid).toBe(true); + expect(result.errors).toHaveLength(0); + + // Verify dataset lookup was called with correct parameters + expect(MongoEvalDatasetCollection.findOne).toHaveBeenCalledWith({ + _id: new Types.ObjectId(validDatasetId), + teamId: new Types.ObjectId(teamId) + }); + }); + }); + + describe('Dataset Existence Validation', () => { + it('should reject when dataset does not exist', async () => { + // Mock dataset not found + const mockFindOne = MongoEvalDatasetCollection.findOne as any; + mockFindOne.mockReturnValue({ + lean: vi.fn().mockResolvedValue(null) + }); + + const result = await validateEvaluationParamsForCreate( + { + name: 'Test Evaluation', + datasetId: validDatasetId, + target: { + type: 'workflow', + config: { + appId: new Types.ObjectId().toString(), + versionId: new Types.ObjectId().toString() + } + }, + evaluators: [validEvaluator] + }, + teamId + ); + + expect(result.isValid).toBe(false); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toMatchObject({ + code: EvaluationErrEnum.datasetCollectionNotFound, + message: 'Dataset not found or access denied', + field: 'datasetId' + }); + }); + + it('should check dataset existence without teamId filter when teamId not provided', async () => { + // Mock dataset found + const mockFindOne = MongoEvalDatasetCollection.findOne as any; + mockFindOne.mockReturnValue({ + lean: vi.fn().mockResolvedValue(mockDataset) + }); + + const result = await validateEvaluationParamsForCreate( + { + name: 'Test Evaluation', + datasetId: validDatasetId, + target: { + type: 'workflow', + config: { + appId: new Types.ObjectId().toString(), + versionId: new Types.ObjectId().toString() + } + }, + evaluators: [validEvaluator] + } + // No teamId provided + ); + + expect(result.isValid).toBe(true); + expect(result.errors).toHaveLength(0); + + // Should search without teamId filter + expect(MongoEvalDatasetCollection.findOne).toHaveBeenCalledWith({ + _id: new Types.ObjectId(validDatasetId) + }); + }); + + it('should enforce team-based access control', async () => { + const differentTeamId = new Types.ObjectId().toString(); + + // Mock dataset not found for this team + const mockFindOne = MongoEvalDatasetCollection.findOne as any; + mockFindOne.mockReturnValue({ + lean: vi.fn().mockResolvedValue(null) // Not found for this team + }); + + const result = await validateEvaluationParamsForCreate( + { + name: 'Test Evaluation', + datasetId: validDatasetId, + target: { + type: 'workflow', + config: { + appId: new Types.ObjectId().toString(), + versionId: new Types.ObjectId().toString() + } + }, + evaluators: [validEvaluator] + }, + differentTeamId + ); + + expect(result.isValid).toBe(false); + expect(result.errors[0].code).toBe(EvaluationErrEnum.datasetCollectionNotFound); + + // Should search with the specified teamId + expect(MongoEvalDatasetCollection.findOne).toHaveBeenCalledWith({ + _id: new Types.ObjectId(validDatasetId), + teamId: new Types.ObjectId(differentTeamId) + }); + }); + }); + + describe('Update Mode Validation', () => { + it('should validate dataset existence in update mode', async () => { + // Mock dataset found + const mockFindOne = MongoEvalDatasetCollection.findOne as any; + mockFindOne.mockReturnValue({ + lean: vi.fn().mockResolvedValue(mockDataset) + }); + + const result = await validateEvaluationParamsForUpdate( + { + datasetId: validDatasetId + // Only updating datasetId + }, + teamId + ); + + expect(result.isValid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('should skip dataset validation when datasetId not provided in update mode', async () => { + const result = await validateEvaluationParamsForUpdate( + { + name: 'Updated Name' + // No datasetId provided + }, + teamId + ); + + expect(result.isValid).toBe(true); + expect(result.errors).toHaveLength(0); + + // Should not call dataset validation when datasetId is not provided + expect(MongoEvalDatasetCollection.findOne).not.toHaveBeenCalled(); + }); + + it('should reject empty datasetId in update mode', async () => { + const result = await validateEvaluationParamsForUpdate( + { + datasetId: '' + }, + teamId + ); + + expect(result.isValid).toBe(false); + expect(result.errors[0]).toMatchObject({ + code: EvaluationErrEnum.evalDatasetIdRequired, + message: 'Dataset ID is required', + field: 'datasetId' + }); + }); + }); + + describe('Database Error Handling', () => { + it('should propagate database errors', async () => { + // Mock database error + const mockFindOne = MongoEvalDatasetCollection.findOne as any; + mockFindOne.mockReturnValue({ + lean: vi.fn().mockRejectedValue(new Error('Database connection failed')) + }); + + await expect( + validateEvaluationParamsForCreate( + { + name: 'Test Evaluation', + datasetId: validDatasetId, + target: { + type: 'workflow', + config: { + appId: new Types.ObjectId().toString(), + versionId: new Types.ObjectId().toString() + } + }, + evaluators: [validEvaluator] + }, + teamId + ) + ).rejects.toThrow('Database connection failed'); + }); + }); +});