Skip to content

fix: list evaluationItem args belowThreshold #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
chanzhi82020 wants to merge 1 commit into from
Closed

fix: list evaluationItem args belowThreshold #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions packages/service/core/evaluation/target/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ export class WorkflowTarget extends EvaluationTarget {
}

async execute(input: TargetInput): Promise<TargetOutput> {
const startTime = Date.now();

// Get application information
const appData = await MongoApp.findById(this.config.appId);
if (!appData) {
Expand Down Expand Up @@ -211,7 +209,7 @@ export class WorkflowTarget extends EvaluationTarget {
actualOutput: response,
retrievalContext: extractRetrievalContext(flowResponses),
usage: flowUsages,
responseTime: Date.now() - startTime,
responseTime: durationSeconds,
chatId,
aiChatItemDataId
};
Expand Down
21 changes: 18 additions & 3 deletions packages/service/core/evaluation/task/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -797,7 +797,11 @@ export class EvaluationTaskService {
}
});
commonPipeline.push({
$match: { hasFailedEvaluator: true }
$match: {
hasFailedEvaluator: true,
'metadata.status': EvaluationStatusEnum.completed, // Only completed items
evaluatorOutputs: { $exists: true, $ne: null, $not: { $size: 0 } } // Valid evaluator outputs
}
});
}
}
Expand Down Expand Up @@ -913,6 +917,17 @@ export class EvaluationTaskService {
// Get the updated item to determine the evalId
const updatedItem = await MongoEvalItem.findById(itemId, 'evalId');
if (updatedItem) {
const cleanupResult = await removeEvaluationItemJobsByItemId(itemId, {
forceCleanActiveJobs: true,
retryAttempts: 3,
retryDelay: 200
});

addLog.debug('Queue cleanup completed for evaluation item deletion', {
itemId,
cleanup: cleanupResult
});

// Reset results and re-queue
const evaluatorOutputs = evaluation.evaluators.map((evaluator) => ({
metricName: evaluator.metric.name
Expand Down Expand Up @@ -976,7 +991,7 @@ export class EvaluationTaskService {
// Retry the job directly (active event will clear error state automatically)
await job.retry();

addLog.info('Evaluation item retried successfully', {
addLog.debug('Evaluation item retried successfully', {
itemId,
evalId: item.evalId,
teamId
Expand Down Expand Up @@ -1016,7 +1031,7 @@ export class EvaluationTaskService {
}
}

addLog.info('All failed evaluation items retry completed', {
addLog.debug('All failed evaluation items retry completed', {
evalId,
teamId,
totalFailedJobs: evaluationFailedJobs.length,
Expand Down
5 changes: 0 additions & 5 deletions packages/service/core/evaluation/task/statusCalculator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ export async function getEvaluationTaskStatus(evalId: string): Promise<Evaluatio
// If all task jobs completed, check evaluation item status
return await getEvaluationTaskStatusFromItems(evalId);
} catch (error) {
addLog.error('Error getting evaluation task status:', { evalId, error });
return EvaluationStatusEnum.error;
}
}
Expand Down Expand Up @@ -105,7 +104,6 @@ async function getEvaluationTaskStatusFromItems(evalId: string): Promise<Evaluat
// Default status
return EvaluationStatusEnum.completed;
} catch (error) {
addLog.error('Error getting evaluation task status from items:', { evalId, error });
return EvaluationStatusEnum.error;
}
}
Expand Down Expand Up @@ -165,7 +163,6 @@ export async function getEvaluationItemStatus(evalItemId: string): Promise<Evalu

return EvaluationStatusEnum.queuing;
} catch (error) {
addLog.error('Error getting evaluation item status:', { evalItemId, error });
return EvaluationStatusEnum.error;
}
}
Expand Down Expand Up @@ -413,7 +410,6 @@ export async function checkEvaluationTaskJobActive(evalId: string): Promise<bool

return hasActiveTaskJob || hasActiveItemJob;
} catch (error) {
addLog.error('Error checking evaluation job active status:', { evalId, error });
return false;
}
}
Expand All @@ -426,7 +422,6 @@ export async function checkEvaluationItemJobActive(evalItemId: string): Promise<
const itemJobs = await evaluationItemQueue.getJobs(['waiting', 'delayed', 'active']);
return itemJobs.some((job) => job.data.evalItemId === evalItemId);
} catch (error) {
addLog.error('Error checking evaluation item job active status:', { evalItemId, error });
return false;
}
}
13 changes: 13 additions & 0 deletions packages/web/i18n/en/admin.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,19 @@
"custom_pdf_parse_timeout": "Custom PDF parsing timeout",
"doc2x_pdf_parse_key": "Doc2x PDF parsing key (lower priority than custom PDF parsing)",
"custom_pdf_parse_price": "Custom PDF parsing price (n points/page)",
"eval_config": "Evaluation Configuration",
"task_concurrency": "Task Concurrency",
"task_concurrency_desc": "Number of evaluation tasks running simultaneously",
"item_concurrency": "Item Concurrency",
"item_concurrency_desc": "Number of items processed concurrently within a single task",
"item_max_retry": "Max Retry Count",
"item_max_retry_desc": "Maximum retry attempts when evaluation items fail",
"default_threshold": "Default Threshold",
"default_threshold_desc": "Default threshold for evaluation (between 0-1)",
"summary_concurrency": "Summary Concurrency",
"data_quality_concurrency": "Data Quality Concurrency",
"dataset_synthesize_concurrency": "Dataset Synthesis Concurrency",
"smart_generate_concurrency": "Smart Generation Concurrency",
"max_upload_files_per_time": "Max files per upload",
"max_upload_files_per_time_desc": "Maximum number of files per upload to a knowledge base",
"max_upload_file_size": "Max file size (MB)",
Expand Down
13 changes: 13 additions & 0 deletions packages/web/i18n/zh-CN/admin.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,19 @@
"custom_pdf_parse_timeout": "自定义 PDF 解析超时时间",
"doc2x_pdf_parse_key": "Doc2x pdf 解析密钥（比自定义 PDF 解析优先级低）",
"custom_pdf_parse_price": "自定义 PDF 解析价格(n 积分/页)",
"eval_config": "评估配置",
"task_concurrency": "任务并发数",
"task_concurrency_desc": "同时执行的评估任务数量",
"item_concurrency": "项目并发数",
"item_concurrency_desc": "单个任务中并发处理的项目数量",
"item_max_retry": "最大重试次数",
"item_max_retry_desc": "评估项目失败时的最大重试次数",
"default_threshold": "默认阈值",
"default_threshold_desc": "评估的默认阈值 (0-1之间)",
"summary_concurrency": "总结并发数",
"data_quality_concurrency": "数据质量并发数",
"dataset_synthesize_concurrency": "数据合成并发数",
"smart_generate_concurrency": "智能生成并发数",
"max_upload_files_per_time": "单次最多上传多少个文件",
"max_upload_files_per_time_desc": "用户上传知识库时，每次上传最多选择多少个文件",
"max_upload_file_size": "上传文件最大大小（M)",
Expand Down
13 changes: 13 additions & 0 deletions packages/web/i18n/zh-Hant/admin.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,19 @@
"custom_pdf_parse_timeout": "自定義 PDF 解析逾時時間",
"doc2x_pdf_parse_key": "Doc2x pdf 解析密鑰（比自定義 PDF 解析優先級低）",
"custom_pdf_parse_price": "自定義 PDF 解析價格(n 積分/頁)",
"eval_config": "評估配置",
"task_concurrency": "任務並發數",
"task_concurrency_desc": "同時執行的評估任務數量",
"item_concurrency": "項目並發數",
"item_concurrency_desc": "單個任務中並發處理的項目數量",
"item_max_retry": "最大重試次數",
"item_max_retry_desc": "評估項目失敗時的最大重試次數",
"default_threshold": "默認閾值",
"default_threshold_desc": "評估的默認闾值 (0-1之間)",
"summary_concurrency": "總結並發數",
"data_quality_concurrency": "数據質量並發數",
"dataset_synthesize_concurrency": "數據合成並發數",
"smart_generate_concurrency": "智能生成並發數",
"max_upload_files_per_time": "單次最多上傳多少個文件",
"max_upload_files_per_time_desc": "用戶上傳知識庫時，每次上傳最多選擇多少個文件",
"max_upload_file_size": "上傳文件最大大小（M)",
Expand Down
86 changes: 38 additions & 48 deletions projects/app/src/pages/dashboard/evaluation/task/detail/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -907,34 +907,6 @@ const Detail = ({ taskId, currentTab }: Props) => {
onClick={handleRefresh}
/>
</MyTooltip>
<MyTooltip label={t('dashboard_evaluation:edit_action')} offset={[0, 15]}>
<IconButton
aria-label="edit"
size={'mdSquare'}
variant={'whitePrimary'}
icon={<MyIcon name={'edit'} w={4} />}
onClick={handleEdit}
/>
</MyTooltip>
<PopoverConfirm
content={t('dashboard_evaluation:confirm_delete_data_in_task')}
type="delete"
confirmText={t('dashboard_evaluation:delete_action')}
onConfirm={handleDelete}
Trigger={
<MyTooltip
label={t('dashboard_evaluation:delete_action')}
offset={[0, 15]}
>
<IconButton
aria-label="delete"
size={'mdSquare'}
variant={'whiteDanger'}
icon={<MyIcon name={'delete'} w={4} />}
/>
</MyTooltip>
}
/>
</>
)}

Expand All @@ -947,29 +919,47 @@ const Detail = ({ taskId, currentTab }: Props) => {
>
{t('dashboard_evaluation:view_full_response')}
</Button>
<PopoverConfirm
content={t('dashboard_evaluation:confirm_delete_data_in_task')}
type="delete"
confirmText={t('dashboard_evaluation:delete_action')}
onConfirm={handleDelete}
Trigger={
<MyTooltip
label={t('dashboard_evaluation:delete_action')}
offset={[0, 15]}
>
<IconButton
aria-label="delete"
size={'mdSquare'}
variant={'whiteDanger'}
icon={<MyIcon name={'delete'} w={4} />}
/>
</MyTooltip>
}
/>
</>
)}

{/* 排队中或评测中状态不显示任何按钮 */}
{/* 对于非排队/评测状态，显示编辑和删除按钮 */}
{selectedItem &&
selectedItem.status !== EvaluationStatusEnum.queuing &&
selectedItem.status !== EvaluationStatusEnum.evaluating && (
<>
<MyTooltip
label={t('dashboard_evaluation:edit_action')}
offset={[0, 15]}
>
<IconButton
aria-label="edit"
size={'mdSquare'}
variant={'whitePrimary'}
icon={<MyIcon name={'edit'} w={4} />}
onClick={handleEdit}
/>
</MyTooltip>
<PopoverConfirm
content={t('dashboard_evaluation:confirm_delete_data_in_task')}
type="delete"
confirmText={t('dashboard_evaluation:delete_action')}
onConfirm={handleDelete}
Trigger={
<MyTooltip
label={t('dashboard_evaluation:delete_action')}
offset={[0, 15]}
>
<IconButton
aria-label="delete"
size={'mdSquare'}
variant={'whiteDanger'}
icon={<MyIcon name={'delete'} w={4} />}
/>
</MyTooltip>
}
/>
</>
)}
</>
)}
</Flex>
Expand Down
0 ...service/core/evaluation/evaluator.test.ts → ...re/evaluation/evaluator/evaluator.test.ts
View file
Open in desktop
File renamed without changes.
0 ...es/service/core/evaluation/target.test.ts → ...ice/core/evaluation/target/target.test.ts
View file
Open in desktop
File renamed without changes.
Loading
Loading