Skip to content

fix: fix eval item processor logic #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
chanzhi82020 wants to merge 1 commit into
base: eval-dev
Choose a base branch
from
Draft

fix: fix eval item processor logic #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 1 addition & 54 deletions packages/global/core/evaluation/api.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ export type ListEvaluationsRequest = PaginationProps<{
searchKey?: string;
appName?: string;
appId?: string;
versionId?: string;
}>;
export type ListEvaluationsResponse = PaginationResponse<EvaluationDisplayType>;

Expand Down Expand Up @@ -79,13 +78,7 @@ export type ListEvaluationItemsResponse = PaginationResponse<EvaluationItemDispl

// Get Evaluation Item Detail
export type EvaluationItemDetailRequest = EvalItemIdQuery;
export type EvaluationItemDetailResponse = {
item: EvaluationItemSchemaType;
dataItem: any;
response?: string;
result?: any;
score?: number;
};
export type EvaluationItemDetailResponse = EvaluationItemSchemaType;

// Update Evaluation Item
export type UpdateEvaluationItemRequest = EvalItemIdQuery & Partial<EvaluationDataItemType>;
Expand All @@ -98,49 +91,3 @@ export type RetryEvaluationItemResponse = MessageResponse;
// Delete Evaluation Item
export type DeleteEvaluationItemRequest = EvalItemIdQuery;
export type DeleteEvaluationItemResponse = MessageResponse;

// ===== DataItem Aggregation API =====

// Query for dataItem ID
export type DataItemIdQuery = { dataItemId: string };

// DataItem List (Grouped by DataItem)
export type DataItemListRequest = PaginationProps<
EvalIdQuery & {
status?: number; // Optional: filter by status
keyword?: string; // Optional: search in dataItem content
}
>;
export type DataItemGroupedItem = {
dataItemId: string;
dataItem: EvaluationDataItemType;
items: EvaluationItemDisplayType[];
statistics?: EvaluationStatistics;
};
export type DataItemListResponse = PaginationResponse<DataItemGroupedItem>;

// Delete DataItem Items
export type DeleteDataItemRequest = DataItemIdQuery & EvalIdQuery;
export type DeleteDataItemResponse = {
message: string;
deletedCount: number;
};

// Retry DataItem Items
export type RetryDataItemRequest = DataItemIdQuery & EvalIdQuery;
export type RetryDataItemResponse = {
message: string;
retriedCount: number;
};

// Update DataItem Items
export type UpdateDataItemRequest = DataItemIdQuery & EvalIdQuery & Partial<EvaluationDataItemType>;
export type UpdateDataItemResponse = {
message: string;
updatedCount: number;
};

// Export All DataItems Results
export type ExportDataItemsResultsRequest = EvalIdQuery & {
format?: 'csv' | 'json';
};
10 changes: 4 additions & 6 deletions packages/global/core/evaluation/type.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,17 +101,17 @@ export type EvaluationDataItemType = EvalDatasetDataSchemaType & {
targetCallParams?: TargetCallParams;
};

// Evaluation item type (atomic: one dataItem + one target + one evaluator)
// Evaluation item type (batch: one dataItem + one target + multiple evaluators)
export type EvaluationItemSchemaType = {
_id: string;
evalId: string;
// Dependent component configurations
dataItem: EvaluationDataItemType;
target: EvalTarget;
evaluator: EvaluatorSchema; // Single evaluator configuration
evaluators: EvaluatorSchema[]; // Multiple evaluator configurations
// Execution results
targetOutput?: TargetOutput; // Actual output from target
evaluatorOutput?: MetricResult; // Result from single evaluator
evaluatorOutputs?: MetricResult[]; // Results from multiple evaluators
status: EvaluationStatusEnum;
retry: number;
finishTime?: Date;
Expand Down Expand Up @@ -157,9 +157,7 @@ export type EvaluationDisplayType = Pick<
sourceMember: SourceMemberType;
};

export type EvaluationItemDisplayType = EvaluationItemSchemaType & {
evalItemId: string;
};
export type EvaluationItemDisplayType = EvaluationItemSchemaType;

export interface CreateEvaluationParams {
name: string;
Expand Down
72 changes: 36 additions & 36 deletions packages/service/core/evaluation/target/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,42 +112,42 @@ export class WorkflowTarget extends EvaluationTarget {
// Construct conversation history based on input.context
const histories: (UserChatItemType | AIChatItemType)[] = [];

if (input.context && input.context.length > 0) {
// Convert context strings to alternating user-ai conversation history
// Assume context format: [user1, ai1, user2, ai2, ...]
for (let i = 0; i < input.context.length; i++) {
const isUser = i % 2 === 0;
const content = input.context[i];

if (isUser) {
// User message
histories.push({
obj: ChatRoleEnum.Human,
value: [
{
type: ChatItemValueTypeEnum.text,
text: {
content
}
}
]
});
} else {
// AI message
histories.push({
obj: ChatRoleEnum.AI,
value: [
{
type: ChatItemValueTypeEnum.text,
text: {
content
}
}
]
});
}
}
}
// if (input.histories && input.histories.length > 0) {
// // Convert histories strings to alternating user-ai conversation history
// // Assume histories format: [user1, ai1, user2, ai2, ...]
// for (let i = 0; i < input.histories.length; i++) {
// const isUser = i % 2 === 0;
// const content = input.histories[i];

// if (isUser) {
// // User message
// histories.push({
// obj: ChatRoleEnum.Human,
// value: [
// {
// type: ChatItemValueTypeEnum.text,
// text: {
// content
// }
// }
// ]
// });
// } else {
// // AI message
// histories.push({
// obj: ChatRoleEnum.AI,
// value: [
// {
// type: ChatItemValueTypeEnum.text,
// text: {
// content
// }
// }
// ]
// });
// }
// }
// }

const chatId = getNanoid();

Expand Down
Loading
Loading