Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion frontend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"dev": "vite",
"mock": "cd src/mock && nodemon --config nodemon.json --inspect=0.0.0.0:9229 mock.cjs --env=development --port=8002",
"build": "vite build",
"test": "node --test tests/basic-functionality.test.cjs tests/bootstrap-auth.test.cjs",
"test": "node --test tests/basic-functionality.test.cjs tests/bootstrap-auth.test.cjs tests/cleaning-logs.test.cjs",
"lint": "eslint .",
"preview": "vite preview"
},
Expand Down
2 changes: 2 additions & 0 deletions frontend/src/i18n/locales/en/common.json
Original file line number Diff line number Diff line change
Expand Up @@ -2338,11 +2338,13 @@
"evalTitle": "Create Evaluation Task",
"evalName": "Evaluation Task Name",
"evalMethod": {
"label": "Evaluation Method",
"auto": "Auto Evaluation",
"cot": "CoT Evaluation",
"qa": "QA Evaluation"
},
"evalModel": "Evaluation Model",
"evalObject": "Evaluation Object",
"evalSource": "Source: Synthesis Task (SYNTHESIS)",
"sourceName": "Source Name: {{name}}",
"startEval": "Start Evaluation",
Expand Down
2 changes: 2 additions & 0 deletions frontend/src/i18n/locales/zh/common.json
Original file line number Diff line number Diff line change
Expand Up @@ -2337,11 +2337,13 @@
"evalTitle": "创建评估任务",
"evalName": "评估任务名称",
"evalMethod": {
"label": "评估方式",
"auto": "自动评估",
"cot": "CoT 评估",
"qa": "QA 评估"
},
"evalModel": "评估模型",
"evalObject": "评估对象",
"evalSource": "来源:合成任务(SYNTHESIS)",
"sourceName": "源名称:{{name}}",
"startEval": "开始评估",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export default function LogsTable({
const [selectedLog, setSelectedLog] = useState(retryCount + 1);
const [streamingLogs, setStreamingLogs] = useState<LogEntry[]>([]);
const [isStreaming, setIsStreaming] = useState(false);
const eventSourceRef = useRef<EventSource | null>(null);
const eventSourceRef = useRef<ReturnType<typeof streamCleaningTaskLog> | null>(null);
const { message } = App.useApp();

// Only stream when task is RUNNING and viewing the latest run
Expand Down Expand Up @@ -123,7 +123,7 @@ export default function LogsTable({
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = `${taskName}_${t("dataCleansing.logTable.nthRun", { num: selectedLog })}.log`;
a.download = `${taskName}_${t("dataCleansing.detail.logTable.nthRun", { num: selectedLog })}.log`;
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
Expand Down
67 changes: 60 additions & 7 deletions frontend/src/pages/DataCleansing/cleansing.api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,71 @@ export function queryCleaningTaskLogByIdUsingGet(taskId: string | number, retryC
return get(`/api/cleaning/tasks/${taskId}/log/${retryCount}`);
}

export function streamCleaningTaskLog(taskId: string | number, retryCount: number = 0): EventSource {
export interface CleaningTaskLogStream {
onmessage: ((event: { data: string }) => void) | null;
onerror: ((error: unknown) => void) | null;
close: () => void;
}

export function streamCleaningTaskLog(taskId: string | number, retryCount: number = 0): CleaningTaskLogStream {
const url = `/api/cleaning/tasks/${taskId}/log/stream?retry_count=${retryCount}`;
return new EventSource(url);
const controller = new AbortController();
const stream: CleaningTaskLogStream = {
onmessage: null,
onerror: null,
close: () => controller.abort(),
};

// Fetch supports the Authorization header supplied by the request interceptor.
void (async () => {
try {
const response = await get(url, null, {
responseType: "stream",
headers: { Accept: "text/event-stream" },
signal: controller.signal,
});
if (!response.headers.get("content-type")?.includes("text/event-stream") || !response.body) {
throw new Error("Invalid log stream response");
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = "";
let data: string[] = [];
try {
while (!controller.signal.aborted) {
const { value, done } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
let end: number;
while (!controller.signal.aborted && (end = buffer.indexOf("\n")) !== -1) {
const line = buffer.slice(0, end).replace(/\r$/, "");
buffer = buffer.slice(end + 1);
if (line === "") {
if (data.length) stream.onmessage?.({ data: data.join("\n") });
data = [];
} else if (line === "data" || line.startsWith("data:")) {
data.push(line.slice(5).replace(/^ /, ""));
}
}
}
} finally {
await reader.cancel();
reader.releaseLock();
}
if (!controller.signal.aborted) {
throw new Error("Log stream ended");
}
} catch (error) {
if (!controller.signal.aborted) stream.onerror?.(error);
}
})();

return stream;
}

export async function downloadCleaningTaskLog(taskId: string | number, retryCount: number = 0): Promise<Blob> {
const url = `/api/cleaning/tasks/${taskId}/log/${retryCount}/download`;
const response = await fetch(url);
if (!response.ok) {
throw new Error('Failed to download log file');
}
return response.blob();
return get(url, null, { responseType: "blob" });
}

export function updateCleaningTaskByIdUsingPut(taskId: string | number, data: any) {
Expand Down
4 changes: 3 additions & 1 deletion frontend/src/pages/DataEvaluation/evaluation.const.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ export function getTaskTypes(t: TFunction) {
export function getEvalMethods(t: TFunction) {
return [
{ label: t("dataEvaluation.create.evalMethods.auto"), value: "AUTO" },
{ label: t("dataEvaluation.create.evalMethods.manual"), value: "MANUAL" },
];
}

Expand All @@ -33,6 +32,9 @@ export const getEvalType = (type: string, t?: TFunction) => {
};

export const getEvalMethod = (type: string, t?: TFunction) => {
if (type === "MANUAL") {
return t ? t("dataEvaluation.create.evalMethods.manual") : "人工评估";
}
if (t) return getEvalMethods(t).find((item) => item.value === type)?.label;
return EVAL_METHODS.find((item) => item.value === type)?.label;
};
Expand Down
14 changes: 0 additions & 14 deletions frontend/src/pages/SynthesisTask/DataSynthesis.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { PlusOutlined } from "@ant-design/icons";
import { Plus, ArrowRight } from "lucide-react";
import { useNavigate } from "react-router";
import { useTranslation } from "react-i18next";
import InstructionTemplateTab from "./components/InstructionTemplateTab";
import SynthesisTaskTab from "./components/SynthesisTaskTab";

export default function DataSynthesisPage() {
Expand Down Expand Up @@ -36,14 +35,6 @@ export default function DataSynthesisPage() {
<h2 className="text-xl font-bold text-gray-900">{t('synthesisTask.title')}</h2>
</div>
<div className="flex items-center gap-2">
<Button
onClick={() => {
navigate("/data/synthesis/task/create-template");
}}
icon={<PlusOutlined />}
>
{t('synthesisTask.actions.createTemplate')}
</Button>
<Button
type="primary"
onClick={() => navigate("/data/synthesis/task/create")}
Expand All @@ -57,11 +48,6 @@ export default function DataSynthesisPage() {
<Tabs
items={[
{ key: "tasks", label: t('synthesisTask.tabs.tasks'), children: <SynthesisTaskTab /> },
{
key: "templates",
label: t('synthesisTask.tabs.templates'),
children: <InstructionTemplateTab />,
},
]}
activeKey={activeTab}
onChange={setActiveTab}
Expand Down
16 changes: 9 additions & 7 deletions frontend/src/pages/SynthesisTask/components/SynthesisTaskTab.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,7 @@ export default function SynthesisTaskTab() {
/>
</Form.Item>
<Form.Item
label={t('synthesisTask.home.modal.evalMethod')}
label={t('synthesisTask.home.modal.evalMethod.label')}
name="evalMethod"
rules={[{ required: true, message: t('synthesisTask.home.modal.evalMethodRequired') }]}
>
Expand All @@ -555,23 +555,25 @@ export default function SynthesisTaskTab() {
/>
</Form.Item>
<Form.Item
label="评估模型"
label={t('synthesisTask.home.modal.evalModel')}
name="modelId"
rules={[{ required: true, message: "请选择评估模型" }]}
rules={[{ required: true, message: t('dataEvaluation.create.messages.modelRequired') }]}
>
<Select
placeholder={modelLoading ? "加载模型中..." : "请选择用于评估的模型"}
placeholder={modelLoading
? t('synthesisTask.home.modal.placeholders.modelLoading')
: t('synthesisTask.home.modal.placeholders.selectModel')}
loading={modelLoading}
options={chatModelOptions}
showSearch
optionFilterProp="label"
/>
</Form.Item>
{currentEvalTask && (
<Form.Item label="评估对象">
<Form.Item label={t('synthesisTask.home.modal.evalObject')}>
<div className="text-xs text-gray-500">
源类型:合成任务(SYNTHESIS)<br />
源名称:{currentEvalTask.name}
{t('synthesisTask.home.modal.evalSource')}<br />
{t('synthesisTask.home.modal.sourceName', { name: currentEvalTask.name })}
</div>
</Form.Item>
)}
Expand Down
11 changes: 10 additions & 1 deletion frontend/src/utils/request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,13 @@ class Request {
throw error;
}

if (config.responseType === "stream") {
return processedResponse;
}
if (config.responseType === "blob") {
return await processedResponse.blob();
}

// 检查响应是否为空
const contentType = processedResponse.headers.get("content-type");
if (contentType && contentType.includes("application/json")) {
Expand Down Expand Up @@ -600,7 +607,9 @@ request.addResponseInterceptor(async (response, config) => {
// 关键点 1: response.data 在原生 fetch 中是不存在的,必须用 .json() 解析
// 关键点 2: 必须用 .clone(),因为流只能读一次。读了克隆的,原版 response 还能留给外面用
// 关键点 3: 必须 await,因为读取流是异步的
resData = await response.clone().json();
resData = response.headers.get("content-type")?.includes("application/json")
? await response.clone().json()
: {};
} catch (e) {
// 如果后端返回的不是 JSON (比如 404 HTML 页面,或者空字符串),json() 会报错
// 这里捕获异常,保证 resData 至少是个空对象,不会导致后面取值 crash
Expand Down
118 changes: 118 additions & 0 deletions frontend/tests/cleaning-logs.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { loadSource } = require('./helpers.cjs');

const tick = () => new Promise(setImmediate);

function fixture(fetch, session = JSON.stringify({ token: 'test-jwt' })) {
const events = [];
const request = loadSource('src/utils/request.ts', {
antd: { message: { error() {} } },
'./loading': { __esModule: true, default: { show() {}, hide() {} } },
'@/utils/errorConfigStore.ts': { errorConfigStore: { getMessage() {} } },
'@/i18n': { __esModule: true, default: { language: 'zh' } },
}, {
fetch,
localStorage: { getItem: key => key === 'session' ? session : null, removeItem() {} },
window: { dispatchEvent: event => events.push(event.type) },
CustomEvent: class { constructor(type) { this.type = type; } },
});
request.setRequireLoginMode(true);
const api = loadSource('src/pages/DataCleansing/cleansing.api.ts', {
'@/utils/request': request,
}, { AbortController, TextDecoder });
return { api, events };
}

test('log stream sends the JWT header and delivers split UTF-8 SSE messages before EOF', async () => {
let pending;
let body;
let cancelled = false;
const { api } = fixture(async (url, config) => {
pending = { url, config };
return new Response(new ReadableStream({
start(controller) { body = controller; },
cancel() { cancelled = true; },
}), { headers: { 'content-type': 'text/event-stream' } });
});
const messages = [];
const errors = [];
const stream = api.streamCleaningTaskLog('task-1', 2);
stream.onmessage = event => {
messages.push(event.data);
if (event.data.includes('[END_OF_STREAM]')) stream.close();
};
stream.onerror = error => errors.push(error);
await tick();
assert.equal(pending.url, '/api/cleaning/tasks/task-1/log/stream?retry_count=2');
assert.equal(pending.config.headers.Authorization, 'Bearer test-jwt');
assert.equal(pending.config.credentials, 'include');
assert.equal(pending.config.headers.Accept, 'text/event-stream');
assert.equal(pending.url.includes('token='), false);

const encoded = new TextEncoder().encode(': heartbeat\r\ndata: {"message":"日志"}\r\n\r\n');
for (const byte of encoded) body.enqueue(Uint8Array.of(byte));
await tick();
assert.deepEqual(messages, ['{"message":"日志"}']);
body.enqueue(new TextEncoder().encode('data: {"message":"[END_OF_STREAM]"}\n\n'));
await tick();
assert.equal(messages.length, 2);
assert.equal(pending.config.signal.aborted, true);
assert.equal(cancelled, true);
assert.deepEqual(errors, []);
});

test('closing a log stream aborts its request without reporting an error', async () => {
let signal;
const { api } = fixture((_url, config) => new Promise((_resolve, reject) => {
signal = config.signal;
signal.addEventListener('abort', () => reject(new DOMException('Aborted', 'AbortError')));
}));
const errors = [];
const stream = api.streamCleaningTaskLog('task-1');
stream.onerror = error => errors.push(error);
await tick();
stream.close();
await tick();
assert.equal(signal.aborted, true);
assert.deepEqual(errors, []);
});

test('log downloads send authentication and preserve the downloaded bytes', async () => {
const bytes = Uint8Array.of(0, 255, 10, 13, 65);
const { api } = fixture(async (url, config) => {
assert.equal(url, '/api/cleaning/tasks/task-1/log/1/download');
assert.equal(config.headers.Authorization, 'Bearer test-jwt');
return new Response(bytes, { headers: { 'content-type': 'application/octet-stream' } });
});
const blob = await api.downloadCleaningTaskLog('task-1', 1);
assert.deepEqual(new Uint8Array(await blob.arrayBuffer()), bytes);
});

test('stream and download authentication failures use the shared login handling', async () => {
const { api, events } = fixture(async () => new Response(
JSON.stringify({ code: 'common.0401', message: 'expired' }),
{ status: 401, headers: { 'content-type': 'application/json' } },
));
const error = await new Promise(resolve => {
const stream = api.streamCleaningTaskLog('task-1');
stream.onerror = resolve;
});
assert.equal(error.status, 401);
assert.deepEqual(events, ['show-login']);
await assert.rejects(api.downloadCleaningTaskLog('task-1'), error => error.status === 401);
});

test('log streams work without a session when login is optional', async () => {
const { api } = fixture(async (_url, config) => {
assert.equal(config.headers.Authorization, undefined);
return new Response('data: done\n\n', { headers: { 'content-type': 'text/event-stream' } });
}, null);
const messages = [];
await new Promise(resolve => {
const stream = api.streamCleaningTaskLog('task-1');
stream.onmessage = event => messages.push(event.data);
stream.onerror = resolve;
});
assert.deepEqual(messages, ['done']);
});
Loading
Loading