Skip to content

Commit 45743f3

Browse files
JasonW404JasonW404
authored
feat: add labeling template. refactor: switch to Poetry, build and deploy of backend Python (#79)
* feat: Enhance annotation module with template management and validation - Added DatasetMappingCreateRequest and DatasetMappingUpdateRequest schemas to handle dataset mapping requests with camelCase and snake_case support. - Introduced Annotation Template schemas including CreateAnnotationTemplateRequest, UpdateAnnotationTemplateRequest, and AnnotationTemplateResponse for managing annotation templates. - Implemented AnnotationTemplateService for creating, updating, retrieving, and deleting annotation templates, including validation of configurations and XML generation. - Added utility class LabelStudioConfigValidator for validating Label Studio configurations and XML formats. - Updated database schema for annotation templates and labeling projects to include new fields and constraints. - Seeded initial annotation templates for various use cases including image classification, object detection, and text classification. * feat: Enhance TemplateForm with improved validation and dynamic field rendering; update LabelStudio config validation for camelCase support * feat: Update docker-compose.yml to mark datamate dataset volume and network as external * feat: Add tag configuration management and related components - Introduced new components for tag selection and browsing in the frontend. - Added API endpoint to fetch tag configuration from the backend. - Implemented tag configuration management in the backend, including loading from YAML. - Enhanced template service to support dynamic tag rendering based on configuration. - Updated validation utilities to incorporate tag configuration checks. - Refactored existing code to utilize the new tag configuration structure. * feat: Refactor LabelStudioTagConfig for improved configuration loading and validation * feat: Update Makefile to include backend-python-docker-build in the build process * feat: Migrate to poetry for better deps management * Add pyyaml dependency and update Dockerfile to use Poetry for dependency management - Added pyyaml (>=6.0.3,<7.0.0) to pyproject.toml dependencies. - Updated Dockerfile to install Poetry and manage dependencies using it. - Improved layer caching by copying only dependency files before the application code. - Removed unnecessary installation of build dependencies to keep the final image size small. * feat: Remove duplicated backend-python-docker-build target from Makefile * fix: airflow is not ready for adding yet * feat: update Python version to 3.12 and remove project installation step in Dockerfile
1 parent 2660845 commit 45743f3

40 files changed

+3222
-261
lines changed

‎Makefile‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ build-%:
1818
$(MAKE) $*-docker-build
1919

2020
.PHONY: build
21-
build: database-docker-build backend-docker-build frontend-docker-build runtime-docker-build
21+
build: backend-docker-build frontend-docker-build runtime-docker-build backend-python-docker-build
2222

2323
.PHONY: create-namespace
2424
create-namespace:
@@ -117,9 +117,9 @@ frontend-docker-build:
117117
runtime-docker-build:
118118
docker build -t datamate-runtime:$(VERSION) . -f scripts/images/runtime/Dockerfile
119119

120-
.PHONY: label-studio-adapter-docker-build
121-
label-studio-adapter-docker-build:
122-
docker build -t label-studio-adapter:$(VERSION) . -f scripts/images/label-studio-adapter/Dockerfile
120+
.PHONY: backend-python-docker-build
121+
backend-python-docker-build:
122+
docker build -t datamate-backend-python:$(VERSION) . -f scripts/images/datamate-python/Dockerfile
123123

124124
.PHONY: deer-flow-docker-build
125125
deer-flow-docker-build:
@@ -132,10 +132,6 @@ deer-flow-docker-build:
132132
mineru-docker-build:
133133
docker build -t datamate-mineru:$(VERSION) . -f scripts/images/mineru/Dockerfile
134134

135-
.PHONY: backend-python-docker-build
136-
backend-python-docker-build:
137-
docker build -t datamate-backend-python:$(VERSION) . -f scripts/images/datamate-python/Dockerfile
138-
139135
.PHONY: backend-docker-install
140136
backend-docker-install:
141137
cd deployment/docker/datamate && docker compose up -d backend

‎frontend/src/hooks/useFetchData.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ export default function useFetchData<T>(
110110
status: getFirstOfArray(filter?.status) || undefined,
111111
tags: filter?.tags?.length ? filter.tags.join(",") : undefined,
112112
page: current - pageOffset,
113-
size: pageSize,
113+
pageSize: pageSize, // Use camelCase for HTTP query params
114114
}),
115115
...additionalPollingFuncs.map((func) => func()),
116116
];

‎frontend/src/hooks/useTagConfig.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { useState, useEffect } from "react";
2+
import { message } from "antd";
3+
import { getTagConfigUsingGet } from "../pages/DataAnnotation/annotation.api";
4+
import type { LabelStudioTagConfig } from "../pages/DataAnnotation/annotation.tagconfig";
5+
import { parseTagConfig, type TagOption } from "../pages/DataAnnotation/annotation.tagconfig";
6+
7+
interface UseTagConfigReturn {
8+
config: LabelStudioTagConfig | null;
9+
objectOptions: TagOption[];
10+
controlOptions: TagOption[];
11+
loading: boolean;
12+
error: string | null;
13+
refetch: () => Promise<void>;
14+
}
15+
16+
/**
17+
* Hook to fetch and manage Label Studio tag configuration
18+
* @param includeLabelingOnly - If true, only include controls with category="labeling" (default: true)
19+
*/
20+
export function useTagConfig(includeLabelingOnly: boolean = true): UseTagConfigReturn {
21+
const [config, setConfig] = useState<LabelStudioTagConfig | null>(null);
22+
const [objectOptions, setObjectOptions] = useState<TagOption[]>([]);
23+
const [controlOptions, setControlOptions] = useState<TagOption[]>([]);
24+
const [loading, setLoading] = useState(true);
25+
const [error, setError] = useState<string | null>(null);
26+
27+
const fetchConfig = async () => {
28+
setLoading(true);
29+
setError(null);
30+
try {
31+
const response = await getTagConfigUsingGet();
32+
if (response.code === 200 && response.data) {
33+
const tagConfig: LabelStudioTagConfig = response.data;
34+
setConfig(tagConfig);
35+
36+
const { objectOptions: objects, controlOptions: controls } =
37+
parseTagConfig(tagConfig, includeLabelingOnly);
38+
setObjectOptions(objects);
39+
setControlOptions(controls);
40+
} else {
41+
const errorMsg = response.message || "获取标签配置失败";
42+
setError(errorMsg);
43+
message.error(errorMsg);
44+
}
45+
} catch (err: any) {
46+
const errorMsg = err.message || "加载标签配置时出错";
47+
setError(errorMsg);
48+
console.error("Failed to fetch tag config:", err);
49+
message.error(errorMsg);
50+
} finally {
51+
setLoading(false);
52+
}
53+
};
54+
55+
useEffect(() => {
56+
fetchConfig();
57+
}, []);
58+
59+
return {
60+
config,
61+
objectOptions,
62+
controlOptions,
63+
loading,
64+
error,
65+
refetch: fetchConfig,
66+
};
67+
}

‎frontend/src/pages/DataAnnotation/Create/components/CreateAnnotationTaskDialog.tsx‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,16 @@ export default function CreateAnnotationTask({
2929
// Fetch datasets
3030
const { data: datasetData } = await queryDatasetsUsingGet({
3131
page: 0,
32-
size: 1000,
32+
pageSize: 1000, // Use camelCase for HTTP params
3333
});
3434
setDatasets(datasetData.content.map(mapDataset) || []);
3535

3636
// Fetch templates
3737
const templateResponse = await queryAnnotationTemplatesUsingGet({
3838
page: 1,
39-
size: 100, // Backend max is 100
39+
size: 100, // Backend max is 100 (template API uses 'size' not 'pageSize')
4040
});
41-
41+
4242
// The API returns: {code, message, data: {content, total, page, ...}}
4343
if (templateResponse.code === 200 && templateResponse.data) {
4444
const fetchedTemplates = templateResponse.data.content || [];
@@ -68,15 +68,13 @@ export default function CreateAnnotationTask({
6868
try {
6969
const values = await form.validateFields();
7070
setSubmitting(true);
71-
7271
// Send templateId instead of labelingConfig
7372
const requestData = {
7473
name: values.name,
7574
description: values.description,
7675
datasetId: values.datasetId,
7776
templateId: values.templateId,
7877
};
79-
8078
await createAnnotationTaskUsingPost(requestData);
8179
message?.success?.("创建标注任务成功");
8280
onClose();
@@ -154,7 +152,6 @@ export default function CreateAnnotationTask({
154152
/>
155153
</Form.Item>
156154
</div>
157-
158155
{/* 描述变为可选 */}
159156
<Form.Item label="描述" name="description">
160157
<TextArea placeholder="（可选）详细描述标注任务的要求和目标" rows={3} />

‎frontend/src/pages/DataAnnotation/Create/components/CreateAnnptationTaskDialog.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@ export default function CreateAnnotationTask({
2929
// Fetch datasets
3030
const { data: datasetData } = await queryDatasetsUsingGet({
3131
page: 0,
32-
size: 1000,
32+
pageSize: 1000, // Use camelCase for HTTP params
3333
});
3434
setDatasets(datasetData.content.map(mapDataset) || []);
3535

3636
// Fetch templates
3737
const templateResponse = await queryAnnotationTemplatesUsingGet({
3838
page: 1,
39-
size: 100, // Backend max is 100
39+
size: 100, // Backend max is 100 (template API uses 'size' not 'pageSize')
4040
});
4141

4242
// The API returns: {code, message, data: {content, total, page, ...}}

‎frontend/src/pages/DataAnnotation/Home/DataAnnotation.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ export default function DataAnnotation() {
111111
cancelText: "取消",
112112
onOk: async () => {
113113
try {
114-
await deleteAnnotationTaskByIdUsingDelete({ m: task.id, proj: task.labelingProjId });
114+
await deleteAnnotationTaskByIdUsingDelete(task.id);
115115
message.success("映射删除成功");
116116
fetchData();
117117
// clear selection if deleted item was selected
@@ -198,7 +198,7 @@ export default function DataAnnotation() {
198198
onOk: async () => {
199199
try {
200200
await Promise.all(
201-
selectedRows.map((r) => deleteAnnotationTaskByIdUsingDelete({ m: r.id, proj: r.labelingProjId }))
201+
selectedRows.map((r) => deleteAnnotationTaskByIdUsingDelete(r.id))
202202
);
203203
message.success("批量删除已完成");
204204
fetchData();

‎frontend/src/pages/DataAnnotation/Template/TemplateForm.tsx‎

Lines changed: 3 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {
1717
updateAnnotationTemplateByIdUsingPut,
1818
} from "../annotation.api";
1919
import type { AnnotationTemplate } from "../annotation.model";
20+
import TagSelector from "./components/TagSelector";
2021

2122
const { TextArea } = Input;
2223
const { Option } = Select;
@@ -111,22 +112,6 @@ const TemplateForm: React.FC<TemplateFormProps> = ({
111112
}
112113
};
113114

114-
const controlTypes = [
115-
{ value: "Choices", label: "选项 (单选/多选)" },
116-
{ value: "RectangleLabels", label: "矩形框 (目标检测)" },
117-
{ value: "PolygonLabels", label: "多边形" },
118-
{ value: "Labels", label: "标签 (文本高亮)" },
119-
{ value: "TextArea", label: "文本区域" },
120-
{ value: "Rating", label: "评分" },
121-
];
122-
123-
const objectTypes = [
124-
{ value: "Image", label: "图像" },
125-
{ value: "Text", label: "文本" },
126-
{ value: "Audio", label: "音频" },
127-
{ value: "Video", label: "视频" },
128-
];
129-
130115
const needsOptions = (type: string) => {
131116
return ["Choices", "RectangleLabels", "PolygonLabels", "Labels"].includes(type);
132117
};
@@ -243,13 +228,7 @@ const TemplateForm: React.FC<TemplateFormProps> = ({
243228
rules={[{ required: true, message: "必填" }]}
244229
style={{ marginBottom: 0, width: 150 }}
245230
>
246-
<Select>
247-
{objectTypes.map((t) => (
248-
<Option key={t.value} value={t.value}>
249-
{t.label}
250-
</Option>
251-
))}
252-
</Select>
231+
<TagSelector type="object" />
253232
</Form.Item>
254233

255234
<Form.Item
@@ -356,13 +335,7 @@ const TemplateForm: React.FC<TemplateFormProps> = ({
356335
rules={[{ required: true, message: "必填" }]}
357336
style={{ marginBottom: 0 }}
358337
>
359-
<Select placeholder="选择控件类型">
360-
{controlTypes.map((t) => (
361-
<Option key={t.value} value={t.value}>
362-
{t.label}
363-
</Option>
364-
))}
365-
</Select>
338+
<TagSelector type="control" />
366339
</Form.Item>
367340

368341
<Form.Item

0 commit comments

Comments
 (0)