Skip to content

Commit 7e22d70

Browse files
maxxgxmaxxgx
authored
feat(inspect): project deletion (#3192)
* add project deletion Signed-off-by: Max Xiang <xiangxiang.ma@intel.com> * return 409 if running jobs Signed-off-by: Max Xiang <xiangxiang.ma@intel.com> * fix tests Signed-off-by: Max Xiang <xiangxiang.ma@intel.com> * fix mypy Signed-off-by: Max Xiang <xiangxiang.ma@intel.com> * fix test Signed-off-by: Max Xiang <xiangxiang.ma@intel.com> * use pascal case for trainable models Signed-off-by: Max Xiang <xiangxiang.ma@intel.com> * remove unused import Signed-off-by: Max Xiang <xiangxiang.ma@intel.com> --------- Signed-off-by: Max Xiang <xiangxiang.ma@intel.com>
1 parent a7e8768 commit 7e22d70

File tree

16 files changed

+498
-59
lines changed

16 files changed

+498
-59
lines changed

‎application/backend/src/api/endpoints/project_endpoints.py‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66

77
from fastapi import APIRouter, Body, Depends, HTTPException, Query, status
88

9-
from api.dependencies import PaginationLimit, get_project_id, get_project_service
9+
from api.dependencies import PaginationLimit, get_job_service, get_pipeline_service, get_project_id, get_project_service
1010
from api.endpoints import API_PREFIX
1111
from pydantic_models import Project, ProjectList, ProjectUpdate
12-
from services import ProjectService
12+
from services import JobService, PipelineService, ProjectService
1313

1414
project_api_prefix_url = API_PREFIX + "/projects"
1515
project_router = APIRouter(
@@ -60,3 +60,34 @@ async def update_project(
6060
if project is None:
6161
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found")
6262
return project
63+
64+
65+
@project_router.delete("/{project_id}")
66+
async def delete_project(
67+
job_service: Annotated[JobService, Depends(get_job_service)],
68+
pipeline_service: Annotated[PipelineService, Depends(get_pipeline_service)],
69+
project_service: Annotated[ProjectService, Depends(get_project_service)],
70+
project_id: Annotated[UUID, Depends(get_project_id)],
71+
) -> None:
72+
"""Endpoint to delete a project by ID"""
73+
project = await project_service.get_project_by_id(project_id)
74+
if project is None:
75+
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found")
76+
active_pipeline = await pipeline_service.get_active_pipeline()
77+
if active_pipeline and active_pipeline.project_id == project_id and active_pipeline.is_running:
78+
raise HTTPException(
79+
status_code=status.HTTP_409_CONFLICT,
80+
detail="Cannot delete project with active pipeline. Please deactivate the pipeline first.",
81+
)
82+
if await job_service.has_running_jobs(project_id=project_id):
83+
raise HTTPException(
84+
status_code=status.HTTP_409_CONFLICT,
85+
detail="Cannot delete project with running jobs. Please cancel the jobs first.",
86+
)
87+
try:
88+
await project_service.delete_project(project_id)
89+
except RuntimeError as err:
90+
raise HTTPException(
91+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
92+
detail=f"Failed to delete project. Deletion rolled back. Error: {str(err)}",
93+
)

‎application/backend/src/api/endpoints/trainable_models_endpoints.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def _get_trainable_models() -> TrainableModelList: # pragma: no cover
2020
the model names are returned. Descriptions can be added manually in the
2121
``_MODEL_DESCRIPTIONS`` mapping below.
2222
"""
23-
model_names = sorted(list_models(case="snake"))
23+
model_names = sorted(list_models(case="pascal"))
2424

2525
return TrainableModelList(trainable_models=model_names)
2626

‎application/backend/src/repositories/base.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,24 @@ async def delete_by_id(self, obj_id: str | UUID) -> None:
155155
await self.db.execute(query)
156156
await self.db.commit()
157157

158+
async def delete_all(self, commit: bool, extra_filters: dict | None = None) -> None:
159+
"""Delete all records matching the filters.
160+
161+
Does NOT commit the transaction.
162+
"""
163+
if extra_filters is None:
164+
extra_filters = {}
165+
166+
combined_filters = extra_filters | self.base_filters
167+
168+
query = expression.delete(self.schema)
169+
if combined_filters:
170+
query = query.filter_by(**combined_filters)
171+
172+
await self.db.execute(query)
173+
if commit:
174+
await self.db.commit()
175+
158176
@staticmethod
159177
def _id_to_str(obj_id: str | UUID) -> str:
160178
if isinstance(obj_id, UUID):

‎application/backend/src/repositories/binary_repo.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@
1010

1111
from anomalib.deploy import ExportType
1212

13+
from pydantic_models.model import ExportParameters
14+
1315
STORAGE_ROOT_PATH = "data"
1416

1517

1618
class FileType(StrEnum):
1719
IMAGES = "images"
1820
MODELS = "models"
1921
SNAPSHOTS = "snapshots"
22+
MODEL_EXPORTS = "model_exports"
2023

2124

2225
class BinaryRepository(metaclass=abc.ABCMeta):
@@ -96,6 +99,17 @@ def stdlib_delete():
9699

97100
await asyncio.to_thread(stdlib_delete)
98101

102+
async def delete_project_folder(self) -> None:
103+
"""
104+
Delete the entire project folder for this file type.
105+
"""
106+
107+
def stdlib_delete_folder():
108+
if os.path.exists(self.project_folder_path):
109+
shutil.rmtree(self.project_folder_path)
110+
111+
await asyncio.to_thread(stdlib_delete_folder)
112+
99113

100114
class DatasetSnapshotBinaryRepository(BinaryRepository):
101115
def __init__(self, project_id: str | UUID):
@@ -162,3 +176,33 @@ def get_weights_file_path(self, format: ExportType, name: str) -> str:
162176
:return: path of the weights file.
163177
"""
164178
return os.path.join(self.model_folder_path, "weights", format, name)
179+
180+
181+
class ModelExportBinaryRepository(BinaryRepository):
182+
def __init__(self, project_id: str | UUID, model_id: str | UUID):
183+
super().__init__(project_id=project_id, file_type=FileType.MODEL_EXPORTS)
184+
self._model_id = str(model_id)
185+
186+
def get_full_path(self, filename: str) -> str:
187+
return os.path.join(self.model_export_folder_path, filename)
188+
189+
@cached_property
190+
def model_export_folder_path(self) -> str:
191+
"""
192+
Get the folder path for model exports.
193+
194+
:return: Folder path for model exports.
195+
"""
196+
return os.path.join(self.project_folder_path, self._model_id)
197+
198+
def get_model_export_path(self, model_name: str, export_params: ExportParameters) -> str:
199+
"""
200+
Get the full path for a dataset snapshot.
201+
202+
:param model_name: name of the model
203+
:param export_params: model export parameters
204+
:return: Full path to the model export zip file.
205+
"""
206+
compression_suffix = f"_{export_params.compression.value}" if export_params.compression else ""
207+
filename = f"{model_name}_{export_params.format.value}{compression_suffix}.zip"
208+
return self.get_full_path(filename)

‎application/backend/src/services/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
from .active_pipeline_service import ActivePipelineService
55
from .configuration_service import ConfigurationService
6+
from .dataset_snapshot_service import DatasetSnapshotService
67
from .dispatch_service import DispatchService
78
from .exceptions import (
89
ActivePipelineConflictError,
@@ -22,6 +23,7 @@
2223
"ActivePipelineConflictError",
2324
"ActivePipelineService",
2425
"ConfigurationService",
26+
"DatasetSnapshotService",
2527
"DispatchService",
2628
"JobService",
2729
"MediaService",

‎application/backend/src/services/configuration_service.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,3 +155,15 @@ async def delete_sink_by_id(self, sink_id: UUID, project_id: UUID) -> None:
155155
sink = await self.get_sink_by_id(sink_id, project_id, db)
156156
sink_repo = SinkRepository(db, project_id=project_id)
157157
await sink_repo.delete_by_id(sink.id)
158+
159+
@staticmethod
160+
async def delete_project_source_db(session: AsyncSession, project_id: UUID, commit: bool = False) -> None:
161+
"""Delete all sources associated with a project from the database."""
162+
source_repo = SourceRepository(session, project_id=project_id)
163+
await source_repo.delete_all(commit=commit)
164+
165+
@staticmethod
166+
async def delete_project_sink_db(session: AsyncSession, project_id: UUID, commit: bool = False) -> None:
167+
"""Delete all sinks associated with a project from the database."""
168+
sink_repo = SinkRepository(session, project_id=project_id)
169+
await sink_repo.delete_all(commit=commit)

‎application/backend/src/services/dataset_snapshot_service.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from loguru import logger
1313
from PIL import Image
1414
from sqlalchemy import func, select
15+
from sqlalchemy.ext.asyncio.session import AsyncSession
1516

1617
from db import get_async_db_session_ctx
1718
from db.schema import ModelDB
@@ -159,6 +160,23 @@ async def delete_snapshot_if_unused(cls, snapshot_id: UUID, project_id: UUID) ->
159160
except Exception as e:
160161
logger.error(f"Error deleting snapshot file {snapshot.filename}: {e}")
161162

163+
@classmethod
164+
async def delete_project_snapshots_db(cls, session: AsyncSession, project_id: UUID, commit: bool = False) -> None:
165+
"""Delete all snapshots associated with a project from the database."""
166+
snapshot_repo = DatasetSnapshotRepository(session, project_id=project_id)
167+
await snapshot_repo.delete_all(commit=commit)
168+
169+
@classmethod
170+
async def cleanup_project_snapshot_files(cls, project_id: UUID) -> None:
171+
"""Cleanup snapshot files for a project."""
172+
try:
173+
# Cleanup project folder (removes all files at once)
174+
snapshot_bin_repo = DatasetSnapshotBinaryRepository(project_id=project_id)
175+
await snapshot_bin_repo.delete_project_folder()
176+
logger.info(f"Cleaned up snapshot files for project {project_id}")
177+
except Exception as e:
178+
logger.warning(f"Failed to cleanup snapshot files for project {project_id}: {e}")
179+
162180
@staticmethod
163181
def extract_snapshot_to_path(snapshot_path: str, temp_dir: str) -> None:
164182
"""Extract images from Parquet snapshot to a temporary directory structure."""

‎application/backend/src/services/job_service.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import anyio
1111
from sqlalchemy.exc import IntegrityError
12+
from sqlalchemy.ext.asyncio.session import AsyncSession
1213
from sse_starlette import ServerSentEvent
1314

1415
from db import get_async_db_session_ctx
@@ -163,3 +164,18 @@ async def cancel_job(cls, job_id: UUID | str) -> JobCancelled:
163164

164165
await repo.update(job, {"status": JobStatus.CANCELED})
165166
return JobCancelled(job_id=job.id)
167+
168+
@classmethod
169+
async def delete_project_jobs_db(cls, session: AsyncSession, project_id: UUID, commit: bool = False) -> None:
170+
"""Delete all jobs associated with a project from the database."""
171+
repo = JobRepository(session)
172+
await repo.delete_all(commit=commit, extra_filters={"project_id": str(project_id)})
173+
174+
@staticmethod
175+
async def has_running_jobs(project_id: str | UUID) -> bool:
176+
async with get_async_db_session_ctx() as session:
177+
repo = JobRepository(session)
178+
count = await repo.get_all_count(
179+
extra_filters={"status": JobStatus.RUNNING, "project_id": str(project_id)},
180+
)
181+
return count > 0

‎application/backend/src/services/media_service.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import numpy as np
88
from loguru import logger
99
from PIL import Image
10+
from sqlalchemy.ext.asyncio.session import AsyncSession
1011

1112
from db import get_async_db_session_ctx
1213
from pydantic_models import Media, MediaList
@@ -152,6 +153,23 @@ async def delete_media(cls, media_id: UUID, project_id: UUID) -> None:
152153
await cls._delete_media_file(project_id=project_id, filename=thumbnail_filename)
153154
await ProjectRepository(session).update_dataset_timestamp(project_id=project_id)
154155

156+
@classmethod
157+
async def delete_project_media_db(cls, session: AsyncSession, project_id: UUID, commit: bool = False) -> None:
158+
"""Delete all media associated with a project from the database."""
159+
media_repo = MediaRepository(session, project_id=project_id)
160+
await media_repo.delete_all(commit=commit)
161+
162+
@classmethod
163+
async def cleanup_project_media_files(cls, project_id: UUID) -> None:
164+
"""Cleanup media files for a project."""
165+
try:
166+
# Cleanup project folder (removes all files at once)
167+
bin_repo = ImageBinaryRepository(project_id=project_id)
168+
await bin_repo.delete_project_folder()
169+
logger.info(f"Cleaned up media files for project {project_id}")
170+
except Exception as e:
171+
logger.warning(f"Failed to cleanup media files for project {project_id}: {e}")
172+
155173
@staticmethod
156174
async def _delete_media_file(project_id: UUID, filename: str) -> None:
157175
bin_repo = ImageBinaryRepository(project_id=project_id)

‎application/backend/src/services/model_service.py‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
from dataclasses import dataclass
99
from multiprocessing.synchronize import Event as EventClass
1010
from pathlib import Path
11-
from uuid import UUID
11+
from uuid import UUID, uuid4
1212

13+
import anyio
1314
import cv2
1415
import numpy as np
1516
import openvino.properties.hint as ov_hints
@@ -18,13 +19,14 @@
1819
from anomalib.models import get_model
1920
from loguru import logger
2021
from PIL import Image
22+
from sqlalchemy.ext.asyncio.session import AsyncSession
2123

2224
from db import get_async_db_session_ctx
2325
from pydantic_models import Model, ModelList, PredictionLabel, PredictionResponse
2426
from pydantic_models.base import Pagination
2527
from pydantic_models.model import ExportParameters
2628
from repositories import ModelRepository
27-
from repositories.binary_repo import ModelBinaryRepository
29+
from repositories.binary_repo import ModelBinaryRepository, ModelExportBinaryRepository
2830
from services import ResourceNotFoundError
2931
from services.dataset_snapshot_service import DatasetSnapshotService
3032
from services.exceptions import DeviceNotFoundError, ResourceType
@@ -114,6 +116,31 @@ async def delete_model(cls, project_id: UUID, model_id: UUID) -> None:
114116
repo = ModelRepository(session, project_id=project_id)
115117
await repo.delete_by_id(model_id)
116118

119+
@classmethod
120+
async def delete_project_models_db(cls, session: AsyncSession, project_id: UUID, commit: bool = False) -> None:
121+
"""Delete all models associated with a project from the database."""
122+
# We still need to handle side effects like snapshot reference counting if possible,
123+
# but since we are deleting the project, all snapshots will be deleted anyway.
124+
# So we can just delete the models.
125+
repo = ModelRepository(session, project_id=project_id)
126+
await repo.delete_all(commit=commit)
127+
128+
@classmethod
129+
async def cleanup_project_model_files(cls, project_id: UUID) -> None:
130+
"""Cleanup model files for a project."""
131+
try:
132+
# Cleanup project folder (removes all model folders at once)
133+
# Note: using dummy model_id since we are deleting the entire project folder
134+
model_binary_repo = ModelBinaryRepository(project_id=project_id, model_id=uuid4())
135+
await model_binary_repo.delete_project_folder()
136+
logger.info(f"Cleaned up model files for project {project_id}")
137+
138+
model_export_bin_repo = ModelExportBinaryRepository(project_id=project_id, model_id=uuid4())
139+
await model_export_bin_repo.delete_project_folder()
140+
logger.info(f"Cleaned up model export files for project {project_id}")
141+
except Exception as e:
142+
logger.warning(f"Failed to cleanup model files for project {project_id}: {e}")
143+
117144
async def export_model(self, project_id: UUID, model_id: UUID, export_parameters: ExportParameters) -> Path:
118145
"""Export a trained model to a zip file.
119146
@@ -129,18 +156,14 @@ async def export_model(self, project_id: UUID, model_id: UUID, export_parameters
129156
if model is None:
130157
raise ResourceNotFoundError(resource_type=ResourceType.MODEL, resource_id=str(model_id))
131158

132-
# Construct export path
133-
name = f"{model.project_id}-{model.name}"
134-
exports_dir = Path("data/exports") / str(model_id) / model.name.title() / name
135-
exports_dir.mkdir(parents=True, exist_ok=True)
136-
137-
compression_suffix = f"_{export_parameters.compression.value}" if export_parameters.compression else ""
138-
filename = f"{model.name}_{export_parameters.format.value}{compression_suffix}.zip"
139-
export_zip_path = exports_dir / filename
159+
bin_repo = ModelExportBinaryRepository(project_id=project_id, model_id=model_id)
160+
export_zip_path = anyio.Path(
161+
bin_repo.get_model_export_path(model_name=model.name, export_params=export_parameters)
162+
)
140163

141164
# Cache check
142-
if export_zip_path.exists():
143-
return export_zip_path
165+
if await export_zip_path.exists():
166+
return Path(export_zip_path)
144167

145168
# Locate checkpoint
146169
model_binary_repo = ModelBinaryRepository(project_id=project_id, model_id=model_id)
@@ -167,7 +190,7 @@ async def export_model(self, project_id: UUID, model_id: UUID, export_parameters
167190
model_name=model.name,
168191
ckpt_path=ckpt_path,
169192
export_parameters=export_parameters,
170-
export_zip_path=export_zip_path,
193+
export_zip_path=Path(export_zip_path),
171194
)
172195

173196
@staticmethod

0 commit comments

Comments
 (0)