|
29 | 29 | from lib.app.helpers import get_annotation_paths |
30 | 30 | from lib.app.helpers import get_paths_and_duplicated_from_csv |
31 | 31 | from lib.app.helpers import reformat_metrics_json |
| 32 | +from lib.app.interface.types import AnnotationStatuses |
32 | 33 | from lib.app.interface.types import AnnotationType |
33 | 34 | from lib.app.interface.types import NotEmptyStr |
34 | 35 | from lib.app.interface.types import Status |
@@ -1646,7 +1647,7 @@ def upload_images_from_s3_bucket_to_project( |
1646 | 1647 | def prepare_export( |
1647 | 1648 | project: Union[NotEmptyStr, dict], |
1648 | 1649 | folder_names: Optional[List[NotEmptyStr]] = None, |
1649 | | - annotation_statuses: Optional[List[NotEmptyStr]] = None, |
| 1650 | + annotation_statuses: Optional[List[AnnotationStatuses]] = None, |
1650 | 1651 | include_fuse: Optional[StrictBool] = False, |
1651 | 1652 | only_pinned=False, |
1652 | 1653 | ): |
@@ -2152,38 +2153,50 @@ def download_export( |
2152 | 2153 | """ |
2153 | 2154 | project_name, folder_name = extract_project_folder(project) |
2154 | 2155 | export_name = export["name"] if isinstance(export, dict) else export |
2155 | | - response = controller.download_export( |
2156 | | - project_name=project_name, |
2157 | | - export_name=export_name, |
2158 | | - folder_path=folder_path, |
2159 | | - extract_zip_contents=extract_zip_contents, |
2160 | | - to_s3_bucket=to_s3_bucket, |
2161 | | - ) |
2162 | | - downloaded_folder_path = response.data |
2163 | 2156 |
|
2164 | 2157 | if to_s3_bucket: |
2165 | | - to_s3_bucket = boto3.Session().resource("s3").Bucket(to_s3_bucket) |
2166 | | - |
2167 | | - files_to_upload = [] |
2168 | | - for file in Path(downloaded_folder_path).rglob("*.*"): |
2169 | | - files_to_upload.append(file) |
2170 | | - |
2171 | | - def _upload_file_to_s3(to_s3_bucket, path, s3_key) -> None: |
2172 | | - controller.upload_file_to_s3( |
2173 | | - to_s3_bucket=to_s3_bucket, path=path, s3_key=s3_key |
| 2158 | + with tempfile.TemporaryDirectory() as tmp: |
| 2159 | + response = controller.download_export( |
| 2160 | + project_name=project_name, |
| 2161 | + export_name=export_name, |
| 2162 | + folder_path=tmp, |
| 2163 | + extract_zip_contents=extract_zip_contents, |
| 2164 | + to_s3_bucket=to_s3_bucket, |
2174 | 2165 | ) |
| 2166 | + downloaded_folder_path = response.data |
| 2167 | + if to_s3_bucket: |
| 2168 | + to_s3_bucket = boto3.Session().resource("s3").Bucket(to_s3_bucket) |
| 2169 | + files_to_upload = [] |
| 2170 | + for file in Path(downloaded_folder_path).rglob("*.*"): |
| 2171 | + files_to_upload.append(file) |
| 2172 | + |
| 2173 | + def _upload_file_to_s3(to_s3_bucket, path, s3_key) -> None: |
| 2174 | + controller.upload_file_to_s3( |
| 2175 | + to_s3_bucket=to_s3_bucket, path=path, s3_key=s3_key |
| 2176 | + ) |
2175 | 2177 |
|
2176 | | - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: |
2177 | | - results = [] |
2178 | | - for path in files_to_upload: |
2179 | | - s3_key = f"{path.as_posix()}" |
2180 | | - results.append( |
2181 | | - executor.submit(_upload_file_to_s3, to_s3_bucket, str(path), s3_key) |
| 2178 | + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: |
| 2179 | + results = [] |
| 2180 | + for path in files_to_upload: |
| 2181 | + s3_key = f"{folder_path}/{path.name}" |
| 2182 | + results.append( |
| 2183 | + executor.submit( |
| 2184 | + _upload_file_to_s3, to_s3_bucket, str(path), s3_key |
| 2185 | + ) |
| 2186 | + ) |
| 2187 | + for future in concurrent.futures.as_completed(results): |
| 2188 | + future.result() |
| 2189 | + logger.info( |
| 2190 | + "Exported to AWS %s/%s", to_s3_bucket.name, str(folder_path) |
2182 | 2191 | ) |
2183 | | - |
2184 | | - for future in concurrent.futures.as_completed(results): |
2185 | | - future.result() |
2186 | | - logger.info("Exported to AWS %s/%s", to_s3_bucket, str(path)) |
| 2192 | + else: |
| 2193 | + controller.download_export( |
| 2194 | + project_name=project_name, |
| 2195 | + export_name=export_name, |
| 2196 | + folder_path=folder_path, |
| 2197 | + extract_zip_contents=extract_zip_contents, |
| 2198 | + to_s3_bucket=to_s3_bucket, |
| 2199 | + ) |
2187 | 2200 |
|
2188 | 2201 |
|
2189 | 2202 | @Trackable |
|
0 commit comments