Skip to content

Commit ed205ca

Browse files
adamantiketomasfarias
authored andcommitted
fix: Avoid heavy imports in operator constructor
After a previous commit [1] improving import time on operator import, this commit fixes the import time issues when instantiating any of the dbt operators. The main issue still present was the enum imports in the operator constructors, which make DAGs slow to import any time they instantiated one of them. Profiling can be easily run locally, with the following command: ```shell python -X importtime -c "from airflow_dbt_python.operators.dbt import DbtRunOperator; DbtRunOperator(task_id='test')" 2>import-times.log ``` [1] 67c8d78
1 parent 3ca0fa2 commit ed205ca

File tree

4 files changed

+39
-37
lines changed

4 files changed

+39
-37
lines changed

airflow_dbt_python/hooks/dbt.py

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import os
77
import pickle
88
from dataclasses import dataclass
9-
from enum import Enum
109
from pathlib import Path
1110
from typing import Any, Optional, Type, Union
1211
from urllib.parse import urlparse
@@ -50,39 +49,9 @@
5049
except ImportError:
5150
from airflow.hooks.base_hook import BaseHook # type: ignore
5251

53-
from .backends import DbtBackend, StrPath, build_backend
54-
55-
56-
class FromStrEnum(Enum):
57-
"""Access enum variants with strings ensuring uppercase."""
58-
59-
@classmethod
60-
def from_str(cls, s: str):
61-
"""Instantiate an Enum from a string."""
62-
return cls[s.replace("-", "_").upper()]
63-
64-
65-
class LogFormat(FromStrEnum):
66-
"""Allowed dbt log formats."""
52+
from airflow_dbt_python.utils.enums import FromStrEnum, LogFormat, Output
6753

68-
DEFAULT = "default"
69-
JSON = "json"
70-
TEXT = "text"
71-
72-
73-
class Output(FromStrEnum):
74-
"""Allowed output arguments."""
75-
76-
JSON = "json"
77-
NAME = "name"
78-
PATH = "path"
79-
SELECTOR = "selector"
80-
81-
def __eq__(self, other):
82-
"""Override equality for string comparison."""
83-
if isinstance(other, str):
84-
return other.upper() == self.name
85-
return Enum.__eq__(self, other)
54+
from .backends import DbtBackend, StrPath, build_backend
8655

8756

8857
def parse_yaml_args(args: Optional[Union[str, dict[str, Any]]]) -> dict[str, Any]:

airflow_dbt_python/operators/dbt.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
from airflow.models.xcom import XCOM_RETURN_KEY
1616
from airflow.version import version
1717

18+
from airflow_dbt_python.utils.enums import LogFormat, Output
19+
1820
# apply_defaults is deprecated in version 2 and beyond. This allows us to
1921
# support version 1 and deal with the deprecation warning.
2022
if int(version[0]) == 1:
@@ -110,8 +112,6 @@ def __init__(
110112
replace_on_push: bool = False,
111113
**kwargs,
112114
) -> None:
113-
from airflow_dbt_python.hooks.dbt import LogFormat
114-
115115
super().__init__(**kwargs)
116116
self.project_dir = project_dir
117117
self.profiles_dir = profiles_dir
@@ -618,8 +618,6 @@ def __init__(
618618
indirect_selection: Optional[str] = None,
619619
**kwargs,
620620
) -> None:
621-
from airflow_dbt_python.hooks.dbt import Output
622-
623621
super().__init__(**kwargs)
624622
self.resource_types = resource_types
625623
self.select = select
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Utils for project operators and hooks."""

airflow_dbt_python/utils/enums.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
"""Enumerations with allowed set of values."""
2+
from enum import Enum
3+
4+
5+
class FromStrEnum(Enum):
6+
"""Access enum variants with strings ensuring uppercase."""
7+
8+
@classmethod
9+
def from_str(cls, s: str):
10+
"""Instantiate an Enum from a string."""
11+
return cls[s.replace("-", "_").upper()]
12+
13+
14+
class LogFormat(FromStrEnum):
15+
"""Allowed dbt log formats."""
16+
17+
DEFAULT = "default"
18+
JSON = "json"
19+
TEXT = "text"
20+
21+
22+
class Output(FromStrEnum):
23+
"""Allowed output arguments."""
24+
25+
JSON = "json"
26+
NAME = "name"
27+
PATH = "path"
28+
SELECTOR = "selector"
29+
30+
def __eq__(self, other):
31+
"""Override equality for string comparison."""
32+
if isinstance(other, str):
33+
return other.upper() == self.name
34+
return Enum.__eq__(self, other)

0 commit comments

Comments
 (0)