Merge branch 'main' into lint-scope-schema-resolution

Author: fresioAS

sqlmesh/core/engine_adapter/base.py

@@ -178,6 +178,7 @@ def with_settings(self, **kwargs: t.Any) -> EngineAdapter:
             "query_execution_tracker": kwargs.pop(
                 "query_execution_tracker", self._query_execution_tracker
             ),
+            "pre_ping": kwargs.pop("pre_ping", self._pre_ping),
             **self._extra_config,
             **kwargs,
         }

sqlmesh/core/engine_adapter/clickhouse.py

@@ -850,7 +850,7 @@ def _build_table_properties_exp(
             primary_key_vals = []
             if isinstance(primary_key, (exp.Tuple, exp.Array)):
                 primary_key_vals = primary_key.expressions
-            if isinstance(ordered_by_raw, exp.Paren):
+            if isinstance(primary_key, exp.Paren):
                 primary_key_vals = [primary_key.this]
 
             if not primary_key_vals:

sqlmesh/core/schema_loader.py

@@ -45,7 +45,7 @@ def create_external_models_file(
     external_model_fqns = set()
 
     for fqn, model in models.items():
-        if model.kind.is_external:
+        if model.kind.is_external and model._path == path:
             external_model_fqns.add(fqn)
         for dep in model.depends_on:
             if dep not in known_models:

tests/core/engine_adapter/test_base.py

@@ -3522,6 +3522,15 @@ def test_pre_ping(mocker: MockerFixture, make_mocked_engine_adapter: t.Callable)
     adapter._connection_pool.get().close.assert_called_once()
 
 
+def test_with_settings_preserves_pre_ping(make_mocked_engine_adapter: t.Callable):
+    adapter = make_mocked_engine_adapter(EngineAdapter, pre_ping=True)
+    assert adapter.with_settings()._pre_ping is True
+    assert adapter.with_settings(pre_ping=False)._pre_ping is False
+
+    adapter = make_mocked_engine_adapter(EngineAdapter)
+    assert adapter.with_settings()._pre_ping is False
+
+
 @pytest.mark.parametrize(
     "partitioned_by",
     [

tests/core/engine_adapter/test_clickhouse.py

@@ -305,6 +305,25 @@ def build_properties_sql(storage_format="", order_by="", primary_key="", propert
         == "ENGINE=MergeTree ORDER BY (a) PRIMARY KEY (a)"
     )
 
+    # A parenthesized single-column PRIMARY_KEY must be unwrapped regardless of the
+    # ORDER_BY shape. Previously the PRIMARY_KEY branch checked ORDER_BY's expression,
+    # so pairing PRIMARY_KEY = (a) with a non-parenthesized ORDER_BY emitted the
+    # malformed "PRIMARY KEY ((a))".
+    assert (
+        build_properties_sql(order_by="ORDER_BY = a,", primary_key="PRIMARY_KEY = (a)")
+        == "ENGINE=MergeTree ORDER BY (a) PRIMARY KEY (a)"
+    )
+
+    assert (
+        build_properties_sql(order_by="ORDER_BY = (a, b),", primary_key="PRIMARY_KEY = (a)")
+        == "ENGINE=MergeTree ORDER BY (a, b) PRIMARY KEY (a)"
+    )
+
+    assert (
+        build_properties_sql(primary_key="PRIMARY_KEY = (a)")
+        == "ENGINE=MergeTree ORDER BY () PRIMARY KEY (a)"
+    )
+
     assert build_properties_sql(order_by="ORDER_BY = a + 1,") == "ENGINE=MergeTree ORDER BY (a + 1)"
 
     assert (

tests/core/test_schema_loader.py

@@ -305,6 +305,53 @@ def _load_external_models():
     assert len(_load_external_models()) == 1
 
 
+def test_create_external_models_skips_models_from_external_models_directory(tmp_path: Path):
+    config = Config(gateways={"": GatewayConfig(connection=DuckDBConnectionConfig())})
+
+    model_dir = tmp_path / c.MODELS
+    model_dir.mkdir()
+
+    with open(model_dir / "table.sql", "w", encoding="utf8") as fd:
+        fd.write(
+            """
+        MODEL (
+            name lake.table,
+            kind FULL,
+        );
+
+        SELECT * FROM landing.source_table
+        """,
+        )
+
+    external_models_dir = tmp_path / c.EXTERNAL_MODELS
+    external_models_dir.mkdir()
+
+    with open(external_models_dir / "source_table.yaml", "w", encoding="utf8") as fd:
+        yaml.dump(
+            [
+                {
+                    "name": "landing.source_table",
+                    "columns": {"a": "int"},
+                }
+            ],
+            fd,
+        )
+
+    ctx = Context(paths=[tmp_path], config=config)
+    ctx.engine_adapter.execute("create schema landing")
+    ctx.engine_adapter.execute("create table landing.source_table as select 1 as a")
+    ctx.engine_adapter.execute("create schema lake")
+
+    ctx.create_external_models()
+
+    assert yaml.load(tmp_path / c.EXTERNAL_MODELS_YAML) == []
+
+    ctx.load()
+    external_models = [model for model in ctx.models.values() if isinstance(model, ExternalModel)]
+    assert len(external_models) == 1
+    assert external_models[0].fqn == '"memory"."landing"."source_table"'
+
+
 def test_no_internal_model_conversion(tmp_path: Path, mocker: MockerFixture):
     engine_adapter_mock = mocker.Mock()
     engine_adapter_mock.columns.return_value = {

tests/integrations/jupyter/test_magics.py

@@ -807,10 +807,7 @@ def test_create_external_models(notebook, loaded_sushi_context, convert_all_html
 
     assert not output.stdout
     assert not output.stderr
-    assert len(output.outputs) == 2
-    converted = sorted(convert_all_html_output_to_text(output))
-    assert 'Unable to get schema for \'"memory"."raw"."model1"\'' in converted[0]
-    assert 'Unable to get schema for \'"memory"."raw"."model2"\'' in converted[1]
+    assert len(output.outputs) == 0
 
     assert external_model_file.exists()
     assert (