From cd509f8b2f57b4b4dafc13512c6803004b3050e4 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 14 Aug 2026 13:15:15 -0700 Subject: [PATCH] feat(workflow-operator): vectorize every named text column, not just one Text Attribute named one column, so a dataset whose text is spread over several could not be used as it stands. A title beside a body, or a description beside a comment, is ordinary, and the only way through was a Concat upstream, which merges the columns into one string and gives up which column each word came from. Naming several columns was not an alternative. CountVectorizer takes a flat sequence of documents, so a two-column frame reaches it as two documents rather than as the rows, and it learns a vocabulary of the column names. The field takes a list, and the generated pipeline opens with a ColumnTransformer that gives each named column its own CountVectorizer and concatenates the results, so a word keeps the column it came from through the feature's prefix. The steps are named by position rather than after the column, keeping a column whose name carries a double underscore away from the separator get_feature_names_out puts between step and feature. The transformer names the columns it reads, so the frame no longer has to be narrowed before the pipeline, and that line goes. Tfidf Transformer stays one switch over the whole matrix, which leaves the shape of the pipeline as it was. Turning Count Vectorizer on still discards the numeric columns, as before: carrying them along needs a way to say which columns are features, which these two families do not have. A workflow written before this holds a bare string in the field, which reads as a list of one, so nothing has to be migrated. Co-Authored-By: Claude Opus 5 (1M context) --- .../sklearn/SklearnClassifierOpDesc.scala | 8 ++-- .../operator/sklearn/SklearnModelOpDesc.scala | 40 +++++++++++++++++-- .../training/SklearnTrainingOpDesc.scala | 4 +- .../SklearnAdaptiveBoostingOpDescSpec.scala | 2 +- .../sklearn/SklearnBaggingOpDescSpec.scala | 2 +- ...SklearnBernoulliNaiveBayesOpDescSpec.scala | 2 +- .../SklearnClassifierOpDescCodegenSpec.scala | 34 ++++++++++++---- ...klearnComplementNaiveBayesOpDescSpec.scala | 2 +- .../SklearnDecisionTreeOpDescSpec.scala | 2 +- .../SklearnDummyClassifierOpDescSpec.scala | 2 +- .../sklearn/SklearnExtraTreeOpDescSpec.scala | 2 +- .../sklearn/SklearnExtraTreesOpDescSpec.scala | 2 +- .../SklearnGaussianNaiveBayesOpDescSpec.scala | 2 +- .../SklearnGradientBoostingOpDescSpec.scala | 2 +- .../sklearn/SklearnKNNOpDescSpec.scala | 2 +- .../sklearn/SklearnLinearSVMOpDescSpec.scala | 2 +- ...klearnLogisticRegressionCVOpDescSpec.scala | 2 +- .../SklearnLogisticRegressionOpDescSpec.scala | 2 +- ...klearnMultiLayerPerceptronOpDescSpec.scala | 2 +- ...learnMultinomialNaiveBayesOpDescSpec.scala | 2 +- .../SklearnNearestCentroidOpDescSpec.scala | 2 +- .../SklearnPassiveAggressiveOpDescSpec.scala | 2 +- .../sklearn/SklearnPerceptronOpDescSpec.scala | 2 +- ...earnProbabilityCalibrationOpDescSpec.scala | 2 +- .../SklearnRandomForestOpDescSpec.scala | 2 +- .../sklearn/SklearnRidgeCVOpDescSpec.scala | 2 +- .../sklearn/SklearnRidgeOpDescSpec.scala | 2 +- .../sklearn/SklearnSDGOpDescSpec.scala | 2 +- .../sklearn/SklearnSVMOpDescSpec.scala | 2 +- ...rnTrainingAdaptiveBoostingOpDescSpec.scala | 2 +- .../SklearnTrainingBaggingOpDescSpec.scala | 2 +- ...rainingBernoulliNaiveBayesOpDescSpec.scala | 2 +- ...ainingComplementNaiveBayesOpDescSpec.scala | 2 +- ...klearnTrainingDecisionTreeOpDescSpec.scala | 2 +- ...arnTrainingDummyClassifierOpDescSpec.scala | 2 +- .../SklearnTrainingExtraTreeOpDescSpec.scala | 2 +- .../SklearnTrainingExtraTreesOpDescSpec.scala | 2 +- ...TrainingGaussianNaiveBayesOpDescSpec.scala | 2 +- ...rnTrainingGradientBoostingOpDescSpec.scala | 2 +- .../SklearnTrainingKNNOpDescSpec.scala | 2 +- ...rnTrainingLinearRegressionOpDescSpec.scala | 2 +- .../SklearnTrainingLinearSVMOpDescSpec.scala | 2 +- ...ainingLogisticRegressionCVOpDescSpec.scala | 2 +- ...TrainingLogisticRegressionOpDescSpec.scala | 2 +- ...ainingMultiLayerPerceptronOpDescSpec.scala | 2 +- ...iningMultinomialNaiveBayesOpDescSpec.scala | 2 +- ...arnTrainingNearestCentroidOpDescSpec.scala | 2 +- .../SklearnTrainingOpDescCodegenSpec.scala | 34 ++++++++++++---- ...nTrainingPassiveAggressiveOpDescSpec.scala | 2 +- .../SklearnTrainingPerceptronOpDescSpec.scala | 2 +- ...ningProbabilityCalibrationOpDescSpec.scala | 2 +- ...klearnTrainingRandomForestOpDescSpec.scala | 2 +- .../SklearnTrainingRidgeCVOpDescSpec.scala | 2 +- .../SklearnTrainingRidgeOpDescSpec.scala | 2 +- .../SklearnTrainingSDGOpDescSpec.scala | 2 +- .../SklearnTrainingSVMOpDescSpec.scala | 2 +- 56 files changed, 146 insertions(+), 76 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala index 92aec692a6e..33c5325b0b9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala @@ -33,6 +33,7 @@ abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc { pyb"""$getImportStatements |from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score |from sklearn.pipeline import make_pipeline + |from sklearn.compose import ColumnTransformer |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer |import numpy as np |from pytexera import * @@ -41,10 +42,11 @@ abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc { | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | Y = table[$target] | X = table.drop($target, axis=1) - | X = ${if (countVectorizer) pyb"X[$text]" else "X"} | if port == 0: - | self.model = make_pipeline(${if (countVectorizer) "CountVectorizer()," - else ""} ${if (tfidfTransformer) "TfidfTransformer()," else ""} ${getImportStatements + | self.model = make_pipeline(${vectorizerStage(c => pyb"$c".toString)} ${if ( + tfidfTransformer + ) "TfidfTransformer()," + else ""} ${getImportStatements .split(" ") .last}()).fit(X, Y) | else: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala index 6665477aeac..f7506ffa820 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala @@ -19,7 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty, JsonPropertyDescription} +import com.fasterxml.jackson.annotation.{ + JsonFormat, + JsonIgnore, + JsonProperty, + JsonPropertyDescription +} import com.kjetland.jackson.jsonSchema.annotations.{ JsonSchemaInject, JsonSchemaInt, @@ -50,12 +55,15 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { var countVectorizer: Boolean = false @JsonSchemaTitle("Text Attribute") - @JsonPropertyDescription("Attribute in your dataset with text to vectorize.") + @JsonPropertyDescription("Attributes in your dataset with text to vectorize.") + // A workflow written before this field took several columns holds a bare string + // here, which reads as a list of one. + @JsonFormat(`with` = Array(JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY)) @JsonSchemaInject( strings = Array( new JsonSchemaString( path = CommonOpDescAnnotation.autofill, - value = CommonOpDescAnnotation.attributeName + value = CommonOpDescAnnotation.attributeNameList ), new JsonSchemaString(path = HideAnnotation.hideTarget, value = "countVectorizer"), new JsonSchemaString(path = HideAnnotation.hideType, value = HideAnnotation.Type.equals), @@ -65,7 +73,7 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { new JsonSchemaInt(path = CommonOpDescAnnotation.autofillAttributeOnPort, value = 0) ) ) - var text: EncodableString = _ + var text: List[EncodableString] = List() @JsonSchemaTitle("Tfidf Transformer") @JsonPropertyDescription("Transform a count matrix to a normalized tf or tf-idf representation.") @@ -79,6 +87,30 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { ) var tfidfTransformer: Boolean = false + /** The pipeline step that turns the named text columns into features, empty when + * Count Vectorizer is off. + * + * One `CountVectorizer` per column rather than one over all of them: + * `CountVectorizer` takes a flat sequence of documents, so handing it several + * columns reads them as one document each rather than as the rows. A + * `ColumnTransformer` gives each its own and concatenates the results, keeping + * the column a word came from distinguishable through the feature's prefix. + * + * The steps are named by position, not after the column, so that a column whose + * name carries a double underscore cannot collide with the separator + * `get_feature_names_out` puts between step and feature. + * + * `renderColumn` writes one column name in the caller's dialect: an encoded + * expression for the operator, a literal for the standalone script. + */ + @JsonIgnore + protected def vectorizerStage(renderColumn: EncodableString => String): String = + if (!countVectorizer) "" + else + text.zipWithIndex + .map { case (column, i) => s"""("text$i", CountVectorizer(), ${renderColumn(column)})""" } + .mkString("ColumnTransformer([", ", ", "]),") + @JsonIgnore def getImportStatements: String diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala index 3f354cc37f5..6d066a5e9b6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala @@ -33,6 +33,7 @@ class SklearnTrainingOpDesc extends SklearnModelOpDesc { override def generatePythonCode(): String = pyb"""$getImportStatements |from sklearn.pipeline import make_pipeline + |from sklearn.compose import ColumnTransformer |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer |import numpy as np |from pytexera import * @@ -41,8 +42,7 @@ class SklearnTrainingOpDesc extends SklearnModelOpDesc { | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | Y = table[$target] | X = table.drop($target, axis=1) - | X = ${if (countVectorizer) pyb"X[$text]" else "X"} - | model = make_pipeline(${if (countVectorizer) "CountVectorizer()," else ""} ${if ( + | model = make_pipeline(${vectorizerStage(c => pyb"$c".toString)} ${if ( tfidfTransformer ) "TfidfTransformer()," else ""} ${getImportStatements.split(" ").last}()).fit(X, Y) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala index 52375db8304..084fa014f3c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnAdaptiveBoostingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnAdaptiveBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala index 3a1759754c4..bc11cade128 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnBaggingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnBaggingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala index b34d9b2e1b9..0d5fa28fc49 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnBernoulliNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala index 9e732002eb9..e964fcca346 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala @@ -56,7 +56,7 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { ): SklearnKNNOpDesc = { val d = new SklearnKNNOpDesc d.target = "label" - d.text = "docs" + d.text = List("docs") d.countVectorizer = countVectorizer d.tfidfTransformer = tfidfTransformer d @@ -69,7 +69,7 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { code should include(s"Y = table[${decodeExpr("label")}]") code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") // Feature-column path: X is kept whole, the text attribute is never read. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include( "self.model = make_pipeline( KNeighborsClassifier()).fit(X, Y)" @@ -81,27 +81,45 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { val code = descriptor(countVectorizer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") - code should not include "X = X\n" + // ColumnTransformer selects the columns itself, so X stays the whole frame. normalized(code) should include( - "self.model = make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)" + s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), KNeighborsClassifier()).fit(X, Y)""" ) code should not include "TfidfTransformer()" } + // One CountVectorizer per column: it reads a flat sequence of documents, so + // several columns handed to one would be read as a document each. The steps are + // named by position, keeping a column named with a double underscore away from + // the separator get_feature_names_out uses. + it should "give each named column its own CountVectorizer" in { + val d = descriptor(countVectorizer = true) + d.text = List("title", "body") + normalized(d.generatePythonCode()) should include( + s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "title" + )}), ("text1", CountVectorizer(), ${decodeExpr( + "body" + )})]), KNeighborsClassifier()).fit(X, Y)""" + ) + } + it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { val code = descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") normalized(code) should include( - "self.model = make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)""" ) } it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { val code = descriptor(tfidfTransformer = true).generatePythonCode() // Without countVectorizer there is no text-column selection. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include( "self.model = make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala index 0d50eb45775..90b67bdb1ed 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnComplementNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnComplementNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala index 44fbf1aeed7..be316adf0f6 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnDecisionTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala index 24b391b6bf5..c670069ae79 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnDummyClassifierOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnDummyClassifierOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala index 96b9ec470b8..6c2f424b171 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnExtraTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnExtraTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala index 332bee1ee35..33de266f3c7 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnExtraTreesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnExtraTreesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala index 9c25894dc18..f4cd969c2e1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala index 111482e6057..67b9de06c80 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnGradientBoostingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnGradientBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala index 57dcc06be49..256a298af94 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnKNNOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnKNNOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala index d0153ae37fb..f09c1f7ed99 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnLinearSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnLinearSVMOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala index 24589453edc..6bdcb8b99aa 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnLogisticRegressionCVOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnLogisticRegressionCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala index 077ecb9ac43..6f6b0e10b58 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnLogisticRegressionOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnLogisticRegressionOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala index 592f5779568..85a54a28828 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnMultiLayerPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala index 3a6b4debfd2..6573e26cd93 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnMultinomialNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala index db61804a79b..ba173413d57 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnNearestCentroidOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnNearestCentroidOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala index d184eb43674..684b008cf7e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnPassiveAggressiveOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnPassiveAggressiveOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala index b28a57c5827..02b932792ce 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnPerceptronOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala index 2eab3102cb1..53a30df5b55 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnProbabilityCalibrationOpDescSpec extends AnyFlatSpec with Matchers d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnProbabilityCalibrationOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala index 2f19aff0ad3..63c0f536e5a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnRandomForestOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnRandomForestOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala index c3fe761c15a..05892469d56 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnRidgeCVOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnRidgeCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala index 1f24306bb6d..b13437a7d6e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnRidgeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnRidgeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala index 516374b0874..cc114667d12 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnSDGOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnSDGOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala index b1a7e9f3b0b..d88f9c71d85 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnSVMOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala index 5eeeae389d9..dc26343aa8e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingAdaptiveBoostingOpDescSpec extends AnyFlatSpec with Matcher d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingAdaptiveBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala index dc5916b4b2b..3ac4d1b612b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingBaggingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingBaggingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala index 7bfe9c0905b..14da4504dbf 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matc d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingBernoulliNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala index 1c30b439b26..b79c579beef 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingComplementNaiveBayesOpDescSpec extends AnyFlatSpec with Mat d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingComplementNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala index 33031a81d7a..f81a8c40952 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingDecisionTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala index 603ee1b5477..fe3e2bb73f4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingDummyClassifierOpDescSpec extends AnyFlatSpec with Matchers d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingDummyClassifierOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala index f43606ed5ab..509055f06b4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingExtraTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingExtraTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala index e404a05c0de..b0d9d1e4594 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingExtraTreesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingExtraTreesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala index 7d193883ccf..045ae4d979d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Match d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingGaussianNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala index dc17a8267a3..90ec0d15de0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingGradientBoostingOpDescSpec extends AnyFlatSpec with Matcher d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingGradientBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala index b67d26ccb22..383bdec47f8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingKNNOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingKNNOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala index 7bd4a2af522..31ae5860a15 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLinearRegressionOpDescSpec extends AnyFlatSpec with Matcher d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLinearRegressionOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala index 2e0cce8265d..2ae012556b9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLinearSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLinearSVMOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala index b0386a50111..15e11eaa2d8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLogisticRegressionCVOpDescSpec extends AnyFlatSpec with Mat d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLogisticRegressionCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala index e5b04f95a36..1e0b63e562c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLogisticRegressionOpDescSpec extends AnyFlatSpec with Match d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLogisticRegressionOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala index d23aca58b45..2e8faaba08b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with Mat d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingMultiLayerPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala index 6e1bda2cf12..3ca3c3706a1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with Ma d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingMultinomialNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala index 46afa31dd1b..a9f0d8efbe8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingNearestCentroidOpDescSpec extends AnyFlatSpec with Matchers d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingNearestCentroidOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala index 0c00ca271b5..114393d374d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala @@ -56,7 +56,7 @@ class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { ): SklearnTrainingKNNOpDesc = { val d = new SklearnTrainingKNNOpDesc d.target = "label" - d.text = "docs" + d.text = List("docs") d.countVectorizer = countVectorizer d.tfidfTransformer = tfidfTransformer d @@ -69,7 +69,7 @@ class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { code should include(s"Y = table[${decodeExpr("label")}]") code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") // Feature-column path: X is kept whole, the text attribute is never read. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include("make_pipeline( KNeighborsClassifier()).fit(X, Y)") code should not include "CountVectorizer()" @@ -78,27 +78,45 @@ class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { val code = descriptor(countVectorizer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") - code should not include "X = X\n" + // ColumnTransformer selects the columns itself, so X stays the whole frame. normalized(code) should include( - "make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)" + s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), KNeighborsClassifier()).fit(X, Y)""" ) code should not include "TfidfTransformer()" } + // One CountVectorizer per column: it reads a flat sequence of documents, so + // several columns handed to one would be read as a document each. The steps are + // named by position, keeping a column named with a double underscore away from + // the separator get_feature_names_out uses. + it should "give each named column its own CountVectorizer" in { + val d = descriptor(countVectorizer = true) + d.text = List("title", "body") + normalized(d.generatePythonCode()) should include( + s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "title" + )}), ("text1", CountVectorizer(), ${decodeExpr( + "body" + )})]), KNeighborsClassifier()).fit(X, Y)""" + ) + } + it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { val code = descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") normalized(code) should include( - "make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)""" ) } it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { val code = descriptor(tfidfTransformer = true).generatePythonCode() // Without countVectorizer there is no text-column selection. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include( "make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala index 519a85c4164..dbb0efacefc 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingPassiveAggressiveOpDescSpec extends AnyFlatSpec with Matche d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingPassiveAggressiveOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala index 2ce66d8fdf9..f3641f91743 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingPerceptronOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala index 21684da7c76..58e6a3c6428 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingProbabilityCalibrationOpDescSpec extends AnyFlatSpec with M d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingProbabilityCalibrationOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala index 045bbd2b5a7..e41ff84488c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingRandomForestOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingRandomForestOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala index 2649692c02d..8dfed45ee92 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingRidgeCVOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingRidgeCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala index 6e2923e4ce5..343463e9c0b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingRidgeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingRidgeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala index 26388758431..2c7b0707241 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingSDGOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingSDGOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala index 88a611af728..6b7476d7a3b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingSVMOpDesc.getOutputSchemas" should