add support for JOIN

Author: fupelaqu

build.sbt

@@ -19,7 +19,7 @@ ThisBuild / organization := "app.softnetwork"
 
 name := "softclient4es"
 
-ThisBuild / version := "0.9.0"
+ThisBuild / version := "0.9.1"
 
 ThisBuild / scalaVersion := scala213
 

documentation/keywords.md

@@ -7,6 +7,7 @@ A list of reserved words recognized by the parser for this engine.
 ## Main clauses
 SELECT  
 FROM  
+JOIN  
 UNNEST  
 WHERE  
 GROUP BY  

documentation/request_structure.md

@@ -45,7 +45,7 @@ Expand an array / nested field into rows. Mapped to Elasticsearch `nested` and i
 ```sql
 SELECT id, phone
 FROM customers
-UNNEST(phones) AS phone;
+JOIN UNNEST(phones) AS phone;
 ```
 
 ---

es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala

@@ -102,7 +102,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(inner_emails.value) as email from index i, unnest(emails) as inner_emails where i.nom = \"Nom\""
+        "select count(inner_emails.value) as email from index i join unnest(emails) as inner_emails where i.nom = \"Nom\""
       )
     results.size shouldBe 1
     val result = results.head
@@ -147,7 +147,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with nested criteria" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))"
+        "select count(inner_emails.value) as count_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))"
       )
     results.size shouldBe 1
     val result = results.head
@@ -206,7 +206,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with filter" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\""
+        "select count(inner_emails.value) as count_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\""
       )
     results.size shouldBe 1
     val result = results.head
@@ -276,7 +276,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with \"and not\" operator" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(distinct inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))"
+        "select count(distinct inner_emails.value) as count_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))"
       )
     results.size shouldBe 1
     val result = results.head
@@ -352,7 +352,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with date filtering" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(distinct inner_emails.value) as count_distinct_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\""
+        "select count(distinct inner_emails.value) as count_distinct_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\""
       )
     results.size shouldBe 1
     val result = results.head
@@ -428,7 +428,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
         |profile_ccm.lastName as lastName,
         |profile_ccm.postalCode as postalCode,
         |profile_ccm.birthYear as birthYear
-        |FROM index, unnest(profiles) as profile_ccm
+        |FROM index join unnest(profiles) as profile_ccm
         |WHERE
         |((profile_ccm.postalCode BETWEEN "10" AND "99999")
         |AND
@@ -621,8 +621,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
            |  min(inner_products.price) as min_price,
            |  max(inner_products.price) as max_price
            |FROM
-           |  stores store,
-           |  UNNEST(store.products LIMIT 10) as inner_products
+           |  stores store
+           |  JOIN UNNEST(store.products LIMIT 10) as inner_products
            |WHERE
            |  (
            |    firstName is not null AND

sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala

@@ -102,7 +102,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(inner_emails.value) as email from index i, unnest(emails) as inner_emails where i.nom = \"Nom\""
+        "select count(inner_emails.value) as email from index i join unnest(emails) as inner_emails where i.nom = \"Nom\""
       )
     results.size shouldBe 1
     val result = results.head
@@ -147,7 +147,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with nested criteria" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))"
+        "select count(inner_emails.value) as count_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))"
       )
     results.size shouldBe 1
     val result = results.head
@@ -206,7 +206,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with filter" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\""
+        "select count(inner_emails.value) as count_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\""
       )
     results.size shouldBe 1
     val result = results.head
@@ -276,7 +276,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with \"and not\" operator" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(distinct inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))"
+        "select count(distinct inner_emails.value) as count_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))"
       )
     results.size shouldBe 1
     val result = results.head
@@ -352,7 +352,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
   it should "perform nested count with date filtering" in {
     val results: Seq[ElasticAggregation] =
       SQLQuery(
-        "select count(distinct inner_emails.value) as count_distinct_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\""
+        "select count(distinct inner_emails.value) as count_distinct_emails from index join unnest(emails) as inner_emails join unnest(profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\""
       )
     results.size shouldBe 1
     val result = results.head
@@ -428,7 +428,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
         |profile_ccm.lastName as lastName,
         |profile_ccm.postalCode as postalCode,
         |profile_ccm.birthYear as birthYear
-        |FROM index, unnest(profiles) as profile_ccm
+        |FROM index join unnest(profiles) as profile_ccm
         |WHERE
         |((profile_ccm.postalCode BETWEEN "10" AND "99999")
         |AND
@@ -621,8 +621,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers {
            |  min(inner_products.price) as min_price,
            |  max(inner_products.price) as max_price
            |FROM
-           |  stores store,
-           |  UNNEST(store.products LIMIT 10) as inner_products
+           |  stores store
+           |  JOIN UNNEST(store.products LIMIT 10) as inner_products
            |WHERE
            |  (
            |    firstName is not null AND

sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala

@@ -1,20 +1,51 @@
 package app.softnetwork.elastic.sql.parser
 
-import app.softnetwork.elastic.sql.query.{From, Table, Unnest}
+import app.softnetwork.elastic.sql.query.{
+  CrossJoin,
+  From,
+  FullJoin,
+  InnerJoin,
+  Join,
+  JoinType,
+  LeftJoin,
+  On,
+  RightJoin,
+  Table,
+  Unnest
+}
 
 trait FromParser {
-  self: Parser with LimitParser =>
+  self: Parser with WhereParser with LimitParser =>
 
-  def unnest: PackratParser[Table] =
-    Unnest.regex ~ start ~ identifier ~ limit.? ~ end ~ alias ^^ { case _ ~ _ ~ i ~ l ~ _ ~ a =>
-      Table(Unnest(i, l), Some(a))
+  def unnest: PackratParser[Unnest] =
+    Unnest.regex ~ start ~ identifier ~ limit.? ~ end ~ alias.? ^^ { case _ ~ i ~ l ~ _ ~ a =>
+      Unnest(i, l, a)
     }
 
-  def table: PackratParser[Table] = identifier ~ alias.? ^^ { case i ~ a => Table(i, a) }
+  def inner_join: PackratParser[JoinType] = InnerJoin.regex ^^ { _ => InnerJoin }
+  def left_join: PackratParser[JoinType] = LeftJoin.regex ^^ { _ => LeftJoin }
+  def right_join: PackratParser[JoinType] = RightJoin.regex ^^ { _ => RightJoin }
+  def full_join: PackratParser[JoinType] = FullJoin.regex ^^ { _ => FullJoin }
+  def cross_join: PackratParser[JoinType] = CrossJoin.regex ^^ { _ => CrossJoin }
+  def join_type: PackratParser[JoinType] =
+    inner_join | left_join | right_join | full_join | cross_join
+
+  def on: PackratParser[On] = On.regex ~> whereCriteria ^^ { rawTokens =>
+    On(
+      processTokens(rawTokens).getOrElse(throw new Exception("ON clause requires criteria"))
+    )
+  }
+
+  def join: PackratParser[Join] = opt(join_type) ~ Join.regex ~ unnest ~ opt(on) ^^ {
+    case jt ~ _ ~ t ~ o => t // Unnest cannot have a join type or an ON clause
+  }
+
+  def table: PackratParser[Table] = identifierRegex ~ alias.? ~ rep(join) ^^ { case i ~ a ~ js =>
+    Table(i, a, js)
+  }
 
-  def from: PackratParser[From] = From.regex ~ rep1sep(unnest | table, separator) ^^ {
-    case _ ~ tables =>
-      From(tables)
+  def from: PackratParser[From] = From.regex ~ rep1sep(table, separator) ^^ { case _ ~ tables =>
+    From(tables)
   }
 
 }

sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala

@@ -111,11 +111,13 @@ trait Parser
   private val reservedKeywords = Seq(
     "select",
     "from",
+    "join",
     "where",
     "group",
     "having",
     "order",
     "limit",
+    "offset",
     "as",
     "by",
     "except",
@@ -229,7 +231,7 @@ trait Parser
   private val identifierRegexStr =
     s"""(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[\\*a-zA-Z_\\-][a-zA-Z0-9_\\-.\\[\\]\\*]*"""
 
-  private val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex
+  val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex
 
   def identifier: PackratParser[Identifier] =
     (Distinct.regex.? ~ identifierRegex ^^ { case d ~ i =>

sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala

@@ -12,29 +12,74 @@ import app.softnetwork.elastic.sql.{
 
 case object From extends Expr("FROM") with TokenRegex
 
+sealed trait JoinType extends TokenRegex
+
+case object InnerJoin extends Expr("INNER") with JoinType
+
+case object LeftJoin extends Expr("LEFT") with JoinType
+
+case object RightJoin extends Expr("RIGHT") with JoinType
+
+case object FullJoin extends Expr("FULL") with JoinType
+
+case object CrossJoin extends Expr("CROSS") with JoinType
+
+case object On extends Expr("ON") with TokenRegex
+
+case class On(criteria: Criteria) extends Updateable {
+  override def sql: String = s" $On $criteria"
+  def update(request: SQLSearchRequest): On = this.copy(criteria = criteria.update(request))
+}
+
+case object Join extends Expr("JOIN") with TokenRegex
+
+sealed trait Join extends Updateable {
+  def source: Source
+  def joinType: Option[JoinType]
+  def on: Option[On]
+  def alias: Option[Alias]
+  override def sql: String =
+    s"${asString(joinType)} $Join $source${asString(on)}"
+
+  override def update(request: SQLSearchRequest): Join
+}
+
 case object Unnest extends Expr("UNNEST") with TokenRegex
 
-case class Unnest(identifier: Identifier, limit: Option[Limit]) extends Source {
-  override def sql: String = s"$Unnest($identifier${asString(limit)})"
+case class Unnest(identifier: Identifier, limit: Option[Limit], alias: Option[Alias] = None)
+    extends Source
+    with Join {
+  override def sql: String = s"$Join $Unnest($identifier${asString(limit)})"
   def update(request: SQLSearchRequest): Unnest =
     this.copy(identifier = identifier.update(request))
   override val name: String = identifier.name
+
+  override def source: Source = this
+
+  override def joinType: Option[JoinType] = None
+
+  override def on: Option[On] = None
 }
 
-case class Table(source: Source, tableAlias: Option[Alias] = None) extends Updateable {
-  override def sql: String = s"$source${asString(tableAlias)}"
-  def update(request: SQLSearchRequest): Table = this.copy(source = source.update(request))
+case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join] = Nil)
+    extends Source {
+  override def sql: String = s"$name${asString(tableAlias)}${joins.map(_.sql).mkString(" ")}"
+  def update(request: SQLSearchRequest): Table = this.copy(joins = joins.map(_.update(request)))
 }
 
 case class From(tables: Seq[Table]) extends Updateable {
   override def sql: String = s" $From ${tables.map(_.sql).mkString(",")}"
   lazy val tableAliases: Map[String, String] = tables
-    .flatMap((table: Table) => table.tableAlias.map(alias => table.source.name -> alias.alias))
-    .toMap
-  lazy val unnests: Seq[(String, String, Option[Limit])] = tables.collect {
-    case Table(u: Unnest, a) =>
-      (a.map(_.alias).getOrElse(u.identifier.name), u.identifier.name, u.limit)
-  }
+    .flatMap((table: Table) => table.tableAlias.map(alias => table.name -> alias.alias))
+    .toMap ++ unnests.map(unnest => unnest._2 -> unnest._1).toMap
+  lazy val unnests: Seq[(String, String, Option[Limit])] = tables
+    .map(_.joins)
+    .collect { case j =>
+      j.collect { case u: Unnest => // extract unnest info
+        (u.alias.map(_.alias).getOrElse(u.identifier.name), u.identifier.name, u.limit)
+      }
+    }
+    .flatten
   def update(request: SQLSearchRequest): From =
     this.copy(tables = tables.map(_.update(request)))
 

sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala

@@ -57,9 +57,7 @@ case class SQLSearchRequest(
 
   lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil)
 
-  lazy val sources: Seq[String] = from.tables.collect { case Table(source: Identifier, _) =>
-    source.sql
-  }
+  lazy val sources: Seq[String] = from.tables.map(_.name)
 
   lazy val topHitsBuckets: Seq[Bucket] = topHitsAggs
     .flatMap(_.bucketNames)