#[222] No info to user about invalid regex

.circleci/config.yml

@@ -0,0 +1,134 @@
+# Taken from https://github.com/code-star/circleci-scala-sbt-git
+
+workflows:
+  version: 2
+  build_and_push_and_deploy:
+    jobs:
+      - build
+      - build_and_publish_docker_images:
+          requires:
+            - build
+          filters:
+            branches:
+              only:
+                - develop
+                - master
+      - staging-deploy-to-swarm:
+          requires:
+            - build_and_publish_docker_images
+          filters:
+            branches:
+              only: develop
+      - deploy-to-swarm:
+          requires:
+            - build_and_publish_docker_images
+          filters:
+            branches:
+              only: master
+
+version: 2
+jobs:
+  build:
+    working_directory: ~/codesearch
+    docker:
+      - image: codestar/circleci-scala-sbt-git:scala-2.12.6-sbt-1.1.6
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - codesearch-{{ checksum "project/Builder.scala" }}-{{ checksum "build.sbt" }}
+            - codesearch
+      - run:
+          command:
+            sbt compile
+      - run:
+          command:
+            sbt core/assembly
+      - run:
+          command:
+            sbt web-server/assembly
+      - run:
+          command: |
+            sbt scalafmt exit
+            git diff --exit-code
+      - save_cache:
+          key: codesearch-{{ checksum "project/Builder.scala" }}-{{ checksum "build.sbt" }}
+          paths:
+            - target/resolution-cache
+            - target/streams
+            - project/target/resolution-cache
+            - project/target/streams
+            - ~/.sbt
+            - ~/.iv2/cache
+            - ~/.m2
+      - save_cache:
+          # Changing this to a different key is the only way to remove old
+          # dependencies from the cache and/or generate a more up-to-date
+          # cache
+          key: codesearch
+          paths:
+            - ~/.sbt
+            - ~/.iv2/cache
+            - ~/.m2
+      - persist_to_workspace:
+          root: ~/codesearch
+          paths:
+            - .dockerignore
+            - docker
+            - scripts
+            - Makefile
+            - codesearch-core.jar
+            - codesearch-server.jar
+
+  build_and_publish_docker_images:
+    working_directory: ~/codesearch
+    machine:
+      docker_layer_caching: true
+    steps:
+      - attach_workspace:
+          at: ~/codesearch
+      - run: docker login quay.io -u "$DOCKER_USER" -p "$DOCKER_PASS"
+      - run:
+          name: Running build core image
+          command: make build-docker-core "branch=${CIRCLE_BRANCH}"
+      - run:
+          name: Running build web-server image
+          command: make build-docker-web-server "branch=${CIRCLE_BRANCH}"
+      - run:
+          name: Running push core image
+          command: make push-docker-core "branch=${CIRCLE_BRANCH}"
+      - run:
+          name: Running push web-server image
+          command: make push-docker-web-server "branch=${CIRCLE_BRANCH}"
+
+  staging-deploy-to-swarm:
+    working_directory: ~/codesearch
+    machine: true
+    steps:
+      - attach_workspace:
+          at: ~/codesearch
+      - add_ssh_keys:
+          fingerprints:
+            - "27:51:c3:38:40:b1:c2:db:ea:9d:bf:d6:c4:93:a2:e5"
+      - run: ssh deployer@$STAGING_HOST "mkdir -p ~/docker && echo $DEPLOYER_PASS | sudo -S mkdir -p /mnt/vol/{portainer/data,postgresql,data,index,logs}" || true
+      - run: scp ./docker/docker-stack-compose.yml deployer@$STAGING_HOST:~/docker
+      - run: ssh deployer@$STAGING_HOST "docker login quay.io -u $DOCKER_USER -p $DOCKER_PASS" || true
+      - run:
+          name: Deploy to remote swarm
+          command: ssh deployer@$STAGING_HOST "SECRET_KEY=$STAGING_SECRET_KEY SERVER_PORT=$SERVER_PORT docker stack deploy --with-registry-auth -c ~/docker/docker-stack-compose.yml codesearch" || true
+
+  deploy-to-swarm:
+    working_directory: ~/codesearch
+    machine: true
+    steps:
+      - attach_workspace:
+          at: ~/codesearch
+      - add_ssh_keys:
+          fingerprints:
+            - "27:51:c3:38:40:b1:c2:db:ea:9d:bf:d6:c4:93:a2:e5"
+      - run: ssh deployer@$HOST "mkdir -p ~/docker && echo $DEPLOYER_PASS | sudo -S mkdir -p /mnt/vol/{portainer/data,postgresql,data,index,logs}" || true
+      - run: scp ./docker/docker-stack-compose.yml deployer@$HOST:~/docker
+      - run: ssh deployer@$HOST "docker login quay.io -u $DOCKER_USER -p $DOCKER_PASS" || true
+      - run:
+          name: Deploy to remote swarm
+          command: ssh deployer@$HOST "SECRET_KEY=$SECRET_KEY SERVER_PORT=$SERVER_PORT docker stack deploy --with-registry-auth -c ~/docker/docker-stack-compose.yml codesearch" || true

README.md

@@ -1,6 +1,4 @@
-# Aelve Codesearch
-
-![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/aelve/codesearch/develop.svg)
+# codesearch
 
 ## Deployment instructions
 

core/src/main/resources/application.conf

@@ -11,6 +11,10 @@ db {
   port = ${?DATABASE_PORT_NUMBER}
 
   dataSourceClass = "slick.jdbc.DatabaseUrlDataSource"
+  properties = {
+    driver = "org.postgresql.Driver"
+    url = "postgres://"${db.user}":"${db.password}"@"${db.host}":"${db.port}"/"${db.name}
+  }
 }
 
 languagesConfig {

core/src/main/scala/codesearch/core/Main.scala

@@ -1,18 +1,15 @@
 package codesearch.core
 
-import java.nio.file.Paths
-
 import cats.effect.{ExitCode, IO, IOApp, Resource}
 import cats.syntax.flatMap._
 import codesearch.core.config.Config
+import codesearch.core.db._
 import codesearch.core.index._
-import codesearch.core.index.directory._
 import codesearch.core.index.repository.Downloader
 import codesearch.core.meta._
 import codesearch.core.model._
 import codesearch.core.util.Unarchiver
 import com.softwaremill.sttp.asynchttpclient.fs2.AsyncHttpClientFs2Backend
-import slick.jdbc.PostgresProfile.api._
 
 object Main extends IOApp {
 
@@ -26,6 +23,7 @@ object Main extends IOApp {
   )
 
   case class LangRep[A <: DefaultTable](
+      db: DefaultDB[A],
       langIndex: LanguageIndex[A],
       metaDownloader: MetaDownloader[IO]
   )
@@ -44,20 +42,11 @@ object Main extends IOApp {
         gemMeta     <- GemMetaDownloader(config.languagesConfig.ruby, downloader)
         npmMeta     <- NpmMetaDownloader(config.languagesConfig.javascript, downloader)
 
-        db = Database.forConfig("db")
-
-        cindexPath = Paths.get("./index/cindex/")
-
-        haskellCindex    = HaskellCindex(cindexPath)
-        rustCindex       = RustCindex(cindexPath)
-        rubyCindex       = RubyCindex(cindexPath)
-        javaScriptCindex = JavaScriptCindex(cindexPath)
-
         langReps = Map(
-          "haskell"    -> LangRep[HackageTable](HaskellIndex(config, db, haskellCindex), hackageMeta),
-          "rust"       -> LangRep[CratesTable](RustIndex(config, db, rustCindex), cratesMeta),
-          "ruby"       -> LangRep[GemTable](RubyIndex(config, db, rubyCindex), gemMeta),
-          "javascript" -> LangRep[NpmTable](JavaScriptIndex(config, db, javaScriptCindex), npmMeta)
+          "haskell"    -> LangRep[HackageTable](HackageDB, HaskellIndex(config), hackageMeta),
+          "rust"       -> LangRep[CratesTable](CratesDB, RustIndex(config), cratesMeta),
+          "ruby"       -> LangRep[GemTable](GemDB, RubyIndex(config), gemMeta),
+          "javascript" -> LangRep[NpmTable](NpmDB, JavaScriptIndex(config), npmMeta)
         )
         exitCode <- Program(langReps) >>= (_.run(params))
       } yield exitCode

core/src/main/scala/codesearch/core/Program.scala

@@ -43,7 +43,7 @@ class Program(langReps: Map[String, LangRep[_ <: DefaultTable]], logger: Logger[
   def initDb(params: Params): IO[Unit] =
     for {
       languages <- findRepositories(params.lang)
-      _         <- languages.traverse_(_.langIndex.initDB)
+      _         <- languages.traverse_(_.db.initDB)
     } yield ()
 
   def downloadMeta(params: Params): IO[Unit] = {

core/src/main/scala/codesearch/core/config/Config.scala

@@ -16,7 +16,6 @@ case class Config(
 
 case class DatabaseConfig(
     dataSourceClass: String,
-    host: String,
     port: Int,
     name: String,
     user: String,

core/src/main/scala/codesearch/core/db/DefaultDB.scala

@@ -12,19 +12,23 @@ import slick.lifted.TableQuery
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 
-trait DefaultDB[T <: DefaultTable] {
+object DefaultDB {
+  lazy val db = Database.forConfig("db")
+}
 
-  def db: Database
+trait DefaultDB[T <: DefaultTable] {
 
   val table: TableQuery[T]
+  lazy val db = DefaultDB.db
 
   def insertOrUpdate[A <: SourcePackage](pack: A): IO[Int] = {
     val insOrUpdate = table.insertOrUpdate(
       (
         pack.name,
         pack.version,
         new Timestamp(System.currentTimeMillis())
-      ))
+      )
+    )
     IO.fromFuture(IO(db.run(insOrUpdate)))
   }
 
@@ -90,3 +94,8 @@ trait NpmDB extends DefaultDB[NpmTable] {
 trait GemDB extends DefaultDB[GemTable] {
   val table = TableQuery[GemTable]
 }
+
+object HackageDB extends HackageDB
+object CratesDB  extends CratesDB
+object NpmDB     extends NpmDB
+object GemDB     extends GemDB

core/src/main/scala/codesearch/core/index/HaskellIndex.scala

@@ -12,16 +12,18 @@ import codesearch.core.db.HackageDB
 import codesearch.core.index.repository.{HackagePackage, SourcesDownloader}
 import codesearch.core.index.directory.Directory._
 import codesearch.core.index.directory.Directory.ops._
-import codesearch.core.index.directory._
+import codesearch.core.index.directory.СindexDirectory
+import codesearch.core.index.directory.СindexDirectory.HaskellCindex
 import codesearch.core.model.{HackageTable, Version}
 import fs2.{Chunk, Stream}
-import slick.jdbc.PostgresProfile.api._
 
-class HaskellIndex(haskellConfig: HaskellConfig, val db: Database, val cindexDir: СindexDirectory)(
+class HaskellIndex(haskellConfig: HaskellConfig)(
     implicit val shift: ContextShift[IO],
     sourcesDownloader: SourcesDownloader[IO, HackagePackage]
 ) extends LanguageIndex[HackageTable] with HackageDB {
 
+  override protected val cindexDir: СindexDirectory = HaskellCindex
+
   override protected def concurrentTasksCount: Int = haskellConfig.concurrentTasksCount
 
   override protected def updateSources(name: String, version: String): IO[Int] = {
@@ -49,8 +51,8 @@ class HaskellIndex(haskellConfig: HaskellConfig, val db: Database, val cindexDir
 }
 
 object HaskellIndex {
-  def apply(config: Config, db: Database, cindexDir: СindexDirectory)(
+  def apply(config: Config)(
       implicit shift: ContextShift[IO],
       sourcesDownloader: SourcesDownloader[IO, HackagePackage]
-  ) = new HaskellIndex(config.languagesConfig.haskell, db, cindexDir)
+  ) = new HaskellIndex(config.languagesConfig.haskell)
 }

core/src/main/scala/codesearch/core/index/JavaScriptIndex.scala

@@ -11,15 +11,17 @@ import codesearch.core.index.repository.{NpmPackage, SourcesDownloader}
 import codesearch.core.index.directory.Directory._
 import codesearch.core.index.directory.Directory.ops._
 import codesearch.core.index.directory.СindexDirectory
+import codesearch.core.index.directory.СindexDirectory.JavaScriptCindex
 import codesearch.core.model.NpmTable
 import fs2.Stream
-import slick.jdbc.PostgresProfile.api._
 
-class JavaScriptIndex(config: JavaScriptConfig, val db: Database, val cindexDir: СindexDirectory)(
+class JavaScriptIndex(config: JavaScriptConfig)(
     implicit val shift: ContextShift[IO],
     sourcesDownloader: SourcesDownloader[IO, NpmPackage]
 ) extends LanguageIndex[NpmTable] with NpmDB {
 
+  override protected val cindexDir: СindexDirectory = JavaScriptCindex
+
   override protected def concurrentTasksCount: Int = config.concurrentTasksCount
 
   override protected def updateSources(name: String, version: String): IO[Int] = {
@@ -33,8 +35,8 @@ class JavaScriptIndex(config: JavaScriptConfig, val db: Database, val cindexDir:
 }
 
 object JavaScriptIndex {
-  def apply(config: Config, db: Database, cindexDir: СindexDirectory)(
+  def apply(config: Config)(
       implicit shift: ContextShift[IO],
       sourcesDownloader: SourcesDownloader[IO, NpmPackage]
-  ) = new JavaScriptIndex(config.languagesConfig.javascript, db, cindexDir)
+  ) = new JavaScriptIndex(config.languagesConfig.javascript)
 }