diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..bb4ae4e --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,134 @@ +# Taken from https://github.com/code-star/circleci-scala-sbt-git + +workflows: + version: 2 + build_and_push_and_deploy: + jobs: + - build + - build_and_publish_docker_images: + requires: + - build + filters: + branches: + only: + - develop + - master + - staging-deploy-to-swarm: + requires: + - build_and_publish_docker_images + filters: + branches: + only: develop + - deploy-to-swarm: + requires: + - build_and_publish_docker_images + filters: + branches: + only: master + +version: 2 +jobs: + build: + working_directory: ~/codesearch + docker: + - image: codestar/circleci-scala-sbt-git:scala-2.12.6-sbt-1.1.6 + steps: + - checkout + - restore_cache: + keys: + - codesearch-{{ checksum "project/Builder.scala" }}-{{ checksum "build.sbt" }} + - codesearch + - run: + command: + sbt compile + - run: + command: + sbt core/assembly + - run: + command: + sbt web-server/assembly + - run: + command: | + sbt scalafmt exit + git diff --exit-code + - save_cache: + key: codesearch-{{ checksum "project/Builder.scala" }}-{{ checksum "build.sbt" }} + paths: + - target/resolution-cache + - target/streams + - project/target/resolution-cache + - project/target/streams + - ~/.sbt + - ~/.iv2/cache + - ~/.m2 + - save_cache: + # Changing this to a different key is the only way to remove old + # dependencies from the cache and/or generate a more up-to-date + # cache + key: codesearch + paths: + - ~/.sbt + - ~/.iv2/cache + - ~/.m2 + - persist_to_workspace: + root: ~/codesearch + paths: + - .dockerignore + - docker + - scripts + - Makefile + - codesearch-core.jar + - codesearch-server.jar + + build_and_publish_docker_images: + working_directory: ~/codesearch + machine: + docker_layer_caching: true + steps: + - attach_workspace: + at: ~/codesearch + - run: docker login quay.io -u "$DOCKER_USER" -p "$DOCKER_PASS" + - run: + name: Running build core image + command: make build-docker-core "branch=${CIRCLE_BRANCH}" + - run: + name: Running build web-server image + command: make build-docker-web-server "branch=${CIRCLE_BRANCH}" + - run: + name: Running push core image + command: make push-docker-core "branch=${CIRCLE_BRANCH}" + - run: + name: Running push web-server image + command: make push-docker-web-server "branch=${CIRCLE_BRANCH}" + + staging-deploy-to-swarm: + working_directory: ~/codesearch + machine: true + steps: + - attach_workspace: + at: ~/codesearch + - add_ssh_keys: + fingerprints: + - "27:51:c3:38:40:b1:c2:db:ea:9d:bf:d6:c4:93:a2:e5" + - run: ssh deployer@$STAGING_HOST "mkdir -p ~/docker && echo $DEPLOYER_PASS | sudo -S mkdir -p /mnt/vol/{portainer/data,postgresql,data,index,logs}" || true + - run: scp ./docker/docker-stack-compose.yml deployer@$STAGING_HOST:~/docker + - run: ssh deployer@$STAGING_HOST "docker login quay.io -u $DOCKER_USER -p $DOCKER_PASS" || true + - run: + name: Deploy to remote swarm + command: ssh deployer@$STAGING_HOST "SECRET_KEY=$STAGING_SECRET_KEY SERVER_PORT=$SERVER_PORT docker stack deploy --with-registry-auth -c ~/docker/docker-stack-compose.yml codesearch" || true + + deploy-to-swarm: + working_directory: ~/codesearch + machine: true + steps: + - attach_workspace: + at: ~/codesearch + - add_ssh_keys: + fingerprints: + - "27:51:c3:38:40:b1:c2:db:ea:9d:bf:d6:c4:93:a2:e5" + - run: ssh deployer@$HOST "mkdir -p ~/docker && echo $DEPLOYER_PASS | sudo -S mkdir -p /mnt/vol/{portainer/data,postgresql,data,index,logs}" || true + - run: scp ./docker/docker-stack-compose.yml deployer@$HOST:~/docker + - run: ssh deployer@$HOST "docker login quay.io -u $DOCKER_USER -p $DOCKER_PASS" || true + - run: + name: Deploy to remote swarm + command: ssh deployer@$HOST "SECRET_KEY=$SECRET_KEY SERVER_PORT=$SERVER_PORT docker stack deploy --with-registry-auth -c ~/docker/docker-stack-compose.yml codesearch" || true diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 4c697d6..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,71 +0,0 @@ -stages: - - build - - publish - - deploy-staging - - deploy - -variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - SSH_CONFIG: "-o StrictHostKeyChecking=no" - -build_and_test: - stage: build - image: registry.gitlab.com/aelve/codesearch/ci-test-env - services: - - docker:dind - script: - - sbt compile - - sbt core/assembly - - sbt web-server/assembly - - sbt scalafmt exit - - git diff --exit-code - artifacts: - paths: - - target/resolution-cache - - target/streams - - project/target/resolution-cache - - project/target/streams - - ~/.sbt - - ~/.iv2/cache - - ~/.m2 - - .dockerignore - - docker - - scripts - - Makefile - - codesearch-core.jar - - codesearch-server.jar - -build_and_publish_docker_images: - stage: publish - image: registry.gitlab.com/aelve/codesearch/ci-test-env - services: - - docker:dind - script: - - make build-docker-core - - make build-docker-web-server - - make push-docker-core - - make push-docker-web-server - only: - - develop - - master - -staging-deploy-to-swarm: - stage: deploy-staging - script: - - ssh deployer@$STAGING_HOST "mkdir -p ~/docker && echo $DEPLOYER_PASS | sudo -S mkdir -p /mnt/vol/{portainer/data,postgresql,data,index,logs}" || true - - scp ./docker/docker-stack-compose.yml deployer@$STAGING_HOST:~/docker - - ssh deployer@$STAGING_HOST "docker login quay.io -u $DOCKER_USER -p $DOCKER_PASS" || true - - ssh deployer@$STAGING_HOST "SECRET_KEY=$STAGING_SECRET_KEY SERVER_PORT=$SERVER_PORT docker stack deploy --with-registry-auth -c ~/docker/docker-stack-compose.yml codesearch" || true - only: - - develop - -deploy-to-swarm: - stage: deploy - script: - - ssh deployer@$HOST "mkdir -p ~/docker && echo $DEPLOYER_PASS | sudo -S mkdir -p /mnt/vol/{portainer/data,postgresql,data,index,logs}" || true - - scp ./docker/docker-stack-compose.yml deployer@$HOST:~/docker - - ssh deployer@$HOST "docker login quay.io -u $DOCKER_USER -p $DOCKER_PASS" || true - - ssh deployer@$HOST "SECRET_KEY=$SECRET_KEY SERVER_PORT=$SERVER_PORT docker stack deploy --with-registry-auth -c ~/docker/docker-stack-compose.yml codesearch" || true - only: - - master \ No newline at end of file diff --git a/README.md b/README.md index 855a6bd..a6bc2fd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ -# Aelve Codesearch - -![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/aelve/codesearch/develop.svg) +# codesearch ## Deployment instructions diff --git a/core/src/main/resources/application.conf b/core/src/main/resources/application.conf index bd0cf41..412b6c3 100644 --- a/core/src/main/resources/application.conf +++ b/core/src/main/resources/application.conf @@ -11,6 +11,10 @@ db { port = ${?DATABASE_PORT_NUMBER} dataSourceClass = "slick.jdbc.DatabaseUrlDataSource" + properties = { + driver = "org.postgresql.Driver" + url = "postgres://"${db.user}":"${db.password}"@"${db.host}":"${db.port}"/"${db.name} + } } languagesConfig { diff --git a/core/src/main/scala/codesearch/core/Main.scala b/core/src/main/scala/codesearch/core/Main.scala index ecbb17f..576103c 100644 --- a/core/src/main/scala/codesearch/core/Main.scala +++ b/core/src/main/scala/codesearch/core/Main.scala @@ -1,18 +1,15 @@ package codesearch.core -import java.nio.file.Paths - import cats.effect.{ExitCode, IO, IOApp, Resource} import cats.syntax.flatMap._ import codesearch.core.config.Config +import codesearch.core.db._ import codesearch.core.index._ -import codesearch.core.index.directory._ import codesearch.core.index.repository.Downloader import codesearch.core.meta._ import codesearch.core.model._ import codesearch.core.util.Unarchiver import com.softwaremill.sttp.asynchttpclient.fs2.AsyncHttpClientFs2Backend -import slick.jdbc.PostgresProfile.api._ object Main extends IOApp { @@ -26,6 +23,7 @@ object Main extends IOApp { ) case class LangRep[A <: DefaultTable]( + db: DefaultDB[A], langIndex: LanguageIndex[A], metaDownloader: MetaDownloader[IO] ) @@ -44,20 +42,11 @@ object Main extends IOApp { gemMeta <- GemMetaDownloader(config.languagesConfig.ruby, downloader) npmMeta <- NpmMetaDownloader(config.languagesConfig.javascript, downloader) - db = Database.forConfig("db") - - cindexPath = Paths.get("./index/cindex/") - - haskellCindex = HaskellCindex(cindexPath) - rustCindex = RustCindex(cindexPath) - rubyCindex = RubyCindex(cindexPath) - javaScriptCindex = JavaScriptCindex(cindexPath) - langReps = Map( - "haskell" -> LangRep[HackageTable](HaskellIndex(config, db, haskellCindex), hackageMeta), - "rust" -> LangRep[CratesTable](RustIndex(config, db, rustCindex), cratesMeta), - "ruby" -> LangRep[GemTable](RubyIndex(config, db, rubyCindex), gemMeta), - "javascript" -> LangRep[NpmTable](JavaScriptIndex(config, db, javaScriptCindex), npmMeta) + "haskell" -> LangRep[HackageTable](HackageDB, HaskellIndex(config), hackageMeta), + "rust" -> LangRep[CratesTable](CratesDB, RustIndex(config), cratesMeta), + "ruby" -> LangRep[GemTable](GemDB, RubyIndex(config), gemMeta), + "javascript" -> LangRep[NpmTable](NpmDB, JavaScriptIndex(config), npmMeta) ) exitCode <- Program(langReps) >>= (_.run(params)) } yield exitCode diff --git a/core/src/main/scala/codesearch/core/Program.scala b/core/src/main/scala/codesearch/core/Program.scala index 469bbeb..514191a 100644 --- a/core/src/main/scala/codesearch/core/Program.scala +++ b/core/src/main/scala/codesearch/core/Program.scala @@ -43,7 +43,7 @@ class Program(langReps: Map[String, LangRep[_ <: DefaultTable]], logger: Logger[ def initDb(params: Params): IO[Unit] = for { languages <- findRepositories(params.lang) - _ <- languages.traverse_(_.langIndex.initDB) + _ <- languages.traverse_(_.db.initDB) } yield () def downloadMeta(params: Params): IO[Unit] = { diff --git a/core/src/main/scala/codesearch/core/config/Config.scala b/core/src/main/scala/codesearch/core/config/Config.scala index 2c4dce4..245b96d 100644 --- a/core/src/main/scala/codesearch/core/config/Config.scala +++ b/core/src/main/scala/codesearch/core/config/Config.scala @@ -16,7 +16,6 @@ case class Config( case class DatabaseConfig( dataSourceClass: String, - host: String, port: Int, name: String, user: String, diff --git a/core/src/main/scala/codesearch/core/db/DefaultDB.scala b/core/src/main/scala/codesearch/core/db/DefaultDB.scala index d23bda7..7de6326 100644 --- a/core/src/main/scala/codesearch/core/db/DefaultDB.scala +++ b/core/src/main/scala/codesearch/core/db/DefaultDB.scala @@ -12,11 +12,14 @@ import slick.lifted.TableQuery import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -trait DefaultDB[T <: DefaultTable] { +object DefaultDB { + lazy val db = Database.forConfig("db") +} - def db: Database +trait DefaultDB[T <: DefaultTable] { val table: TableQuery[T] + lazy val db = DefaultDB.db def insertOrUpdate[A <: SourcePackage](pack: A): IO[Int] = { val insOrUpdate = table.insertOrUpdate( @@ -24,7 +27,8 @@ trait DefaultDB[T <: DefaultTable] { pack.name, pack.version, new Timestamp(System.currentTimeMillis()) - )) + ) + ) IO.fromFuture(IO(db.run(insOrUpdate))) } @@ -90,3 +94,8 @@ trait NpmDB extends DefaultDB[NpmTable] { trait GemDB extends DefaultDB[GemTable] { val table = TableQuery[GemTable] } + +object HackageDB extends HackageDB +object CratesDB extends CratesDB +object NpmDB extends NpmDB +object GemDB extends GemDB diff --git a/core/src/main/scala/codesearch/core/index/HaskellIndex.scala b/core/src/main/scala/codesearch/core/index/HaskellIndex.scala index 0f5bd82..0b9c565 100644 --- a/core/src/main/scala/codesearch/core/index/HaskellIndex.scala +++ b/core/src/main/scala/codesearch/core/index/HaskellIndex.scala @@ -12,16 +12,18 @@ import codesearch.core.db.HackageDB import codesearch.core.index.repository.{HackagePackage, SourcesDownloader} import codesearch.core.index.directory.Directory._ import codesearch.core.index.directory.Directory.ops._ -import codesearch.core.index.directory._ +import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory.HaskellCindex import codesearch.core.model.{HackageTable, Version} import fs2.{Chunk, Stream} -import slick.jdbc.PostgresProfile.api._ -class HaskellIndex(haskellConfig: HaskellConfig, val db: Database, val cindexDir: СindexDirectory)( +class HaskellIndex(haskellConfig: HaskellConfig)( implicit val shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, HackagePackage] ) extends LanguageIndex[HackageTable] with HackageDB { + override protected val cindexDir: СindexDirectory = HaskellCindex + override protected def concurrentTasksCount: Int = haskellConfig.concurrentTasksCount override protected def updateSources(name: String, version: String): IO[Int] = { @@ -49,8 +51,8 @@ class HaskellIndex(haskellConfig: HaskellConfig, val db: Database, val cindexDir } object HaskellIndex { - def apply(config: Config, db: Database, cindexDir: СindexDirectory)( + def apply(config: Config)( implicit shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, HackagePackage] - ) = new HaskellIndex(config.languagesConfig.haskell, db, cindexDir) + ) = new HaskellIndex(config.languagesConfig.haskell) } diff --git a/core/src/main/scala/codesearch/core/index/JavaScriptIndex.scala b/core/src/main/scala/codesearch/core/index/JavaScriptIndex.scala index 416867f..0b429d0 100644 --- a/core/src/main/scala/codesearch/core/index/JavaScriptIndex.scala +++ b/core/src/main/scala/codesearch/core/index/JavaScriptIndex.scala @@ -11,15 +11,17 @@ import codesearch.core.index.repository.{NpmPackage, SourcesDownloader} import codesearch.core.index.directory.Directory._ import codesearch.core.index.directory.Directory.ops._ import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory.JavaScriptCindex import codesearch.core.model.NpmTable import fs2.Stream -import slick.jdbc.PostgresProfile.api._ -class JavaScriptIndex(config: JavaScriptConfig, val db: Database, val cindexDir: СindexDirectory)( +class JavaScriptIndex(config: JavaScriptConfig)( implicit val shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, NpmPackage] ) extends LanguageIndex[NpmTable] with NpmDB { + override protected val cindexDir: СindexDirectory = JavaScriptCindex + override protected def concurrentTasksCount: Int = config.concurrentTasksCount override protected def updateSources(name: String, version: String): IO[Int] = { @@ -33,8 +35,8 @@ class JavaScriptIndex(config: JavaScriptConfig, val db: Database, val cindexDir: } object JavaScriptIndex { - def apply(config: Config, db: Database, cindexDir: СindexDirectory)( + def apply(config: Config)( implicit shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, NpmPackage] - ) = new JavaScriptIndex(config.languagesConfig.javascript, db, cindexDir) + ) = new JavaScriptIndex(config.languagesConfig.javascript) } diff --git a/core/src/main/scala/codesearch/core/index/LanguageIndex.scala b/core/src/main/scala/codesearch/core/index/LanguageIndex.scala index e987d0d..d3d5d33 100644 --- a/core/src/main/scala/codesearch/core/index/LanguageIndex.scala +++ b/core/src/main/scala/codesearch/core/index/LanguageIndex.scala @@ -25,13 +25,11 @@ import scala.sys.process.Process trait LanguageIndex[A <: DefaultTable] { self: DefaultDB[A] => - def initDB: IO[Unit] - protected implicit def shift: ContextShift[IO] protected val logger: SelfAwareStructuredLogger[IO] = Slf4jLogger.unsafeCreate[IO] - def cindexDir: СindexDirectory + protected def cindexDir: СindexDirectory protected def concurrentTasksCount: Int @@ -50,8 +48,8 @@ trait LanguageIndex[A <: DefaultTable] { def dropTempIndexFile = IO(Files.deleteIfExists(cindexDir.tempIndexDirAs[NioPath])) def createCSearchDir = IO( - if (Files.notExists(cindexDir.root)) - Files.createDirectories(cindexDir.root) + if (Files.notExists(СindexDirectory.root)) + Files.createDirectories(СindexDirectory.root) ) def indexPackages(packageDirs: Seq[NioPath]): IO[Unit] = { diff --git a/core/src/main/scala/codesearch/core/index/RubyIndex.scala b/core/src/main/scala/codesearch/core/index/RubyIndex.scala index 1c37ed7..2f3c8c1 100644 --- a/core/src/main/scala/codesearch/core/index/RubyIndex.scala +++ b/core/src/main/scala/codesearch/core/index/RubyIndex.scala @@ -10,18 +10,20 @@ import codesearch.core.db.GemDB import codesearch.core.index.directory.Directory._ import codesearch.core.index.directory.Directory.ops._ import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory.RubyCindex import codesearch.core.index.repository.{GemPackage, SourcesDownloader} import codesearch.core.model.GemTable import io.circe.fs2._ import fs2.Stream import fs2.io.file -import slick.jdbc.PostgresProfile.api._ -class RubyIndex(rubyConfig: RubyConfig, val db: Database, val cindexDir: СindexDirectory)( +class RubyIndex(rubyConfig: RubyConfig)( implicit val shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, GemPackage] ) extends LanguageIndex[GemTable] with GemDB { + override protected val cindexDir: СindexDirectory = RubyCindex + override protected def concurrentTasksCount: Int = rubyConfig.concurrentTasksCount override protected def updateSources(name: String, version: String): IO[Int] = { @@ -41,8 +43,8 @@ class RubyIndex(rubyConfig: RubyConfig, val db: Database, val cindexDir: Сindex } object RubyIndex { - def apply(config: Config, db: Database, cindexDir: СindexDirectory)( + def apply(config: Config)( implicit shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, GemPackage] - ) = new RubyIndex(config.languagesConfig.ruby, db, cindexDir) + ) = new RubyIndex(config.languagesConfig.ruby) } diff --git a/core/src/main/scala/codesearch/core/index/RustIndex.scala b/core/src/main/scala/codesearch/core/index/RustIndex.scala index 859980e..e6ec163 100644 --- a/core/src/main/scala/codesearch/core/index/RustIndex.scala +++ b/core/src/main/scala/codesearch/core/index/RustIndex.scala @@ -9,15 +9,15 @@ import codesearch.core.db.CratesDB import codesearch.core.index.directory.Directory._ import codesearch.core.index.directory.Directory.ops._ import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory.RustCindex import codesearch.core.index.repository.{CratesPackage, SourcesDownloader} import codesearch.core.model.CratesTable import codesearch.core.util.Helper import fs2.Stream import io.circe.Decoder import io.circe.fs2._ -import slick.jdbc.PostgresProfile.api._ -class RustIndex(rustConfig: RustConfig, val db: Database, val cindexDir: СindexDirectory)( +class RustIndex(rustConfig: RustConfig)( implicit val shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, CratesPackage] ) extends LanguageIndex[CratesTable] with CratesDB { @@ -29,6 +29,8 @@ class RustIndex(rustConfig: RustConfig, val db: Database, val cindexDir: Сindex "archive.zip" ) + override protected val cindexDir: СindexDirectory = RustCindex + override protected def concurrentTasksCount: Int = rustConfig.concurrentTasksCount override protected def updateSources(name: String, version: String): IO[Int] = { @@ -56,8 +58,8 @@ class RustIndex(rustConfig: RustConfig, val db: Database, val cindexDir: Сindex } object RustIndex { - def apply(config: Config, db: Database, cindexDir: СindexDirectory)( + def apply(config: Config)( implicit shift: ContextShift[IO], sourcesDownloader: SourcesDownloader[IO, CratesPackage] - ) = new RustIndex(config.languagesConfig.rust, db, cindexDir) + ) = new RustIndex(config.languagesConfig.rust) } diff --git a/core/src/main/scala/codesearch/core/index/directory/CindexDirectory.scala b/core/src/main/scala/codesearch/core/index/directory/CindexDirectory.scala index 2912cd5..1e90e7b 100644 --- a/core/src/main/scala/codesearch/core/index/directory/CindexDirectory.scala +++ b/core/src/main/scala/codesearch/core/index/directory/CindexDirectory.scala @@ -1,14 +1,12 @@ package codesearch.core.index.directory -import java.nio.file.Path +import java.nio.file.{Path, Paths} import codesearch.core.syntax.path._ +import codesearch.core.index.directory.СindexDirectory.root trait СindexDirectory { - /** Defined main path to index */ - def root: Path - /** Defines package repository name */ def packageRepository: String @@ -17,58 +15,61 @@ trait СindexDirectory { * @param D is implicit instance of [[DirAs]] trait * @tparam O is return type */ - def indexDirAs[O](implicit D: DirAs[O]): O = D.dir(packageRepository, root) + def indexDirAs[O](implicit D: DirAs[O]): O = D.dir(packageRepository) /** Function returns temporary index directory in representation depending from type parameter * * @param D is implicit instance of [[DirAs]] trait * @tparam O is return type */ - def tempIndexDirAs[O](implicit D: DirAs[O]): O = D.tempDir(packageRepository, root) + def tempIndexDirAs[O](implicit D: DirAs[O]): O = D.tempDir(packageRepository) /** Function returns path to file that contains directories to index in representation depending from type parameter * * @param D is implicit instance of [[DirAs]] trait * @tparam O is return type */ - def dirsToIndex[O](implicit D: DirAs[O]): O = D.dirsToIndex(packageRepository, root) + def dirsToIndex[O](implicit D: DirAs[O]): O = D.dirsToIndex(packageRepository) } -final case class HaskellCindex(root: Path) extends СindexDirectory { - def packageRepository: String = "hackage" -} +object СindexDirectory { + private[index] val root: Path = Paths.get("./index/cindex/") -final case class JavaScriptCindex(root: Path) extends СindexDirectory { - def packageRepository: String = "npm" -} + final object HaskellCindex extends СindexDirectory { + def packageRepository: String = "hackage" + } -final case class RubyCindex(root: Path) extends СindexDirectory { - def packageRepository: String = "gem" -} + final object JavaScriptCindex extends СindexDirectory { + def packageRepository: String = "npm" + } + + final object RubyCindex extends СindexDirectory { + def packageRepository: String = "gem" + } -final case class RustCindex(root: Path) extends СindexDirectory { - def packageRepository: String = "crates" + final object RustCindex extends СindexDirectory { + def packageRepository: String = "crates" + } } trait DirAs[A] { - def dir(packageManager: String, root: Path): A - def tempDir(packageManager: String, root: Path): A - def dirsToIndex(packageManager: String, root: Path): A + def dir(packageManager: String): A + def tempDir(packageManager: String): A + def dirsToIndex(packageManager: String): A } object DirAs { implicit def asString: DirAs[String] = new DirAs[String] { - private def fullPath(relativePath: Path): String = relativePath.toFile.getCanonicalPath - override def dirsToIndex(packageManager: String, root: Path): String = - s"${asPath.dirsToIndex(packageManager, root)}" - override def dir(packageManager: String, root: Path): String = fullPath(asPath.dir(packageManager, root)) - override def tempDir(packageManager: String, root: Path): String = fullPath(asPath.tempDir(packageManager, root)) + private def fullPath(relativePath: Path): String = relativePath.toFile.getCanonicalPath + override def dirsToIndex(packageManager: String): String = s"${asPath.dirsToIndex(packageManager)}" + override def dir(packageManager: String): String = fullPath(asPath.dir(packageManager)) + override def tempDir(packageManager: String): String = fullPath(asPath.tempDir(packageManager)) } implicit def asPath: DirAs[Path] = new DirAs[Path] { - private def index(packageManager: String, root: Path): String = s".${packageManager}_csearch_index" - override def dirsToIndex(packageManager: String, root: Path): Path = root / s".${packageManager}_dirs_for_index" - override def dir(packageManager: String, root: Path): Path = root / index(packageManager, root) - override def tempDir(packageManager: String, root: Path): Path = root / s"${index(packageManager, root)}.tmp" + private def index(packageManager: String): String = s".${packageManager}_csearch_index" + override def dirsToIndex(packageManager: String): Path = root / s".${packageManager}_dirs_for_index" + override def dir(packageManager: String): Path = root / index(packageManager) + override def tempDir(packageManager: String): Path = root / s"${index(packageManager)}.tmp" } } diff --git a/core/src/main/scala/codesearch/core/index/directory/Extractor.scala b/core/src/main/scala/codesearch/core/index/directory/Extractor.scala index 0cb2eea..bf3a172 100644 --- a/core/src/main/scala/codesearch/core/index/directory/Extractor.scala +++ b/core/src/main/scala/codesearch/core/index/directory/Extractor.scala @@ -1,6 +1,5 @@ package codesearch.core.index.directory -import java.io.File import java.nio.file.Path import cats.effect.Sync @@ -45,9 +44,13 @@ private[index] trait Extractor { val dir = unarchived.toFile dir.listFiles .filter(_.isDirectory) - .foreach(_.listFiles.foreach(file => - if (file.isDirectory) moveDirectoryToDirectory(file, dir, false) - else moveFileToDirectory(file, dir, false))) + .foreach( + _.listFiles.foreach( + file => + if (file.isDirectory) moveDirectoryToDirectory(file, dir, false) + else moveFileToDirectory(file, dir, false) + ) + ) unarchived } } diff --git a/core/src/main/scala/codesearch/core/index/repository/Downloader.scala b/core/src/main/scala/codesearch/core/index/repository/Downloader.scala index 38c3bff..fc05a16 100644 --- a/core/src/main/scala/codesearch/core/index/repository/Downloader.scala +++ b/core/src/main/scala/codesearch/core/index/repository/Downloader.scala @@ -1,6 +1,5 @@ package codesearch.core.index.repository -import java.io.File import java.nio.ByteBuffer import java.nio.file.Path import java.nio.file.StandardOpenOption.{CREATE, TRUNCATE_EXISTING} diff --git a/core/src/main/scala/codesearch/core/regex/lexer/Tokenizer.scala b/core/src/main/scala/codesearch/core/regex/lexer/Tokenizer.scala index dfbbda5..c0fb08c 100644 --- a/core/src/main/scala/codesearch/core/regex/lexer/Tokenizer.scala +++ b/core/src/main/scala/codesearch/core/regex/lexer/Tokenizer.scala @@ -46,5 +46,7 @@ object Tokenizer { private def parserAnyStringBeforeSpecialSymbol[_: P] = P((!" " ~ !specialSymbols ~ AnyChar).rep(1).!.map(Literal)) private def parseStringWithSpecialSymbols[_: P] = - P(parserEscaped | parserCharSet | parserAnyStringBeforeSpecialSymbol | parseSpaces | parseRepetitionSeq | parserSpecialSymbol).rep + P( + parserEscaped | parserCharSet | parserAnyStringBeforeSpecialSymbol | parseSpaces | parseRepetitionSeq | parserSpecialSymbol + ).rep } diff --git a/core/src/main/scala/codesearch/core/search/HaskellSearch.scala b/core/src/main/scala/codesearch/core/search/HaskellSearch.scala index 398c84a..f873d7e 100644 --- a/core/src/main/scala/codesearch/core/search/HaskellSearch.scala +++ b/core/src/main/scala/codesearch/core/search/HaskellSearch.scala @@ -1,11 +1,13 @@ package codesearch.core.search import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory._ import codesearch.core.index.repository.Extensions import codesearch.core.index.repository.Extensions.HaskellExtensions -class HaskellSearch(val cindexDir: СindexDirectory) extends Search { - override protected def extensions: Extensions = HaskellExtensions +class HaskellSearch extends Search { + override protected def cindexDir: СindexDirectory = HaskellCindex + override protected def extensions: Extensions = HaskellExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://hackage.haskell.org/package/$packageName-$version" } diff --git a/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala b/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala index e568733..9b6b77b 100644 --- a/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala +++ b/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala @@ -1,11 +1,13 @@ package codesearch.core.search import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory.JavaScriptCindex import codesearch.core.index.repository.Extensions import codesearch.core.index.repository.Extensions.JavaScriptExtensions -class JavaScriptSearch(val cindexDir: СindexDirectory) extends Search { - override protected def extensions: Extensions = JavaScriptExtensions +class JavaScriptSearch extends Search { + override protected def cindexDir: СindexDirectory = JavaScriptCindex + override protected def extensions: Extensions = JavaScriptExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://www.npmjs.com/package/$packageName/v/$version" } diff --git a/core/src/main/scala/codesearch/core/search/RubySearch.scala b/core/src/main/scala/codesearch/core/search/RubySearch.scala index 2dc3f55..1ac8bcf 100644 --- a/core/src/main/scala/codesearch/core/search/RubySearch.scala +++ b/core/src/main/scala/codesearch/core/search/RubySearch.scala @@ -1,11 +1,13 @@ package codesearch.core.search import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory.RubyCindex import codesearch.core.index.repository.Extensions import codesearch.core.index.repository.Extensions.RubyExtensions -class RubySearch(val cindexDir: СindexDirectory) extends Search { - override protected def extensions: Extensions = RubyExtensions +class RubySearch extends Search { + override protected def cindexDir: СindexDirectory = RubyCindex + override protected def extensions: Extensions = RubyExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://rubygems.org/gems/$packageName/versions/$version" } diff --git a/core/src/main/scala/codesearch/core/search/RustSearch.scala b/core/src/main/scala/codesearch/core/search/RustSearch.scala index 6e0ba8f..99b3ad6 100644 --- a/core/src/main/scala/codesearch/core/search/RustSearch.scala +++ b/core/src/main/scala/codesearch/core/search/RustSearch.scala @@ -1,11 +1,13 @@ package codesearch.core.search import codesearch.core.index.directory.СindexDirectory +import codesearch.core.index.directory.СindexDirectory.RustCindex import codesearch.core.index.repository.Extensions import codesearch.core.index.repository.Extensions.RustExtensions -class RustSearch(val cindexDir: СindexDirectory) extends Search { - override protected def extensions: Extensions = RustExtensions +class RustSearch extends Search { + override protected def cindexDir: СindexDirectory = RustCindex + override protected def extensions: Extensions = RustExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://docs.rs/crate/$packageName/$version" } diff --git a/core/src/main/scala/codesearch/core/search/Search.scala b/core/src/main/scala/codesearch/core/search/Search.scala index b49fe43..ae30da3 100644 --- a/core/src/main/scala/codesearch/core/search/Search.scala +++ b/core/src/main/scala/codesearch/core/search/Search.scala @@ -1,7 +1,6 @@ package codesearch.core.search import java.net.URLDecoder -import java.nio.file.{Path => NioPath} import ammonite.ops.{Path, pwd} import cats.data.NonEmptyVector @@ -10,38 +9,60 @@ import cats.syntax.option._ import codesearch.core.config.{Config, SnippetConfig} import codesearch.core.index.directory.СindexDirectory import codesearch.core.index.repository.Extensions -import codesearch.core.search.Search.{CSearchPage, CSearchResult, CodeSnippet, Package, PackageResult, snippetConfig} +import codesearch.core.regex.PreciseMatch +import codesearch.core.regex.space.SpaceInsensitiveString +import codesearch.core.search.Search.{CSearchResult, CodeSnippet, Package, PackageResult, snippetConfig} import codesearch.core.search.SnippetsGrouper.SnippetInfo import codesearch.core.util.Helper.readFileAsync +import com.google.re2j._ import fs2.{Pipe, Stream} import io.chrisdavenport.log4cats.SelfAwareStructuredLogger import io.chrisdavenport.log4cats.slf4j.Slf4jLogger -import codesearch.core.regex.RegexConstructor -import scala.sys.process.Process +import scala.sys.process.{Process, ProcessLogger} trait Search { + protected def cindexDir: СindexDirectory protected def extensions: Extensions protected val logger: SelfAwareStructuredLogger[IO] = Slf4jLogger.unsafeCreate[IO] - def cindexDir: СindexDirectory - - implicit val root: NioPath = cindexDir.root - - def search(request: SearchRequest): IO[CSearchPage] = { - for { - lines <- csearch(request) - results <- Stream - .emits(lines) - .through(SnippetsGrouper.groupLines(snippetConfig)) - .drop(snippetConfig.pageSize * (request.page - 1)) - .take(snippetConfig.pageSize) - .evalMap(createSnippet) - .through(groupByPackage) - .compile - .toList - } yield CSearchPage(results.sortBy(_.pack.name), lines.size) + def checkRegexpForValid(regexp: String): IO[Pattern] = { + IO { Pattern.compile(regexp) } + } + + def search(request: SearchRequest): IO[Response] = { + checkRegexpForValid(request.query).attempt.flatMap { + case Left(error) => + IO(createErrorResponse(error)) + case Right(_) => + val entity = csearch(request) + for { + results <- Stream + .emits(entity.lists) + .through(SnippetsGrouper.groupLines(snippetConfig)) + .drop(snippetConfig.pageSize * (request.page - 1)) + .take(snippetConfig.pageSize) + .evalMap(createSnippet) + .through(groupByPackage) + .compile + .toList + } yield CSearchPage(results.sortBy(_.pack.name), entity.lists.length) + } + } + + private def createErrorResponse(error: Throwable): ErrorResponse = + ErrorResponse(error.getMessage.substring(0, 1).toUpperCase + error.getMessage.substring(1, error.getMessage.length)) + + private def csearch(request: SearchRequest): SearchByIndexResult = { + val indexDir = cindexDir.indexDirAs[String] + val env = ("CSEARCHINDEX", indexDir) + var stderr = new String + val log = ProcessLogger((o: String) => o, (e: String) => stderr = e) + val test = for { + results <- IO((Process(arguments(request), None, env) #| Seq("head", "-1001")).lineStream_!(log).toList) + } yield SearchByIndexResult(results, ErrorResponse(stderr)) + test.unsafeRunSync() } /** @@ -69,16 +90,6 @@ trait Search { */ protected def buildRepUrl(packageName: String, version: String): String - private def csearch(request: SearchRequest): IO[List[String]] = { - val indexDir = cindexDir.indexDirAs[String] - val env = ("CSEARCHINDEX", indexDir) - - for { - _ <- logger.debug(s"running CSEARCHINDEX=$indexDir ${arguments(request).mkString(" ")}") - results <- IO((Process(arguments(request), None, env) #| Seq("head", "-1001")).lineStream.toList) - } yield results - } - private def arguments(request: SearchRequest): List[String] = { def extensionsRegex: String = extensions.sourceExtensions.mkString(".*\\.(", "|", ")$") @@ -87,8 +98,10 @@ trait Search { case None => if (request.sourcesOnly) extensionsRegex else ".*" } - val query: String = - RegexConstructor(request.query, request.insensitive, request.spaceInsensitive, request.preciseMatch) + val query: String = { + val preciseMatch: String = if (request.preciseMatch) PreciseMatch(request.query) else request.query + if (request.spaceInsensitive) SpaceInsensitiveString(preciseMatch) else preciseMatch + } request.filter match { case Some(filter) => List("csearch", "-n", "-f", forExtensions, query, filter) @@ -154,11 +167,6 @@ object Search { * @param data code snippets grouped by package * @param total number of total matches */ - final case class CSearchPage( - data: Seq[PackageResult], - total: Int - ) - /** * * @param relativePath path into package sources @@ -204,3 +212,25 @@ object Search { result: CodeSnippet ) } +sealed trait Response +final case class SearchByIndexResult(lists: List[String], error: ErrorResponse) +final case class ErrorResponse(message: String) extends Response +final case class CSearchPage( + data: Seq[PackageResult], + total: Int +) extends Response +final case class SuccessResponse( + updated: String, + packages: Seq[PackageResult], + query: String, + filter: Option[String], + filePath: Option[String], + insensitive: Boolean, + space: Boolean, + precise: Boolean, + sources: Boolean, + page: Int, + totalMatches: Int, + callURI: String, + lang: String +) extends Response diff --git a/core/src/main/scala/codesearch/core/search/SearchRequest.scala b/core/src/main/scala/codesearch/core/search/SearchRequest.scala index 82e3025..8b0bd93 100644 --- a/core/src/main/scala/codesearch/core/search/SearchRequest.scala +++ b/core/src/main/scala/codesearch/core/search/SearchRequest.scala @@ -32,7 +32,8 @@ case class SearchRequest( def stringify(x: Boolean): String = if (x) "on" else "off" uri"$host/$lang/search?query=$query&filter=$filter&filePath=$filePath&insensitive=${stringify(insensitive)}&space=${stringify( - spaceInsensitive)}&precise=${stringify(preciseMatch)}&sources=${stringify(sourcesOnly)}" + spaceInsensitive + )}&precise=${stringify(preciseMatch)}&sources=${stringify(sourcesOnly)}" } } @@ -57,7 +58,7 @@ object SearchRequest { isEnabled(spaceInsensitive), isEnabled(preciseMatch), isEnabled(sourcesOnly), - page.toInt, + page.toInt ) } diff --git a/core/src/test/resources/integration/meta/haskell.tar.gz b/core/src/test/resources/integration/meta/haskell.tar.gz deleted file mode 100644 index 4967736..0000000 Binary files a/core/src/test/resources/integration/meta/haskell.tar.gz and /dev/null differ diff --git a/core/src/test/resources/integration/meta/node.json b/core/src/test/resources/integration/meta/node.json deleted file mode 100644 index c662a0b..0000000 --- a/core/src/test/resources/integration/meta/node.json +++ /dev/null @@ -1,724 +0,0 @@ -{ - "total_rows": 906836, - "offset": 0, - "rows": [ - { - "id": "00-components", - "key": "00-components", - "value": { - "rev": "3-d1a20a6184b1b5726ad774276ab97fc9" - }, - "doc": { - "_id": "00-components", - "_rev": "3-d1a20a6184b1b5726ad774276ab97fc9", - "name": "00-components", - "dist-tags": { - "latest": "0.5.0" - }, - "versions": { - "0.3.0": { - "name": "00-components", - "version": "0.3.0", - "private": false, - "main": "dist/index.js", - "module": "dist/index.js", - "scripts": { - "start": "react-scripts start", - "build": "rm -rf dist && NODE_ENV=production babel src/components --out-dir dist --copy-files --ignore __tests__,spec.js,test.js,__snapshots__", - "test": "react-scripts test", - "eject": "react-scripts eject", - "storybook": "start-storybook -p 9009", - "build-storybook": "build-storybook" - }, - "eslintConfig": { - "extends": "react-app" - }, - "browserslist": [ - ">0.2%", - "not dead", - "not ie <= 11", - "not op_mini all" - ], - "peerDependencies": { - "react": "^16.x", - "react-dom": "^16.x", - "styled-components": "^4.x" - }, - "devDependencies": { - "@babel/cli": "^7.2.3", - "@babel/core": "^7.2.2", - "@storybook/addon-actions": "^4.1.11", - "@storybook/addon-links": "^4.1.11", - "@storybook/addons": "^4.1.11", - "@storybook/react": "^4.1.11", - "babel-loader": "^8.0.5", - "react": "^16.8.1", - "react-dom": "^16.8.1", - "react-scripts": "^2.1.5", - "styled-components": "^4.1.3" - }, - "gitHead": "cfd7ed22723ffc7a873bb59d5b98d434d5a4c5e1", - "description": "This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).", - "_id": "00-components@0.3.0", - "_nodeVersion": "10.14.0", - "_npmVersion": "lerna/3.11.1/node@v10.14.0+x64 (darwin)", - "dist": { - "integrity": "sha512-MH0s62YBZ+wdhBESSn5Jrvb9Bhl7iKz0nfSukEgU3gpVCnsrFfhwqtE8Cfxr4hzL/dq9SQbvFQvz9S/6IqIlEQ==", - "shasum": "f62344ee75f425f93fccb18aedfb9c0250777666", - "tarball": "https://registry.npmjs.org/00-components/-/00-components-0.3.0.tgz", - "fileCount": 4, - "unpackedSize": 4276, - "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcZDP8CRA9TVsSAnZWagAAmzAQAIgvHq2BJmtOgPYKV0ax\nAxoEBTtejxhbqIxYCyiZLizVGWlm7ZwY36PHg7a2G6FfCvbewVrLs90xAwHn\nD6huepKQeGe+8300cgCeGU74wkFZLBx5LwTp9zt53u8G7dnaXxxDaQyGzc4x\nqFX9zueQQGewas2x5/ttX4feauLwTxauc/kg+QPFyXFBXVRxdgb9crU9xJLV\nRasF9yZI3ii8w6RzAZWYgfjJewVIbRuFN5kYD8GJTnXUaeQTY+MxYtGGvlbs\nXalWAKUI2DR9UZS2qYoC3+jYe1YhQ8EQ+VdVCYNW4lJ8/2aaNLAI28kknlcv\nckk/3ZVBGdJJa0mCbY8OVaYVvi84wibH2X7RbSkzPaNZc5RIs2u+n/2gUVlg\nE9gBdir0THfYF/0cVZpNpcCXHB8o6gx8hXIOkN3zVmUAYBFl/7eaTROXMAgN\nFFQv4x7i6iqGuSswQ1GpwxzVx1XK/4p0HLTBZ/Xikawk1rnl5mCoDqYtYHPc\nfTbcKcHfUqhNkXpvaYldPXGkyBo7h5dQ2Ha02CX2hmLMFW1uMs672Hq9VrNN\nf5UpS4dChkaSbs6Ew3NTtDBDvVorHI3Y+h9WBfuRfIULb+4HT/wJzQYBflvH\nrK2a8n82+eSwEfHsJZn85yP5tmhKdoIwqOqS5uYcLUPeGeLvwRVnF+fXw+93\ndGm2\r\n=alPx\r\n-----END PGP SIGNATURE-----\r\n" - }, - "maintainers": [ - { - "name": "storo90", - "email": "sergio.toro.castano@gmail.com" - } - ], - "_npmUser": { - "name": "storo90", - "email": "sergio.toro.castano@gmail.com" - }, - "directories": {}, - "_npmOperationalInternal": { - "host": "s3://npm-registry-packages", - "tmp": "tmp/00-components_0.3.0_1550070779743_0.15989475586043866" - }, - "_hasShrinkwrap": false - }, - "0.4.0": { - "name": "00-components", - "version": "0.4.0", - "private": false, - "main": "dist/index.js", - "module": "dist/index.js", - "scripts": { - "start": "react-scripts start", - "build": "rm -rf dist && NODE_ENV=production babel src/components --out-dir dist --copy-files --ignore __tests__,spec.js,test.js,__snapshots__", - "test": "react-scripts test", - "eject": "react-scripts eject", - "storybook": "start-storybook -p 9009", - "build-storybook": "build-storybook" - }, - "eslintConfig": { - "extends": "react-app" - }, - "browserslist": [ - ">0.2%", - "not dead", - "not ie <= 11", - "not op_mini all" - ], - "peerDependencies": { - "react": "^16.x", - "react-dom": "^16.x", - "styled-components": "^4.x" - }, - "devDependencies": { - "@babel/cli": "^7.2.3", - "@babel/core": "^7.2.2", - "@storybook/addon-actions": "^4.1.11", - "@storybook/addon-links": "^4.1.11", - "@storybook/addons": "^4.1.11", - "@storybook/react": "^4.1.11", - "babel-loader": "^8.0.5", - "react": "^16.8.1", - "react-dom": "^16.8.1", - "react-scripts": "^2.1.5", - "styled-components": "^4.1.3" - }, - "gitHead": "54d920abb861cd232cc6ac86cdfbecd6942c0fd5", - "description": "This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).", - "_id": "00-components@0.4.0", - "_nodeVersion": "10.14.0", - "_npmVersion": "lerna/3.11.1/node@v10.14.0+x64 (darwin)", - "dist": { - "integrity": "sha512-fys0y6WwFUVji5UGsfLVkg+mICTVRn+oTYlR85MItZAxXIcUAr8jU9EXiPT1SHzdp9rDZGbdFgdMR1gPu1w4sg==", - "shasum": "f61140ab5b44812899efac35f94bf2011f5d7721", - "tarball": "https://registry.npmjs.org/00-components/-/00-components-0.4.0.tgz", - "fileCount": 11, - "unpackedSize": 22344, - "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcZWPzCRA9TVsSAnZWagAA6/QP/1OlwM11vv5qrAJKxKyb\nM8N0+bSAXGO0FFqz8Hz6dblC2Zfal9H1EuLny6oS6TFALOJzewXXpiaCZMZr\nrFWSPSaPRSYahfNavkB0W+VZTcLMQieVg4zgbTt7OMSfLnK60sp8O1qgazBj\nnIruxL/DAmGWShV87t8SuAOMw091wIU0OXcmBXQGYat+LaTjVHDupCe/SrjD\n2ii7sOI5JdNaCM63pRoebpjG01icYvmpwoO0BIScN4qmrRO5nideW1ySVTKg\n3UiMQf6rJwJreb+Z+OLFJME09FB772n6X7uUPaNhQ8eIj6tVewXDJxNZ8JjZ\nWgR1IXUMLqFUTE3568OZOghybuPOIz5t/3uePpB9DpLIMWMmgXfOF3+GXxM6\ni2K3KRoay3EtZLuA/+totegz1KAbHqtIeFakj0bF24kbevTOWwvig/W6hcYx\n8HT9NCIGf+3c0PLx4wMlPOqHH8mPw7QMCTCo+5n57hKapV+EL3VPXlT5tmxX\nyU6KGAkvmOMGBIhOLSHYGU1C6j4YDk5Tt2rrguLO2BOlMmkS15hyx07Z6iRS\nhAcYPU7pueREloV4Xwt+7DI/IT6ad/kcNZQ4fV+x7vz5RNqTPafyRkAOc2Jx\nUclzd1vZNY9gsEF9GCBESti6fUoRB2bz9tnJmowvzHi4iqr5JpE8njlC75Fk\nBmEq\r\n=oqmh\r\n-----END PGP SIGNATURE-----\r\n" - }, - "maintainers": [ - { - "name": "storo90", - "email": "sergio.toro.castano@gmail.com" - } - ], - "_npmUser": { - "name": "storo90", - "email": "sergio.toro.castano@gmail.com" - }, - "directories": {}, - "_npmOperationalInternal": { - "host": "s3://npm-registry-packages", - "tmp": "tmp/00-components_0.4.0_1550148595011_0.29098139401825707" - }, - "_hasShrinkwrap": false - }, - "0.5.0": { - "name": "00-components", - "version": "0.5.0", - "private": false, - "main": "dist/index.js", - "module": "dist/index.js", - "scripts": { - "start": "react-scripts start", - "build": "rm -rf dist && NODE_ENV=production babel src/components --out-dir dist --copy-files --ignore __tests__,spec.js,test.js,__snapshots__", - "test": "react-scripts test", - "eject": "react-scripts eject", - "storybook": "start-storybook -p 9009", - "build-storybook": "build-storybook" - }, - "eslintConfig": { - "extends": "react-app" - }, - "browserslist": [ - ">0.2%", - "not dead", - "not ie <= 11", - "not op_mini all" - ], - "peerDependencies": { - "react": "^16.x", - "react-dom": "^16.x", - "styled-components": "^4.x" - }, - "devDependencies": { - "@babel/cli": "^7.2.3", - "@babel/core": "^7.2.2", - "@storybook/addon-actions": "^4.1.11", - "@storybook/addon-links": "^4.1.11", - "@storybook/addons": "^4.1.11", - "@storybook/react": "^4.1.11", - "babel-loader": "^8.0.5", - "react": "^16.8.1", - "react-dom": "^16.8.1", - "react-scripts": "^2.1.5", - "styled-components": "^4.1.3" - }, - "gitHead": "f7c6df1436ba9397c1f6480336b7f8fb76f1e424", - "description": "This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).", - "_id": "00-components@0.5.0", - "_nodeVersion": "10.14.0", - "_npmVersion": "lerna/3.11.1/node@v10.14.0+x64 (darwin)", - "dist": { - "integrity": "sha512-g9oGt7rDcgWRdZA1Yvny/5ulhQKsj5CSPyAKOgrQ7TxvTtFPiiqz8NqThg7aB9+v/coc5RFMk26zhKTRrW2sKg==", - "shasum": "66161701a55b7435492baffa505df605b84b15fe", - "tarball": "https://registry.npmjs.org/00-components/-/00-components-0.5.0.tgz", - "fileCount": 11, - "unpackedSize": 22343, - "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcZW/GCRA9TVsSAnZWagAABMMP/3Lnz0TvF0oSPr1B9t+e\ntDWw6SgG13R1IefsMGpaRz+DEpIvdHXG/ID4EgLMEcPG9abyS4vQO5MTRJ2G\ngB9+3c30G1DJ8/DwS+EVCKjp9qMnxyYuMUGValM8rbfHQxD7do3peVMIG7Ba\n/E7IKTVj1OuVPHC/SL6FU2Td29Lvqd5sxgRKIH8GZfFZ9oho31j9L9btnIgf\nhF+8azsjsRHJC9+xS7w/0i/GwtyepY8Ny6pS5ap+TOjMbkGyxTBX14H/kFqI\nnP72V1zLXGyJqUh5aTk1zC2kUdIDQh0bHJjjdum26LV0T60607wVAn8ZU+IU\nKuY5LDTUb1GZUa3e61SzQl16tuPijnZg3WW552flvgb/PGKrUEjiM1TZGQ6S\nkxJ8NVI/FKdtTlo1fC7IOWi9+kS8xzmCVZkQ5zmDxSSo7U7xwmvSB5Bff3o7\nVMkYp/vYqZoCGPW1DrID1P2r6BSEX1ymMxX8xw+xcmVCR0OUAAO/hpK+hnxq\nX6KapfvUYTCmkA+MX3V7MBNaL3G7u0FFsq6bmEllNzGa0gxE0qB3demxm4Yn\nk5wAupgTv0LrxBterBZJZLD7wCH57AgTqoWGjxFJwNN+lzmpxDflCHT9Ivy7\nY4KYzqjxSKT2MKi5skNxy9JPk1NgbeNMzMRVjBgM9cPsv12ae1sMRk+fEbEd\noGZP\r\n=sZeK\r\n-----END PGP SIGNATURE-----\r\n" - }, - "maintainers": [ - { - "name": "storo90", - "email": "sergio.toro.castano@gmail.com" - } - ], - "_npmUser": { - "name": "storo90", - "email": "sergio.toro.castano@gmail.com" - }, - "directories": {}, - "_npmOperationalInternal": { - "host": "s3://npm-registry-packages", - "tmp": "tmp/00-components_0.5.0_1550151622314_0.1417557481704288" - }, - "_hasShrinkwrap": false - } - }, - "time": { - "created": "2019-02-13T15:12:59.742Z", - "0.3.0": "2019-02-13T15:12:59.897Z", - "modified": "2019-02-14T13:40:26.566Z", - "0.4.0": "2019-02-14T12:49:55.140Z", - "0.5.0": "2019-02-14T13:40:22.460Z" - }, - "maintainers": [ - { - "name": "storo90", - "email": "sergio.toro.castano@gmail.com" - } - ], - "description": "This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).", - "readme": "This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).\n\n## Available Scripts\n\nIn the project directory, you can run:\n\n### `npm start`\n\nRuns the app in the development mode.
\nOpen [http://localhost:3000](http://localhost:3000) to view it in the browser.\n\nThe page will reload if you make edits.
\nYou will also see any lint errors in the console.\n\n### `npm test`\n\nLaunches the test runner in the interactive watch mode.
\nSee the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.\n\n### `npm run build`\n\nBuilds the app for production to the `build` folder.
\nIt correctly bundles React in production mode and optimizes the build for the best performance.\n\nThe build is minified and the filenames include the hashes.
\nYour app is ready to be deployed!\n\nSee the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.\n\n### `npm run eject`\n\n**Note: this is a one-way operation. Once you `eject`, you can’t go back!**\n\nIf you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.\n\nInstead, it will copy all the configuration files and the transitive dependencies (Webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own.\n\nYou don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it.\n\n## Learn More\n\nYou can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).\n\nTo learn React, check out the [React documentation](https://reactjs.org/).\n\n### Code Splitting\n\nThis section has moved here: https://facebook.github.io/create-react-app/docs/code-splitting\n\n### Analyzing the Bundle Size\n\nThis section has moved here: https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size\n\n### Making a Progressive Web App\n\nThis section has moved here: https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app\n\n### Advanced Configuration\n\nThis section has moved here: https://facebook.github.io/create-react-app/docs/advanced-configuration\n\n### Deployment\n\nThis section has moved here: https://facebook.github.io/create-react-app/docs/deployment\n\n### `npm run build` fails to minify\n\nThis section has moved here: https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify\n", - "readmeFilename": "README.md" - } - }, - { - "id": "00-test", - "key": "00-test", - "value": { - "rev": "1-93895bac1edbdda191c339495e8a085e" - }, - "doc": { - "_id": "00-test", - "_rev": "1-93895bac1edbdda191c339495e8a085e", - "name": "00-test", - "dist-tags": { - "latest": "1.0.0" - }, - "versions": { - "1.0.0": { - "name": "00-test", - "version": "1.0.0", - "description": "", - "main": "dist/webpack-numbers.js", - "scripts": { - "build": "webpack" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "lodash": "^4.17.4" - }, - "_id": "00-test@1.0.0", - "_npmVersion": "5.5.1", - "_nodeVersion": "8.9.3", - "_npmUser": { - "name": "eddiejones", - "email": "eddieeddiejones@gmail.com" - }, - "dist": { - "integrity": "sha512-rWiOyHI24fe2MqjiNzKfMxrTsXYfZ7CTZqDJNbTP6/cgWhQfytlxxAi7+Gevu4LuOEBbWMCjAaLtzQDEWCQELA==", - "shasum": "87d40d69aac238428e46fc6b17a2af84c49686e0", - "tarball": "https://registry.npmjs.org/00-test/-/00-test-1.0.0.tgz" - }, - "maintainers": [ - { - "name": "eddiejones", - "email": "eddieeddiejones@gmail.com" - } - ], - "_npmOperationalInternal": { - "host": "s3://npm-registry-packages", - "tmp": "tmp/00-test-1.0.0.tgz_1514518877810_0.4351237171795219" - } - } - }, - "readme": "ERROR: No README data found!", - "maintainers": [ - { - "name": "eddiejones", - "email": "eddieeddiejones@gmail.com" - } - ], - "time": { - "modified": "2017-12-29T03:41:17.870Z", - "created": "2017-12-29T03:41:17.870Z", - "1.0.0": "2017-12-29T03:41:17.870Z" - }, - "keywords": [], - "license": "ISC", - "readmeFilename": "" - } - }, - { - "id": "00.demo", - "key": "00.demo", - "value": { - "rev": "2-d8be83643db0806388c7514660806e9a" - }, - "doc": { - "_id": "00.demo", - "_rev": "2-d8be83643db0806388c7514660806e9a", - "name": "00.demo", - "dist-tags": { - "latest": "1.0.0" - }, - "versions": { - "1.0.0": { - "name": "00.demo", - "version": "1.0.0", - "description": "Tiny NodeJS Static Web server", - "main": "./src/index.js", - "bin": { - "anydoor": "bin/anydoor" - }, - "scripts": { - "mocha": "mocha", - "cover": "istanbul cover _mocha test/mocha.js" - }, - "keywords": [], - "author": "", - "license": "ISC", - "devDependencies": { - "@types/chai": "^4.1.4", - "@types/jquery": "^3.3.5", - "@types/node": "^10.5.5", - "babel-eslint": "^8.2.6", - "chai": "^4.1.2", - "eslint": "^5.2.0", - "eslint-loader": "^2.1.0", - "eslint-plugin-html": "^4.0.5", - "istanbul": "^0.4.5", - "jquery": "^3.3.1", - "mocha": "^5.2.0" - }, - "dependencies": { - "chalk": "^2.1.0", - "handlebars": "^4.0.10", - "yargs": "^12.0.1" - }, - "_id": "00.demo@1.0.0", - "_npmVersion": "6.2.0", - "_nodeVersion": "8.11.1", - "_npmUser": { - "name": "wanqing3", - "email": "1028431602@qq.com" - }, - "dist": { - "integrity": "sha512-Ke0g9sYsxrP9K9BUXuLP8AvLnixBBtnxxOVLYe5PKBthN332lRRV9lASjawPUokDmchLlwtMKmXDzYbg1rxrnA==", - "shasum": "b2788c6964863010fe783f5cd05aba7459e6643a", - "tarball": "https://registry.npmjs.org/00.demo/-/00.demo-1.0.0.tgz", - "fileCount": 14, - "unpackedSize": 9942, - "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJbbkbCCRA9TVsSAnZWagAAW9oP/3CcgZeY0GI5224BC1Ez\nhSlEUQlgubCryBRw/ZNBZbBw+X81XPCcVI57P3gcJEhlW/7JZQzKxShCb3gf\n9SSpU2R6nJkZEl6JXsLBP40M01EeFz5mmt1UNsj/kMm2zHTVX1mvbwrZ7JPD\nviUG1KpNUSizEscgFdULDMtTy/rffjcO7xp8uKfwgCNT6KsufFoUMygo9l31\nn+kypug391SJZgjmFBqORaK9xM2kg0j6o+1pCCyCG/Tp9+NELAEtafcDncGf\n4cCPf+UqxuRMYgWaSTRPCP6OUFtM0Q07WgPLyrumy5dRC6Hcf3OREHy7gJCO\n50DYecWNM8DwhrpQfKrwzCPYZqi1V0Vp8eX4be+neoKwMHuUHDhevHy6a1sH\nOcJoR5X43bYruOYvE4uPrsaVP2yHBYG8eZeLGBykshQluGD0/VDh4jgCE9rr\nOwI4Zi3tUJiO06g82WaC5dfMWUzHosZAweDr7HpqL+s6jWAOFboJ7wXlA8EZ\nEz8ycymMZVuS9uL0hzOpnjPlYo+j56Auwtk7eGYg8x72yKih0ho5v/mOXcof\nc3x7l111gAG2VoONALIAf23lAwOxYXuPPUXmPRrfr6f9zrFGZrjKfBlsan/F\nZMfnk0GeiF2vwhNObQY/DF45t+GpnAajcxRPqXRf2eV7jF36bjKV61/wOgir\nS73A\r\n=khm/\r\n-----END PGP SIGNATURE-----\r\n" - }, - "maintainers": [ - { - "name": "wanqing3", - "email": "1028431602@qq.com" - } - ], - "directories": {}, - "_npmOperationalInternal": { - "host": "s3://npm-registry-packages", - "tmp": "tmp/00.demo_1.0.0_1533953729435_0.7444299847080198" - }, - "_hasShrinkwrap": false, - "deprecated": "just" - } - }, - "time": { - "created": "2018-08-11T02:15:29.434Z", - "1.0.0": "2018-08-11T02:15:29.790Z", - "modified": "2018-08-26T17:02:03.765Z" - }, - "maintainers": [ - { - "name": "wanqing3", - "email": "1028431602@qq.com" - } - ], - "description": "Tiny NodeJS Static Web server", - "keywords": [], - "license": "ISC", - "readme": "# anydoor\n\nTiny NodeJS Static Web server\n\n## 安装\n\n```\nnpm i -g anydoor\n```\n\n## 使用方法\n\n```\nanydoor # 把当前文件夹作为静态资源服务器根目录\n\nanydoor -p 8080 # 设置端口号为 8080\n\nanydoor -h localhost # 设置 host 为 localhost\n\nanydoor -d /usr # 设置根目录为 /usr\n```\n", - "readmeFilename": "README.md" - } - }, - { - "id": "000-webpack", - "key": "000-webpack", - "value": { - "rev": "1-e0366923eef4091a888c14c3403a6c90" - }, - "doc": { - "_id": "000-webpack", - "_rev": "1-e0366923eef4091a888c14c3403a6c90", - "name": "000-webpack", - "dist-tags": { - "latest": "1.0.0" - }, - "versions": { - "1.0.0": { - "name": "000-webpack", - "version": "1.0.0", - "description": "", - "main": "server.js", - "dependencies": { - "express": "^4.14.1", - "mysql": "^2.13.0" - }, - "devDependencies": { - "babel-core": "^6.18.2", - "babel-loader": "^6.2.8", - "babel-preset-es2015": "^6.18.0", - "babel-preset-react": "^6.16.0", - "react": "^15.4.1", - "react-dom": "^15.4.1", - "webpack": "^2.2.1" - }, - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", - "start": "nodemon ./server.js", - "watch": "webpack -w" - }, - "author": "", - "license": "ISC", - "_id": "000-webpack@1.0.0", - "_shasum": "f1fd0accc02c5be1c76180c7e24b468f9bd68020", - "_from": ".", - "_npmVersion": "3.10.9", - "_nodeVersion": "6.9.2", - "_npmUser": { - "name": "tianzheng", - "email": "1312788455@qq.com" - }, - "dist": { - "shasum": "f1fd0accc02c5be1c76180c7e24b468f9bd68020", - "tarball": "https://registry.npmjs.org/000-webpack/-/000-webpack-1.0.0.tgz" - }, - "maintainers": [ - { - "name": "tianzheng", - "email": "1312788455@qq.com" - } - ], - "_npmOperationalInternal": { - "host": "packages-12-west.internal.npmjs.com", - "tmp": "tmp/000-webpack-1.0.0.tgz_1488356059950_0.42828399664722383" - } - } - }, - "readme": "ERROR: No README data found!", - "maintainers": [ - { - "name": "tianzheng", - "email": "1312788455@qq.com" - } - ], - "time": { - "modified": "2017-03-01T08:14:20.209Z", - "created": "2017-03-01T08:14:20.209Z", - "1.0.0": "2017-03-01T08:14:20.209Z" - }, - "license": "ISC", - "readmeFilename": "" - } - }, - { - "id": "05-feb", - "key": "05-feb", - "value": { - "rev": "1-15a079b0599332cfbde5480150f332a5" - }, - "doc": { - "_id": "05-feb", - "_rev": "1-15a079b0599332cfbde5480150f332a5", - "name": "05-feb", - "description": "this is just a demo", - "dist-tags": { - "latest": "1.0.0" - }, - "versions": { - "1.0.0": { - "name": "05-feb", - "version": "1.0.0", - "description": "this is just a demo", - "main": "first_demo.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": { - "name": "shilpa.mahevaiah" - }, - "license": "ISC", - "dependencies": { - "express": "^4.16.2" - }, - "_id": "05-feb@1.0.0", - "_npmVersion": "5.5.1", - "_nodeVersion": "9.2.0", - "_npmUser": { - "name": "shilpa.mahadevaiah", - "email": "shilpa.mahadevaiah@accenture.com" - }, - "dist": { - "integrity": "sha512-UcGOSYBd5rQ/X1Q2MVquzk5eZ79J4f1pmIosNKKVqp8td/0i1aSeB7jug1A+DO51twgrjU92yIluZO9gKFJW5A==", - "shasum": "bc85483978d45b53ae36a13a1a3016afdb20b258", - "tarball": "https://registry.npmjs.org/05-feb/-/05-feb-1.0.0.tgz" - }, - "maintainers": [ - { - "name": "shilpa.mahadevaiah", - "email": "shilpa.mahadevaiah@accenture.com" - } - ], - "_npmOperationalInternal": { - "host": "s3://npm-registry-packages", - "tmp": "tmp/05-feb-1.0.0.tgz_1517813075453_0.94389269920066" - } - } - }, - "readme": "ERROR: No README data found!", - "maintainers": [ - { - "name": "shilpa.mahadevaiah", - "email": "shilpa.mahadevaiah@accenture.com" - } - ], - "time": { - "modified": "2018-02-05T06:44:36.481Z", - "created": "2018-02-05T06:44:36.481Z", - "1.0.0": "2018-02-05T06:44:36.481Z" - }, - "author": { - "name": "shilpa.mahevaiah" - }, - "license": "ISC", - "readmeFilename": "" - } - }, - { - "id": "05-npm-test", - "key": "05-npm-test", - "value": { - "rev": "2-cd455f17f57bcc529ab85bedaf4a74a1" - }, - "doc": { - "_id": "05-npm-test", - "_rev": "2-cd455f17f57bcc529ab85bedaf4a74a1", - "name": "05-npm-test", - "description": "05-npm-test 2016-0906-1310", - "dist-tags": { - "latest": "1.0.0" - }, - "versions": { - "1.0.0": { - "name": "05-npm-test", - "version": "1.0.0", - "description": "05-npm-test 2016-0906-1310", - "main": "index-2016-0906-1300.js", - "dependencies": { - "lodash": "^4.15.0" - }, - "devDependencies": {}, - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "ISC", - "_id": "05-npm-test@1.0.0", - "_shasum": "03db74ad0c984f4b73662fc9570e0e554a43fe19", - "_from": ".", - "_npmVersion": "3.10.3", - "_nodeVersion": "6.5.0", - "_npmUser": { - "name": "jasonc1025", - "email": "jasonc@eande.world" - }, - "dist": { - "shasum": "03db74ad0c984f4b73662fc9570e0e554a43fe19", - "tarball": "https://registry.npmjs.org/05-npm-test/-/05-npm-test-1.0.0.tgz" - }, - "maintainers": [ - { - "name": "jasonc1025", - "email": "jasonc@eande.world" - } - ], - "_npmOperationalInternal": { - "host": "packages-12-west.internal.npmjs.com", - "tmp": "tmp/05-npm-test-1.0.0.tgz_1473199601887_0.4248608644120395" - } - } - }, - "readme": "ERROR: No README data found!", - "maintainers": [ - { - "name": "jasonc1025", - "email": "jasonc@eande.world" - } - ], - "time": { - "modified": "2016-09-06T22:06:42.114Z", - "created": "2016-09-06T22:06:42.114Z", - "1.0.0": "2016-09-06T22:06:42.114Z" - }, - "license": "ISC", - "readmeFilename": "" - } - }, - { - "id": "05module_package", - "key": "05module_package", - "value": { - "rev": "2-b74c65c358f7211d2867eda1397ff4df" - }, - "doc": { - "_id": "05module_package", - "_rev": "2-b74c65c358f7211d2867eda1397ff4df", - "name": "05module_package", - "description": "my package", - "dist-tags": { - "latest": "1.0.0" - }, - "versions": { - "1.0.0": { - "name": "05module_package", - "version": "1.0.0", - "description": "my package", - "main": "test.js", - "dependencies": { - "express": "^4.14.0" - }, - "devDependencies": {}, - "scripts": { - "test": "node test" - }, - "keywords": [ - "my", - "package", - "test" - ], - "author": { - "name": "guoyang" - }, - "license": "ISC", - "_id": "05module_package@1.0.0", - "_shasum": "c6e30380a0af62c8fd3d99d72bba79a179425a19", - "_from": ".", - "_npmVersion": "2.15.1", - "_nodeVersion": "4.4.3", - "_npmUser": { - "name": "guoyang08", - "email": "zzuligy@163.com" - }, - "dist": { - "shasum": "c6e30380a0af62c8fd3d99d72bba79a179425a19", - "tarball": "https://registry.npmjs.org/05module_package/-/05module_package-1.0.0.tgz" - }, - "maintainers": [ - { - "name": "guoyang08", - "email": "zzuligy@163.com" - } - ], - "_npmOperationalInternal": { - "host": "packages-16-east.internal.npmjs.com", - "tmp": "tmp/05module_package-1.0.0.tgz_1471853459880_0.14190255664288998" - } - } - }, - "readme": "ERROR: No README data found!", - "maintainers": [ - { - "name": "guoyang08", - "email": "zzuligy@163.com" - } - ], - "time": { - "modified": "2016-08-22T08:11:01.407Z", - "created": "2016-08-22T08:11:01.407Z", - "1.0.0": "2016-08-22T08:11:01.407Z" - }, - "keywords": [ - "my", - "package", - "test" - ], - "author": { - "name": "guoyang" - }, - "license": "ISC", - "readmeFilename": "" - } - } - ] -} \ No newline at end of file diff --git a/core/src/test/resources/integration/meta/ruby.gz b/core/src/test/resources/integration/meta/ruby.gz deleted file mode 100644 index d28fe33..0000000 Binary files a/core/src/test/resources/integration/meta/ruby.gz and /dev/null differ diff --git a/core/src/test/resources/integration/meta/rust.zip b/core/src/test/resources/integration/meta/rust.zip deleted file mode 100644 index 1b5c1ac..0000000 Binary files a/core/src/test/resources/integration/meta/rust.zip and /dev/null differ diff --git a/core/src/test/scala/codesearch/core/regex/lexer/TokenizerSpec.scala b/core/src/test/scala/codesearch/core/regex/lexer/TokenizerSpec.scala index 690c6cd..9b8f45a 100644 --- a/core/src/test/scala/codesearch/core/regex/lexer/TokenizerSpec.scala +++ b/core/src/test/scala/codesearch/core/regex/lexer/TokenizerSpec.scala @@ -25,17 +25,21 @@ class TokenizerSpec extends FreeSpec with Matchers { testParseAndRender( "Hello World + ?", - Seq(Literal("Hello"), - Space(" "), - Literal("World"), - Space(" "), - SpecialSymbol("+"), - Space(" "), - SpecialSymbol("?")) + Seq( + Literal("Hello"), + Space(" "), + Literal("World"), + Space(" "), + SpecialSymbol("+"), + Space(" "), + SpecialSymbol("?") + ) ) - testParseAndRender("Hello World [^Gared]", - Seq(Literal("Hello"), Space(" "), Literal("World"), Space(" "), CharSet("[^Gared]"))) + testParseAndRender( + "Hello World [^Gared]", + Seq(Literal("Hello"), Space(" "), Literal("World"), Space(" "), CharSet("[^Gared]")) + ) testParseAndRender( "Hello World [^Gared] (Bale) \\Symbol", @@ -101,12 +105,16 @@ class TokenizerSpec extends FreeSpec with Matchers { testParseAndRender("|", Seq(SpecialSymbol("|"))) testParseAndRender("^", Seq(SpecialSymbol("^"))) testParseAndRender("$", Seq(SpecialSymbol("$"))) - testParseAndRender("ax.,.c", - Seq(Literal("ax"), SpecialSymbol("."), Literal(","), SpecialSymbol("."), Literal("c"))) + testParseAndRender( + "ax.,.c", + Seq(Literal("ax"), SpecialSymbol("."), Literal(","), SpecialSymbol("."), Literal("c")) + ) testParseAndRender("a|^", Seq(Literal("a"), SpecialSymbol("|"), SpecialSymbol("^"))) testParseAndRender("a|b", Seq(Literal("a"), SpecialSymbol("|"), Literal("b"))) - testParseAndRender("(a)|b", - Seq(SpecialSymbol("("), Literal("a"), SpecialSymbol(")"), SpecialSymbol("|"), Literal("b"))) + testParseAndRender( + "(a)|b", + Seq(SpecialSymbol("("), Literal("a"), SpecialSymbol(")"), SpecialSymbol("|"), Literal("b")) + ) testParseAndRender("a*", Seq(Literal("a"), SpecialSymbol("*"))) testParseAndRender("a??", Seq(Literal("a"), SpecialSymbol("?"), SpecialSymbol("?"))) } @@ -121,7 +129,8 @@ class TokenizerSpec extends FreeSpec with Matchers { testParseAndRender("a{2,3}", Seq(Literal("a"), RepetitionSeq("{2,3}"))) testParseAndRender( "a{2, 3}", - Seq(Literal("a"), SpecialSymbol("{"), Literal("2,"), Space(" "), Literal("3"), SpecialSymbol("}"))) + Seq(Literal("a"), SpecialSymbol("{"), Literal("2,"), Space(" "), Literal("3"), SpecialSymbol("}")) + ) testParseAndRender("a{,3}", Seq(Literal("a"), SpecialSymbol("{"), Literal(",3"), SpecialSymbol("}"))) testParseAndRender("a{2,}", Seq(Literal("a"), RepetitionSeq("{2,}"))) testParseAndRender("a{,}", Seq(Literal("a"), SpecialSymbol("{"), Literal(","), SpecialSymbol("}"))) @@ -166,7 +175,9 @@ class TokenizerSpec extends FreeSpec with Matchers { "RoundTrip cases" - { val cases = Source.fromResource("regex/cases.txt").getLines - cases.foreach(caseString => caseString shouldBe roundTrip(caseString)) + cases.foreach { caseString => + caseString shouldBe roundTrip(caseString) + } } } diff --git a/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala b/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala index 14152a8..fe24da7 100644 --- a/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala +++ b/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala @@ -14,7 +14,7 @@ class SnippetsGrouperSpec extends WordSpec with Matchers { "3models/0.3.0/Graphics/Model/DirectX.hs:14:import Data.Attoparsec.ByteString.Char8 as A", "3models/0.3.0/Graphics/Model/DirectX.hs:15:import qualified Data.ByteString as B", "3models/0.3.0/Graphics/Model/DirectX.hs:16:import Data.Traversable", - "3models/0.3.0/Graphics/Model/DirectX.hs:17:import Data.Word", + "3models/0.3.0/Graphics/Model/DirectX.hs:17:import Data.Word" ) val snippets: List[SnippetInfo] = fs2.Stream @@ -37,7 +37,7 @@ class SnippetsGrouperSpec extends WordSpec with Matchers { val matchedLines = Seq( "3models/0.3.0/Graphics/Model/DirectX.hs:28:} deriving Show", - "3models/0.3.0/Graphics/Model/DirectX.hs:39:} deriving Show", + "3models/0.3.0/Graphics/Model/DirectX.hs:39:} deriving Show" ) val snippets: List[SnippetInfo] = fs2.Stream @@ -64,7 +64,7 @@ class SnippetsGrouperSpec extends WordSpec with Matchers { val matchedLines = Seq( "3models/0.3.0/Graphics/Model/DirectX.hs:14:import Data.Attoparsec.ByteString.Char8 as A", - "3models/0.3.0/Graphics/Model/Obj.hs:16:import Data.Attoparsec.ByteString.Char8 as A", + "3models/0.3.0/Graphics/Model/Obj.hs:16:import Data.Attoparsec.ByteString.Char8 as A" ) val snippets: List[SnippetInfo] = fs2.Stream diff --git a/core/src/test/scala/integration/IntegrationHaskellSpec.scala b/core/src/test/scala/integration/IntegrationHaskellSpec.scala deleted file mode 100644 index 48368db..0000000 --- a/core/src/test/scala/integration/IntegrationHaskellSpec.scala +++ /dev/null @@ -1,109 +0,0 @@ -package integration - -import java.nio.file.Paths - -import cats.data.NonEmptyVector -import cats.effect.IO -import codesearch.core.index._ -import codesearch.core.index.directory.HaskellCindex -import codesearch.core.index.repository.Downloader -import codesearch.core.meta._ -import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} -import codesearch.core.search.{HaskellSearch, Search, SearchRequest} -import codesearch.core.util.Unarchiver -import integration.fakes.FakeDownloader -import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} - -class IntegrationHaskellSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { - - override val container = PostgreSQLContainer() - val haskellCindex = HaskellCindex(Paths.get("./index/test/cindex/")) - val searcher: Search = new HaskellSearch(haskellCindex) - - "Integration Haskell Spec" in new TestFixture { - - httpClient.use { implicit backend => - implicit val downloader: Downloader[IO] = Downloader.create[IO] - val hackageDownloader: FakeDownloader[IO] = FakeDownloader[IO](getMetaData("integration/meta/haskell.tar.gz")) - val unarchiver = Unarchiver[IO] - val haskellIndex = HaskellIndex(config, database, haskellCindex) - - for { - hackageMeta <- HackageMetaDownloader(config.languagesConfig.haskell, unarchiver, hackageDownloader) - _ <- haskellIndex.initDB - _ <- hackageMeta.downloadMeta - _ <- haskellIndex.updatePackages(Some(20)) - _ <- haskellIndex.buildIndex - } yield () - }.unsafeRunSync() - - searchResultsMustBe( - SearchRequest( - lang = "haskell", - query = "Tupel", - filter = None, - filePath = None, - insensitive = false, - spaceInsensitive = false, - preciseMatch = false, - sourcesOnly = false, - page = 1 - ), - 1, - Seq( - PackageResult( - Package("3d-graphics-examples-0.0.0.2", "https://hackage.haskell.org/package/3d-graphics-examples-0.0.0.2"), - Seq( - CodeSnippet( - "src/mountains/Mountains.hs", - "hackage/3d-graphics-examples/0.0.0.2/src/mountains/Mountains.hs", - 32, - NonEmptyVector.of(36), - Seq( - "", - "", - "type Pair t = (t, t)", - "type Tupel3 t = (t, t, t)", - "", - "data TerrainDrawMode = TerrainPoints | TerrainWireframe | TerrainSolid", - " deriving (Eq, Show)", - "", - "data Distribution = UniformDistribution | NormalDistribution" - ) - ) - ) - ) - ) - ) - - searchResultsMustBe( - SearchRequest( - lang = "haskell", - query = "#!/usr.*runghc", - filter = None, - filePath = Some(".*hs"), - insensitive = false, - spaceInsensitive = false, - preciseMatch = false, - sourcesOnly = true, - page = 1 - ), - 1, - Seq( - PackageResult( - Package("3d-graphics-examples-0.0.0.2", "https://hackage.haskell.org/package/3d-graphics-examples-0.0.0.2"), - Seq( - CodeSnippet( - "Setup.lhs", - "hackage/3d-graphics-examples/0.0.0.2/Setup.lhs", - 0, - NonEmptyVector.of(1), - Seq("#!/usr/bin/env runghc", "", "> import Distribution.Simple", "> main = defaultMain") - ) - ) - ) - ) - ) - } -} diff --git a/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala b/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala deleted file mode 100644 index 66a53fc..0000000 --- a/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala +++ /dev/null @@ -1,121 +0,0 @@ -package integration - -import java.nio.file.Paths - -import cats.data.NonEmptyVector -import cats.effect.IO -import codesearch.core.index._ -import codesearch.core.index.directory.JavaScriptCindex -import codesearch.core.index.repository.Downloader -import codesearch.core.meta._ -import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} -import codesearch.core.search.{JavaScriptSearch, Search, SearchRequest} -import codesearch.core.util.Unarchiver -import integration.fakes.FakeDownloader -import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} - -class IntegrationJavaScriptSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { - - override val container = PostgreSQLContainer() - val javaScriptCindex = JavaScriptCindex(Paths.get("./index/test/cindex/")) - val searcher: Search = new JavaScriptSearch(javaScriptCindex) - - "Integration JavaScript Spec" in new TestFixture { - - httpClient.use { implicit backend => - implicit val downloader: Downloader[IO] = Downloader.create[IO] - val npmDownloader: FakeDownloader[IO] = FakeDownloader[IO](getMetaData("integration/meta/node.json")) - val unarchiver = Unarchiver[IO] - val nodeIndex = JavaScriptIndex(config, database, javaScriptCindex) - - for { - hackageMeta <- NpmMetaDownloader(config.languagesConfig.javascript, npmDownloader) - _ <- nodeIndex.initDB - _ <- hackageMeta.downloadMeta - _ <- nodeIndex.updatePackages(Some(15)) - _ <- nodeIndex.buildIndex - } yield () - }.unsafeRunSync() - - searchResultsMustBe( - SearchRequest( - lang = "javascript", - query = "* > dom", - filter = None, - filePath = None, - insensitive = true, - spaceInsensitive = true, - preciseMatch = true, - sourcesOnly = true, - page = 1 - ), - 2, - Seq( - PackageResult( - Package("000-webpack-1.0.0", "https://www.npmjs.com/package/000-webpack/v/1.0.0"), - Seq( - CodeSnippet( - "www/public/js/index.js", - "npm/000-webpack/1.0.0/www/public/js/index.js", - 1882, - NonEmptyVector.of(1886, 1888), - Seq( - "/**", - " * DOMProperty exports lookup objects that can be used like functions:", - " *", - " * > DOMProperty.isValid['id']", - " * true", - " * > DOMProperty.isValid['foobar']", - " * undefined", - " *", - " * Although this may be confusing, it performs better in general.", - " *", - " * @see http://jsperf.com/key-exists" - ) - ) - ) - ) - ) - ) - - searchResultsMustBe( - SearchRequest( - lang = "javascript", - query = "import styled", - filter = Some("css"), - filePath = None, - insensitive = false, - spaceInsensitive = false, - preciseMatch = true, - sourcesOnly = true, - page = 1 - ), - 1, - Seq( - PackageResult( - Package("00-components-0.5.0", "https://www.npmjs.com/package/00-components/v/0.5.0"), - Seq( - CodeSnippet( - "dist/Button.js", - "npm/00-components/0.5.0/dist/Button.js", - 19, - NonEmptyVector.of(23), - Seq( - " return data;", - "}", - "", - "import styled, { css } from \"styled-components\";", - "import defaultTheme from \"./theme/default\";", - "var Button = styled.a(_templateObject(), function (_ref) {", - " var theme = _ref.theme;", - " return theme.fontFamily;", - "}, function (props) {" - ) - ) - ) - ) - ) - ) - } -} diff --git a/core/src/test/scala/integration/IntegrationRubySpec.scala b/core/src/test/scala/integration/IntegrationRubySpec.scala deleted file mode 100644 index 2d0b48e..0000000 --- a/core/src/test/scala/integration/IntegrationRubySpec.scala +++ /dev/null @@ -1,134 +0,0 @@ -package integration - -import java.nio.file.Paths - -import cats.data.NonEmptyVector -import cats.effect.IO -import codesearch.core.index._ -import codesearch.core.index.directory.RubyCindex -import codesearch.core.index.repository.Downloader -import codesearch.core.meta._ -import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} -import codesearch.core.search.{RubySearch, Search, SearchRequest} -import codesearch.core.util.Unarchiver -import integration.fakes.FakeDownloader -import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} - -class IntegrationRubySpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { - - override val container = PostgreSQLContainer() - val rubyCindex = RubyCindex(Paths.get("./index/test/cindex/")) - val searcher: Search = new RubySearch(rubyCindex) - - "Integration Ruby Spec" in new TestFixture { - - httpClient.use { implicit backend => - implicit val downloader: Downloader[IO] = Downloader.create[IO] - val gemDownloader: FakeDownloader[IO] = FakeDownloader[IO](getMetaData("integration/meta/ruby.gz")) - val unarchiver = Unarchiver[IO] - val rubyIndex = RubyIndex(config, database, rubyCindex) - - for { - gemMeta <- GemMetaDownloader(config.languagesConfig.ruby, gemDownloader) - _ <- rubyIndex.initDB - _ <- gemMeta.downloadMeta - _ <- rubyIndex.updatePackages(Some(14)) - _ <- rubyIndex.buildIndex - } yield () - }.unsafeRunSync() - - searchResultsMustBe( - SearchRequest( - lang = "ruby", - query = "class flex4sdk", - filter = None, - filePath = None, - insensitive = true, - spaceInsensitive = false, - preciseMatch = false, - sourcesOnly = false, - page = 1 - ), - 1, - Seq( - PackageResult( - Package("sprout-flex4sdk-tool-4.2.14", "https://rubygems.org/gems/sprout-flex4sdk-tool/versions/4.2.14"), - Seq( - CodeSnippet( - "lib/sprout/flex4sdk/version.rb", - "gem/sprout-flex4sdk-tool/4.2.14/lib/sprout/flex4sdk/version.rb", - 0, - NonEmptyVector.of(2), - Seq( - "module Sprout", - " class Flex4SDK #:nodoc:", - " module VERSION #:nodoc:", - " MAJOR = 4", - " MINOR = 2", - " TINY = 14", - "" - ) - ) - ) - ) - ) - ) - - searchResultsMustBe( - SearchRequest( - lang = "ruby", - query = "fest", - filter = None, - filePath = None, - insensitive = false, - spaceInsensitive = false, - preciseMatch = false, - sourcesOnly = false, - page = 1 - ), - 2, - Seq( - PackageResult( - Package("sprout-flex4-bundle-0.1.4", "https://rubygems.org/gems/sprout-flex4-bundle/versions/0.1.4"), - Seq( - CodeSnippet( - "lib/sprout/generators/component/component_generator.rb", - "gem/sprout-flex4-bundle/0.1.4/lib/sprout/generators/component/component_generator.rb", - 0, - NonEmptyVector.of(4), - Seq( - "", - "class ComponentGenerator < Sprout::Generator::NamedBase # :nodoc:", - "", - " def manifest", - " record do |m|", - " if(!user_requested_test)", - " m.directory full_class_dir", - " m.template 'Component.mxml', full_class_path.gsub(/.as$/, '.mxml')", - " end" - ) - ), - CodeSnippet( - "lib/sprout/generators/project/project_generator.rb", - "gem/sprout-flex4-bundle/0.1.4/lib/sprout/generators/project/project_generator.rb", - 0, - NonEmptyVector.of(4), - Seq( - "", - "class ProjectGenerator < Sprout::Generator::NamedBase # :nodoc:", - "", - " def manifest", - " record do |m|", - " base = class_name", - " m.directory base", - " m.directory File.join(base, 'assets/skins', project_name)", - " m.directory File.join(base, 'bin')" - ) - ) - ) - ) - ) - ) - } -} diff --git a/core/src/test/scala/integration/IntegrationRustSpec.scala b/core/src/test/scala/integration/IntegrationRustSpec.scala deleted file mode 100644 index ec96a9e..0000000 --- a/core/src/test/scala/integration/IntegrationRustSpec.scala +++ /dev/null @@ -1,136 +0,0 @@ -package integration - -import java.nio.file.Paths - -import cats.data.NonEmptyVector -import cats.effect.IO -import codesearch.core.index._ -import codesearch.core.index.directory.RustCindex -import codesearch.core.index.repository.Downloader -import codesearch.core.meta._ -import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} -import codesearch.core.search.{RustSearch, Search, SearchRequest} -import codesearch.core.util.Unarchiver -import integration.fakes.FakeDownloader -import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} - -class IntegrationRustSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { - - override val container = PostgreSQLContainer() - val rustCindex = RustCindex(Paths.get("./index/test/cindex/")) - val searcher: Search = new RustSearch(rustCindex) - - "Integration Rust Spec" in new TestFixture { - - httpClient.use { implicit backend => - implicit val downloader: Downloader[IO] = Downloader.create[IO] - val cratesDownloader: FakeDownloader[IO] = FakeDownloader[IO](getMetaData("integration/meta/rust.zip")) - val unarchiver = Unarchiver[IO] - val rustIndex = RustIndex(config, database, rustCindex) - - for { - hackageMeta <- CratesMetaDownloader(config.languagesConfig.rust, unarchiver, cratesDownloader) - _ <- rustIndex.initDB - _ <- hackageMeta.downloadMeta - _ <- rustIndex.updatePackages(Some(15)) - _ <- rustIndex.buildIndex - } yield () - }.unsafeRunSync() - - searchResultsMustBe( - SearchRequest( - lang = "rust", - query = "#[derive", - filter = Some("ErrorKind"), - filePath = Some("lapin-futures-tls-internal.*6"), - insensitive = true, - spaceInsensitive = false, - preciseMatch = true, - sourcesOnly = false, - page = 1 - ), - 2, - Seq( - PackageResult( - Package("lapin-futures-tls-internal-0.6.0", "https://docs.rs/crate/lapin-futures-tls-internal/0.6.0"), - Seq( - CodeSnippet( - "src/error.rs", - "crates/lapin-futures-tls-internal/0.6.0/src/error.rs", - 10, - NonEmptyVector.of(14), - Seq( - "/// means that this type guaranteed to be both sendable and usable across", - "/// threads, and that you'll be able to use the downcasting feature of the", - "/// `failure::Error` type.", - "#[derive(Debug)]", - "pub struct Error {", - " inner: Context,", - "}", - "", - "/// The different kinds of errors that can be reported." - ) - ), - CodeSnippet( - "src/error.rs", - "crates/lapin-futures-tls-internal/0.6.0/src/error.rs", - 19, - NonEmptyVector.of(23), - Seq( - "///", - "/// Even though we expose the complete enumeration of possible error variants, it is not", - "/// considered stable to exhaustively match on this enumeration: do it at your own risk.", - "#[derive(Debug, Fail)]", - "pub enum ErrorKind {", - " /// Failure to parse an Uri", - " #[fail(display = \"Uri parsing error: {:?}\", _0)]", - " UriParsingError(String),", - " /// Failure to resolve a domain name" - ) - ) - ) - ) - ) - ) - - searchResultsMustBe( - SearchRequest( - lang = "rust", - query = "Box::new(AMQP", - filter = Some("AMQPFrame"), - filePath = None, - insensitive = true, - spaceInsensitive = false, - preciseMatch = true, - sourcesOnly = true, - page = 1 - ), - 1, - Seq( - PackageResult( - Package("lapin-futures-0.17.0", "https://docs.rs/crate/lapin-futures/0.17.0"), - Seq( - CodeSnippet( - "src/transport.rs", - "crates/lapin-futures/0.17.0/src/transport.rs", - 337, - NonEmptyVector.of(341), - Seq( - "", - " let mut codec = AMQPCodec { frame_max: 8192 };", - " let mut buffer = BytesMut::with_capacity(8192);", - " let frame = AMQPFrame::Header(0, 10, Box::new(AMQPContentHeader {", - " class_id: 10,", - " weight: 0,", - " body_size: 64,", - " properties: BasicProperties::default()", - " }));" - ) - ) - ) - ) - ) - ) - } -} diff --git a/core/src/test/scala/integration/IntegrationSpecBase.scala b/core/src/test/scala/integration/IntegrationSpecBase.scala deleted file mode 100644 index be38f0a..0000000 --- a/core/src/test/scala/integration/IntegrationSpecBase.scala +++ /dev/null @@ -1,27 +0,0 @@ -package integration - -import codesearch.core.search.Search.PackageResult -import codesearch.core.search.{Search, SearchRequest} -import slick.jdbc.PostgresProfile.api._ -import com.dimafeng.testcontainers.PostgreSQLContainer -import org.scalatest.Matchers - -trait IntegrationSpecBase extends Matchers { - - val container: PostgreSQLContainer - val searcher: Search - - def database = Database.forURL( - driver = "org.postgresql.Driver", - url = s"${container.jdbcUrl}?user=${container.username}&password=${container.password}" - ) - - def searchResultsMustBe(lookFor: SearchRequest, totalResults: Int, result: Seq[PackageResult]): Unit = { - val searchResult = searcher - .search(lookFor) - .unsafeRunSync() - - searchResult.total shouldBe totalResults - searchResult.data shouldBe result - } -} diff --git a/core/src/test/scala/integration/TestFixture.scala b/core/src/test/scala/integration/TestFixture.scala deleted file mode 100644 index 50cd9d9..0000000 --- a/core/src/test/scala/integration/TestFixture.scala +++ /dev/null @@ -1,71 +0,0 @@ -package integration - -import java.net.URI -import java.nio.ByteBuffer -import java.nio.file.Paths - -import cats.effect.{IO, Resource} -import codesearch.core.{BlockingEC, config} -import codesearch.core.config._ -import com.softwaremill.sttp.SttpBackend -import com.softwaremill.sttp.asynchttpclient.fs2.AsyncHttpClientFs2Backend -import fs2.Stream -import fs2.io.file - -import scala.concurrent.ExecutionContext.Implicits.global - -trait TestFixture { - - implicit val cs = IO.contextShift(global) - - def httpClient: Resource[IO, SttpBackend[IO, Stream[IO, ByteBuffer]]] = - Resource.make(IO(AsyncHttpClientFs2Backend[IO]()))(client => IO(client.close())) - - def getMetaData(path: String): Stream[IO, Byte] = { - val resource = Paths.get(getClass.getClassLoader.getResource(path).toURI) - file.readAll[IO](resource, BlockingEC, 4096) - } - - def config = Config( - DatabaseConfig( - dataSourceClass = "slick.jdbc.DatabaseUrlDataSource", - host = "localhost", - port = 5432, - name = "sourcesdb", - user = "postgres", - password = "postgres", - ), - SnippetConfig( - pageSize = 30, - linesBefore = 3, - linesAfter = 5 - ), - LanguagesConfig( - HaskellConfig( - repoIndexUrl = new URI("http://hackage.haskell.org/packages/index.tar.gz"), - repoArchivePath = Paths.get("./data/test/meta/haskell/index.tar.gz"), - repoPath = Paths.get("./data/test/meta/haskell/"), - concurrentTasksCount = 30 - ), - RubyConfig( - repoIndexUrl = new URI("http://rubygems.org/latest_specs.4.8.gz"), - repoArchivePath = Paths.get("./data/test/meta/ruby/ruby_index.gz"), - repoJsonPath = Paths.get("./data/test/meta/ruby/ruby_index.json"), - scriptPath = Paths.get("./scripts/update_index.rb"), - concurrentTasksCount = 30 - ), - RustConfig( - repoIndexUrl = new URI("https://github.com/rust-lang/crates.io-index/archive/master.zip"), - repoArchivePath = Paths.get("./data/test/meta/rust/archive.zip"), - repoPath = Paths.get("./data/test/meta/rust/"), - concurrentTasksCount = 30 - ), - JavaScriptConfig( - repoIndexUrl = new URI("https://replicate.npmjs.com/_all_docs?include_docs=true"), - repoJsonPath = Paths.get("./data/test/meta/npm/npm_packages_index.json"), - concurrentTasksCount = 30 - ) - ), - MetricsConfig(false) - ) -} diff --git a/core/src/test/scala/integration/fakes/FakeDownloader.scala b/core/src/test/scala/integration/fakes/FakeDownloader.scala deleted file mode 100644 index 1c7ca0f..0000000 --- a/core/src/test/scala/integration/fakes/FakeDownloader.scala +++ /dev/null @@ -1,32 +0,0 @@ -package integration.fakes - -import java.nio.file.Path -import java.nio.file.StandardOpenOption.{CREATE, TRUNCATE_EXISTING} - -import cats.effect.{ContextShift, Sync} -import cats.syntax.flatMap._ -import cats.syntax.functor._ -import codesearch.core.BlockingEC -import codesearch.core.index.repository.Downloader -import com.softwaremill.sttp.Uri -import fs2.Stream -import fs2.io.file - -case class FakeDownloader[F[_]: Sync: ContextShift](data: Stream[F, Byte]) extends Downloader[F] { - - /** - * Function download sources from remote resource and save in file. - * - * @param from is uri of remote resource. - * @param to is path for downloaded file. If the file does not exist for this path will be create. - * @return downloaded archive file with sources. - */ - def download(from: Uri, to: Path): F[Path] = - Sync[F].delay(to.getParent.toFile.mkdirs) >> data - .through(file.writeAll(to, BlockingEC, List(CREATE, TRUNCATE_EXISTING))) - .compile - .drain - .as(to) - - def download(from: Uri): Stream[F, Byte] = data -} diff --git a/docker/ci-test-env/Dockerfile b/docker/ci-test-env/Dockerfile deleted file mode 100644 index 66d7f32..0000000 --- a/docker/ci-test-env/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM codestar/circleci-scala-sbt-git:scala-2.12.6-sbt-1.1.6 - -RUN apk --no-cache add \ - make \ - nodejs \ - nodejs-npm \ - go \ - musl-dev \ - curl \ - wget \ - ruby \ - ruby-json - - -RUN go get github.com/aelve/codesearch-engine/cmd/... -RUN rm -rf /var/cache/apk/* - -ENV PATH /root/go/bin:$PATH \ No newline at end of file diff --git a/project/Builder.scala b/project/Builder.scala index 0c1f677..9a60076 100644 --- a/project/Builder.scala +++ b/project/Builder.scala @@ -50,7 +50,8 @@ object Builder { "org.codehaus.janino" % "janino" % "3.0.11", "com.typesafe.play" %% "play-json" % "2.6.9", "com.github.mpilquist" %% "simulacrum" % "0.13.0", - "org.typelevel" %% "cats-core" % "1.2.0" + "org.typelevel" %% "cats-core" % "1.2.0", + "com.google.re2j" % "re2j" % "1.2" ) ) @@ -61,8 +62,6 @@ object Builder { assemblyJarName in assembly := "codesearch-core.jar", assemblyOutputPath in assembly := baseDirectory.value / "../codesearch-core.jar", libraryDependencies ++= Seq( - "org.testcontainers" % "postgresql" % "1.11.1", - "com.dimafeng" %% "testcontainers-scala" % "0.23.0" % "test", "com.typesafe.slick" %% "slick" % "3.2.3", "com.typesafe.slick" %% "slick-hikaricp" % "3.2.3", "org.postgresql" % "postgresql" % "42.2.2", diff --git a/web-server/app/codesearch/web/controllers/Application.scala b/web-server/app/codesearch/web/controllers/Application.scala index 2926820..d68461c 100644 --- a/web-server/app/codesearch/web/controllers/Application.scala +++ b/web-server/app/codesearch/web/controllers/Application.scala @@ -4,7 +4,6 @@ import codesearch.core.db.{CratesDB, GemDB, HackageDB, NpmDB} import com.github.marlonlom.utilities.timeago.TimeAgo import javax.inject.Inject import play.api.mvc.{Action, AnyContent, InjectedController} -import slick.jdbc.PostgresProfile.api._ import scala.concurrent.ExecutionContext @@ -16,13 +15,6 @@ class Application @Inject()( implicit val executionContext: ExecutionContext ) extends InjectedController { - val database: Database = Database.forConfig("db") - - val HackageDB: HackageDB = new HackageDB { val db: Database = database } - val CratesDB: CratesDB = new CratesDB { val db: Database = database } - val NpmDB: NpmDB = new NpmDB { val db: Database = database } - val GemDB: GemDB = new GemDB { val db: Database = database } - def index: Action[AnyContent] = Action.async { implicit request => HackageDB.updated .zip(HackageDB.getSize) @@ -30,15 +22,18 @@ class Application @Inject()( .zip(NpmDB.updated.zip(NpmDB.getSize)) .zip(GemDB.updated.zip(GemDB.getSize)) .map { - case ((((updatedHackage, sizeHackage), (updatedCrates, sizeCrates)), (updatedNpm, sizeNpm)), - (updatedGem, sizeGem)) => + case ( + (((updatedHackage, sizeHackage), (updatedCrates, sizeCrates)), (updatedNpm, sizeNpm)), + (updatedGem, sizeGem) + ) => Ok( views.html.index( LangInfo(updatedHackage.getTime, sizeHackage), LangInfo(updatedCrates.getTime, sizeCrates), LangInfo(updatedGem.getTime, sizeGem), LangInfo(updatedNpm.getTime, sizeNpm) - )) + ) + ) } } diff --git a/web-server/app/codesearch/web/controllers/CratesSearcher.scala b/web-server/app/codesearch/web/controllers/CratesSearcher.scala index c032cae..1b15a4e 100644 --- a/web-server/app/codesearch/web/controllers/CratesSearcher.scala +++ b/web-server/app/codesearch/web/controllers/CratesSearcher.scala @@ -1,9 +1,6 @@ package codesearch.web.controllers -import java.nio.file.Paths - import codesearch.core.db.{CratesDB, DefaultDB} -import codesearch.core.index.directory.RustCindex import codesearch.core.model.CratesTable import codesearch.core.search.RustSearch import javax.inject.Inject @@ -14,7 +11,7 @@ import scala.concurrent.ExecutionContext class CratesSearcher @Inject()( implicit override val executionContext: ExecutionContext ) extends InjectedController with SearchController[CratesTable] { - override def db: DefaultDB[CratesTable] = new CratesDB { val db = database } - override lazy val searchEngine: RustSearch = new RustSearch(RustCindex(Paths.get("./index/cindex/"))) + override def db: DefaultDB[CratesTable] = CratesDB + override lazy val searchEngine: RustSearch = new RustSearch() override def lang: String = "rust" } diff --git a/web-server/app/codesearch/web/controllers/GemSearcher.scala b/web-server/app/codesearch/web/controllers/GemSearcher.scala index f926ef1..1719393 100644 --- a/web-server/app/codesearch/web/controllers/GemSearcher.scala +++ b/web-server/app/codesearch/web/controllers/GemSearcher.scala @@ -1,9 +1,6 @@ package codesearch.web.controllers -import java.nio.file.Paths - import codesearch.core.db.{DefaultDB, GemDB} -import codesearch.core.index.directory.RubyCindex import codesearch.core.model.GemTable import codesearch.core.search.RubySearch import javax.inject.Inject @@ -14,8 +11,8 @@ import scala.concurrent.ExecutionContext class GemSearcher @Inject()( implicit override val executionContext: ExecutionContext ) extends InjectedController with SearchController[GemTable] { - override def db: DefaultDB[GemTable] = new GemDB { val db = database } - override lazy val searchEngine: RubySearch = new RubySearch(RubyCindex(Paths.get("./index/cindex/"))) + override def db: DefaultDB[GemTable] = GemDB + override lazy val searchEngine: RubySearch = new RubySearch() override def lang: String = "ruby" } diff --git a/web-server/app/codesearch/web/controllers/HackageSearcher.scala b/web-server/app/codesearch/web/controllers/HackageSearcher.scala index a91f4cf..ac88ea7 100644 --- a/web-server/app/codesearch/web/controllers/HackageSearcher.scala +++ b/web-server/app/codesearch/web/controllers/HackageSearcher.scala @@ -1,9 +1,6 @@ package codesearch.web.controllers -import java.nio.file.Paths - import codesearch.core.db.{DefaultDB, HackageDB} -import codesearch.core.index.directory.HaskellCindex import codesearch.core.model.HackageTable import codesearch.core.search.HaskellSearch import javax.inject.Inject @@ -14,7 +11,7 @@ import scala.concurrent.ExecutionContext class HackageSearcher @Inject()( implicit override val executionContext: ExecutionContext ) extends InjectedController with SearchController[HackageTable] { - override def db: DefaultDB[HackageTable] = new HackageDB { val db = database } - override lazy val searchEngine: HaskellSearch = new HaskellSearch(HaskellCindex(Paths.get("./index/cindex/"))) + override def db: DefaultDB[HackageTable] = HackageDB + override lazy val searchEngine: HaskellSearch = new HaskellSearch() override def lang: String = "haskell" } diff --git a/web-server/app/codesearch/web/controllers/NpmSearcher.scala b/web-server/app/codesearch/web/controllers/NpmSearcher.scala index 39cf278..cac5a9c 100644 --- a/web-server/app/codesearch/web/controllers/NpmSearcher.scala +++ b/web-server/app/codesearch/web/controllers/NpmSearcher.scala @@ -1,9 +1,6 @@ package codesearch.web.controllers -import java.nio.file.Paths - import codesearch.core.db.{DefaultDB, NpmDB} -import codesearch.core.index.directory.JavaScriptCindex import codesearch.core.model.NpmTable import codesearch.core.search.JavaScriptSearch import javax.inject.Inject @@ -14,8 +11,7 @@ import scala.concurrent.ExecutionContext class NpmSearcher @Inject()( implicit override val executionContext: ExecutionContext ) extends InjectedController with SearchController[NpmTable] { - override def db: DefaultDB[NpmTable] = new NpmDB { val db = database } - override lazy val searchEngine: JavaScriptSearch = new JavaScriptSearch( - JavaScriptCindex(Paths.get("./index/cindex/"))) - override def lang: String = "js" + override def db: DefaultDB[NpmTable] = NpmDB + override lazy val searchEngine: JavaScriptSearch = new JavaScriptSearch() + override def lang: String = "js" } diff --git a/web-server/app/codesearch/web/controllers/SearchController.scala b/web-server/app/codesearch/web/controllers/SearchController.scala index 2f2dcd1..5ffb6a3 100644 --- a/web-server/app/codesearch/web/controllers/SearchController.scala +++ b/web-server/app/codesearch/web/controllers/SearchController.scala @@ -5,12 +5,10 @@ import cats.instances.future._ import codesearch.core.db.DefaultDB import codesearch.core.index.directory.Directory import codesearch.core.model.DefaultTable -import codesearch.core.search.Search.CSearchPage -import codesearch.core.search.{Search, SearchRequest} +import codesearch.core.search._ import codesearch.core.util.Helper import com.github.marlonlom.utilities.timeago.TimeAgo import play.api.mvc.{Action, AnyContent, InjectedController} -import slick.jdbc.PostgresProfile.api._ import scala.concurrent.{ExecutionContext, Future} @@ -23,8 +21,6 @@ trait SearchController[V <: DefaultTable] { self: InjectedController => def searchEngine: Search def lang: String - val database = Database.forConfig("db") - def index: Action[AnyContent] = Action.async { implicit request => db.updated.map( updated => @@ -57,21 +53,27 @@ trait SearchController[V <: DefaultTable] { self: InjectedController => case CSearchPage(results, total) => Ok( views.html.searchResults( - updated = TimeAgo.using(updated.getTime), - packages = results, - query = searchRequest.query, - filter = searchRequest.filter, - filePath = searchRequest.filePath, - insensitive = searchRequest.insensitive, - space = searchRequest.spaceInsensitive, - precise = searchRequest.preciseMatch, - sources = searchRequest.sourcesOnly, - page = searchRequest.page, - totalMatches = total, - callURI = searchRequest.callURI(host).toString, - lang = lang + response = SuccessResponse( + updated = TimeAgo.using(updated.getTime), + packages = results, + query = searchRequest.query, + filter = searchRequest.filter, + filePath = searchRequest.filePath, + insensitive = searchRequest.insensitive, + space = searchRequest.spaceInsensitive, + precise = searchRequest.preciseMatch, + sources = searchRequest.sourcesOnly, + page = searchRequest.page, + totalMatches = total, + callURI = searchRequest.callURI(host).toString, + lang = lang + ) ) ) + case er @ ErrorResponse(_) => + Ok( + views.html.searchResults(er) + ) } unsafeToFuture } } diff --git a/web-server/app/views/resultFrame.scala.html b/web-server/app/views/resultFrame.scala.html index 2e9de4d..df1af1a 100644 --- a/web-server/app/views/resultFrame.scala.html +++ b/web-server/app/views/resultFrame.scala.html @@ -1,13 +1,10 @@ +@import java.net.URLEncoder + @import codesearch.core.regex.RegexConstructor -@import codesearch.core.util.Helper -@import scala.collection.mutable -@import scala.collection.mutable.ListBuffer -@import scala.collection.immutable -@import play.twirl.api._ @import codesearch.core.search.Search.PackageResult +@import codesearch.core.util.Helper @import codesearch.web.SnippetHelper - -@import java.net.URLEncoder +@import play.twirl.api._ @( lang: String, insensitive: Boolean, diff --git a/web-server/app/views/searchBox.scala.html b/web-server/app/views/searchBox.scala.html index 6eb530e..96f4367 100644 --- a/web-server/app/views/searchBox.scala.html +++ b/web-server/app/views/searchBox.scala.html @@ -1,13 +1,5 @@ -@( - actSearch: String, - query: String, - filter: Option[String], - filePath: Option[String], - insensitive: Boolean, - space: Boolean, - precise: Boolean, - sources: Boolean -) +@(actSearch: String, query: String, filter: Option[String], filePath: Option[String], insensitive: Boolean, + space: Boolean, precise: Boolean, sources: Boolean)
diff --git a/web-server/app/views/searchBoxError.scala.html b/web-server/app/views/searchBoxError.scala.html new file mode 100644 index 0000000..a747104 --- /dev/null +++ b/web-server/app/views/searchBoxError.scala.html @@ -0,0 +1,107 @@ +@(errorMessageT: String) + +
+
+
+
Please provide a string to search for.
+ @if(errorMessageT.nonEmpty) { +
️⚠️ @{errorMessageT}
+ } +
+ +
+ +
+
+
3+ characters are required.
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ diff --git a/web-server/app/views/searchResults.scala.html b/web-server/app/views/searchResults.scala.html index c949666..4339d7b 100644 --- a/web-server/app/views/searchResults.scala.html +++ b/web-server/app/views/searchResults.scala.html @@ -1,29 +1,25 @@ -@import codesearch.core.search.Search.PackageResult - +@import codesearch.core.search.{ErrorResponse, Response, SuccessResponse} @( - updated: String, - packages: Seq[PackageResult], - query: String, - filter: Option[String], - filePath: Option[String], - insensitive: Boolean, - space: Boolean, - precise: Boolean, - sources: Boolean, - page: Int, - totalMatches: Int, - callURI: String, - lang: String + response: Response ) - @headExtra = { - - - - } +@headExtra = { + + + +} - @wrapper(s"Codesearch | $lang", headExtra) { - @searchBox(s"/$lang/search", query, filter, filePath, insensitive, space, precise, sources) - @resultFrame(lang, insensitive, space, precise, query, updated, packages, totalMatches) - @pagination(page, totalMatches, callURI) +@response match { + case SuccessResponse(updated, packages, query, filter, filePath, insensitive, space, precise, sources, page, totalMatches, callURI, lang) => { + @wrapper(s"Codesearch | $lang", headExtra) { + @searchBox(s"/$lang/search", query, filter, filePath, insensitive, space, precise, sources) + @resultFrame(lang, insensitive, space, precise, query, updated, packages, totalMatches) + @pagination(page, totalMatches, callURI) + } + } + case ErrorResponse(message) => { + @wrapper(s"Codesearch ", headExtra) { + @searchBoxError(message) + } } + } diff --git a/web-server/app/views/sourceCode.scala.html b/web-server/app/views/sourceCode.scala.html index 0a84ee7..b489f7c 100644 --- a/web-server/app/views/sourceCode.scala.html +++ b/web-server/app/views/sourceCode.scala.html @@ -1,9 +1,9 @@ @import codesearch.core.search.Search.Package -@import scala.collection.mutable -@import scala.collection.mutable.ListBuffer -@import scala.collection.immutable -@import play.twirl.api._ @import codesearch.web.SnippetHelper +@import play.twirl.api._ + +@import scala.collection.{immutable, mutable} +@import scala.collection.mutable.ListBuffer @( sourceCode: List[String],