diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..c99d2e73 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/Gemfile b/Gemfile index e20b1260..afbfdfd5 100644 --- a/Gemfile +++ b/Gemfile @@ -11,16 +11,23 @@ gem 'bootsnap', '>= 1.1.0', require: false group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] + gem 'bullet' end group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + gem 'meta_request' end group :test do + gem 'rspec-rails' + gem 'capybara' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] + +gem "pghero" +gem 'rack-mini-profiler' diff --git a/Gemfile.lock b/Gemfile.lock index fccf6f5f..98c00628 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,145 +1,213 @@ GEM remote: https://rubygems.org/ specs: - actioncable (5.2.3) - actionpack (= 5.2.3) + actioncable (5.2.8.1) + actionpack (= 5.2.8.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) + actionmailer (5.2.8.1) + actionpack (= 5.2.8.1) + actionview (= 5.2.8.1) + activejob (= 5.2.8.1) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.3) - actionview (= 5.2.3) - activesupport (= 5.2.3) - rack (~> 2.0) + actionpack (5.2.8.1) + actionview (= 5.2.8.1) + activesupport (= 5.2.8.1) + rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.3) - activesupport (= 5.2.3) + actionview (5.2.8.1) + activesupport (= 5.2.8.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.3) - activesupport (= 5.2.3) + activejob (5.2.8.1) + activesupport (= 5.2.8.1) globalid (>= 0.3.6) - activemodel (5.2.3) - activesupport (= 5.2.3) - activerecord (5.2.3) - activemodel (= 5.2.3) - activesupport (= 5.2.3) + activemodel (5.2.8.1) + activesupport (= 5.2.8.1) + activerecord (5.2.8.1) + activemodel (= 5.2.8.1) + activesupport (= 5.2.8.1) arel (>= 9.0) - activestorage (5.2.3) - actionpack (= 5.2.3) - activerecord (= 5.2.3) - marcel (~> 0.3.1) - activesupport (5.2.3) + activestorage (5.2.8.1) + actionpack (= 5.2.8.1) + activerecord (= 5.2.8.1) + marcel (~> 1.0.0) + activesupport (5.2.8.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) + addressable (2.8.5) + public_suffix (>= 2.0.2, < 6.0) arel (9.0.0) - bindex (0.6.0) - bootsnap (1.4.2) - msgpack (~> 1.0) - builder (3.2.3) - byebug (11.0.1) - concurrent-ruby (1.1.5) - crass (1.0.4) - erubi (1.8.0) - ffi (1.10.0) - globalid (0.4.2) - activesupport (>= 4.2.0) - i18n (1.6.0) + bindex (0.8.1) + bootsnap (1.16.0) + msgpack (~> 1.2) + builder (3.2.4) + bullet (7.1.2) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) + byebug (11.1.3) + capybara (3.36.0) + addressable + matrix + mini_mime (>= 0.1.3) + nokogiri (~> 1.8) + rack (>= 1.6.0) + rack-test (>= 0.6.3) + regexp_parser (>= 1.5, < 3.0) + xpath (~> 3.2) + concurrent-ruby (1.2.2) + crass (1.0.6) + date (3.3.3) + diff-lcs (1.5.0) + erubi (1.12.0) + ffi (1.16.3) + globalid (1.1.0) + activesupport (>= 5.0) + i18n (1.14.1) concurrent-ruby (~> 1.0) listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) ruby_dep (~> 1.2) - loofah (2.2.3) + loofah (2.21.4) crass (~> 1.0.2) - nokogiri (>= 1.5.9) - mail (2.7.1) + nokogiri (>= 1.12.0) + mail (2.8.1) mini_mime (>= 0.1.1) - marcel (0.3.3) - mimemagic (~> 0.3.2) - method_source (0.9.2) - mimemagic (0.3.3) - mini_mime (1.0.1) - mini_portile2 (2.4.0) - minitest (5.11.3) - msgpack (1.2.9) - nio4r (2.3.1) - nokogiri (1.10.2) - mini_portile2 (~> 2.4.0) - pg (1.1.4) - puma (3.12.1) - rack (2.0.6) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (5.2.3) - actioncable (= 5.2.3) - actionmailer (= 5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) - activemodel (= 5.2.3) - activerecord (= 5.2.3) - activestorage (= 5.2.3) - activesupport (= 5.2.3) + net-imap + net-pop + net-smtp + marcel (1.0.2) + matrix (0.4.2) + meta_request (0.7.4) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 7.1) + method_source (1.0.0) + mini_mime (1.1.5) + mini_portile2 (2.8.5) + minitest (5.20.0) + msgpack (1.7.2) + net-imap (0.3.7) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.1) + timeout + net-smtp (0.4.0) + net-protocol + nio4r (2.5.9) + nokogiri (1.13.10) + mini_portile2 (~> 2.8.0) + racc (~> 1.4) + pg (1.5.4) + pghero (2.8.3) + activerecord (>= 5) + public_suffix (5.0.3) + puma (3.12.6) + racc (1.7.1) + rack (2.2.8) + rack-contrib (2.4.0) + rack (< 4) + rack-mini-profiler (3.1.1) + rack (>= 1.2.0) + rack-test (2.1.0) + rack (>= 1.3) + rails (5.2.8.1) + actioncable (= 5.2.8.1) + actionmailer (= 5.2.8.1) + actionpack (= 5.2.8.1) + actionview (= 5.2.8.1) + activejob (= 5.2.8.1) + activemodel (= 5.2.8.1) + activerecord (= 5.2.8.1) + activestorage (= 5.2.8.1) + activesupport (= 5.2.8.1) bundler (>= 1.3.0) - railties (= 5.2.3) + railties (= 5.2.8.1) sprockets-rails (>= 2.0.0) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.2) - railties (5.2.3) - actionpack (= 5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (1.5.0) + loofah (~> 2.19, >= 2.19.1) + railties (5.2.8.1) + actionpack (= 5.2.8.1) + activesupport (= 5.2.8.1) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) - rake (12.3.2) - rb-fsevent (0.10.3) - rb-inotify (0.10.0) + rake (13.1.0) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) ffi (~> 1.0) + regexp_parser (2.8.2) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-rails (5.1.2) + actionpack (>= 5.2) + activesupport (>= 5.2) + railties (>= 5.2) + rspec-core (~> 3.10) + rspec-expectations (~> 3.10) + rspec-mocks (~> 3.10) + rspec-support (~> 3.10) + rspec-support (3.12.1) ruby_dep (1.5.0) - sprockets (3.7.2) + sprockets (4.2.1) concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.2.1) - actionpack (>= 4.0) - activesupport (>= 4.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) sprockets (>= 3.0.0) - thor (0.20.3) + thor (1.3.0) thread_safe (0.3.6) - tzinfo (1.2.5) + timeout (0.4.0) + tzinfo (1.2.11) thread_safe (~> 0.1) + uniform_notifier (1.16.0) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) bindex (>= 0.4.0) railties (>= 5.0) - websocket-driver (0.7.0) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.3) + websocket-extensions (0.1.5) + xpath (3.2.0) + nokogiri (~> 1.8) PLATFORMS ruby DEPENDENCIES bootsnap (>= 1.1.0) + bullet byebug + capybara listen (>= 3.0.5, < 3.2) + meta_request pg (>= 0.18, < 2.0) + pghero puma (~> 3.11) + rack-mini-profiler rails (~> 5.2.3) + rspec-rails tzinfo-data web-console (>= 3.3.0) diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..09176d7f 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.where(from: @from, to: @to).order(:start_time).preload(bus: :services) end end diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad0..00000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c0..00000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639fc..00000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • - diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb deleted file mode 100644 index fa1de9aa..00000000 --- a/app/views/trips/_trip.html.erb +++ /dev/null @@ -1,5 +0,0 @@ -
  • <%= "Отправление: #{trip.start_time}" %>
    • -
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • -
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • -
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • -
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..0fb8d681 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -2,15 +2,26 @@ <%= "Автобусы #{@from.name} – #{@to.name}" %>

    - <%= "В расписании #{@trips.count} рейсов" %> + <%= "В расписании #{@trips.load.size} рейсов" %>

    <% @trips.each do |trip| %> - <%= render "delimiter" %> + ==================================================== <% end %> diff --git a/bin/setup b/bin/setup index f294207b..2ba8ba2d 100755 --- a/bin/setup +++ b/bin/setup @@ -28,8 +28,14 @@ chdir APP_ROOT do puts "\n== Preparing database ==" system! 'bin/rails db:setup' - puts "\n== Loading data from fixtures/small.json ==" - system! 'bin/rake reload_json[fixtures/small.json]' + # puts "\n== Loading data from fixtures/small.json ==" + # system! 'bin/rake reload_json[fixtures/small.json]' + + # puts "\n== Loading data from fixtures/large.json ==" + # system! 'bin/rake reload_json[fixtures/large.json]' + + puts "\n== Loading data from fixtures/example.json ==" + system! 'bin/rake reload_json[fixtures/example.json]' puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' diff --git a/case-study.md b/case-study.md new file mode 100644 index 00000000..6e3a4829 --- /dev/null +++ b/case-study.md @@ -0,0 +1,133 @@ +# Загрузка файла + +Прикрутил логи и загрузил в `pgbadger`. Первое на что обращаешь внимание - кол-во запросов. + +У нас нет `Rails 6`, поэтому вместо `insert_all` будем писать запросы на `SQL`. +``` +City.insert_all(cities_names.map { |name| { name: name } }) +``` + +Готово! большой файл загружается за 4 секунд: +``` +== Loading data from fixtures/large.json == +Done! it took: 4.770822 sec. + +### + + $ b rails c +Loading development environment (Rails 5.2.8.1) +irb(main):001:0> Bus.count + (0.6ms) SELECT COUNT(*) FROM "buses" +=> 1000 +irb(main):002:0> Service.count + (0.5ms) SELECT COUNT(*) FROM "services" +=> 10 +irb(main):003:0> Trip.count + (7.6ms) SELECT COUNT(*) FROM "trips" +=> 100000 +irb(main):004:0> City.count + (0.5ms) SELECT COUNT(*) FROM "cities" +=> 10 +irb(main):005:0> Bus.last.services + Bus Load (0.3ms) SELECT "buses".* FROM "buses" ORDER BY "buses"."id" DESC LIMIT $1 [["LIMIT", 1]] + Service Load (1.2ms) SELECT "services".* FROM "services" INNER JOIN "buses_services" ON "services"."id" = "buses_services"."service_id" WHERE "buses_services"."bus_id" = $1 LIMIT $2 [["bus_id", 1000], ["LIMIT", 11]] +=> #, #, #, #, #, #, #, #, #]> + +``` + +# Загрузка страницы + +Начальное время - 16 секунд. +``` +Completed 200 OK in 16361ms (Views: 15437.2ms | ActiveRecord: 910.2ms) +``` +Судя по логам, есть работа для `bullet`. + +``` + CACHE Bus Load (0.0ms) SELECT "buses".* FROM "buses" WHERE "buses"."id" = $1 LIMIT $2 [["id", 793], ["LIMIT", 1]] + ↳ app/views/trips/_trip.html.erb:5 + Rendered trips/_trip.html.erb (3.4ms) + CACHE Service Load (0.0ms) SELECT "services".* FROM "services" INNER JOIN "buses_services" ON "services"."id" = "buses_services"."service_id" WHERE "buses_services"."bus_id" = $1 [["bus_id", 793]] +``` + +Гем говорит о целесообразности добавления `.includes(:bus)`. Правда, что странно - предлагает это сделать во вью. Мне кажется, лучше будет сделать в контроллере - и будем использовать `preload` в вместо `includes`. + +Время изменилось, но не существенно: +``` +Completed 200 OK in 13197ms (Views: 12532.7ms | ActiveRecord: 649.9ms) +``` + +Молодец, `bullet`! Подсказывает не забыть подгрузить `services` тоже. Все, больше для него работы нет. + + +``` +Completed 200 OK in 13902ms (Views: 13840.6ms | ActiveRecord: 47.7ms) +``` + +Установили `rack-mini-profiler`. Обращаем внимание на то, что его включение в `initializer` добавляет добрых 10 секунд! + +``` +Completed 200 OK in 20086ms (Views: 20004.5ms | ActiveRecord: 64.6ms) +``` + +Видим, что много занимает рендеринг `_services`. + +Пробуем воспользоваться рельсовым рендерингом коллекции: +``` + <% render services %> +``` + +Видим результат! + +``` +Completed 200 OK in 14502ms (Views: 14465.4ms | ActiveRecord: 31.2ms) +``` + +Не совсем ясно, что можно улучшить с `services`. Смотрим в соседний по весу `_trip` и видим использование `present?`. Но ведь мы знаем, что `any?` в данном случае лучше! +``` +Completed 200 OK in 12502ms (Views: 12462.2ms | ActiveRecord: 33.8ms) +``` + +Учитывая большое кол-во данных, кажется, нам не обойтись без кэширования `partials`. +Добавили `cache` в темплейты, включили `rails dev:cache`. + +В результате имеем результат: +``` +Completed 200 OK in 979ms (Views: 939.0ms | ActiveRecord: 35.3ms) +``` + +До сих пор мы (помимо `preload` и `any?`) занимались улучшением загрузки `partials`. +Что кажется верным, т.к. бд не видится боттлнеком в данном примере. Но все же добавим индексы: +``` +add_index :cities, :name, unique: true +add_index :buses_services, :bus_id +add_index :trips, [:from_id, :to_id] +``` + +Так. Давайте все же считать кэширование "нечестным" способом оптимизации здесь, и посмотрим, что сможем "выжать" из страницы без него. + +Снимаем показатели: +``` +Completed 200 OK in 4403ms (Views: 4382.7ms | ActiveRecord: 15.7ms) +``` + +Много времени занимает `delimiter`. Видимо, подхватывание `partial` не бесплатно, и раз уж мы говорим об оптимизации, нам можно обойтись без отдельного темплейта в данном случае. + +``` +Completed 200 OK in 3043ms (Views: 3022.1ms | ActiveRecord: 16.9ms) +``` + +Что ж, пойдем дальше - и весь код `html` соберем в `index.html.erb`. + +Уже неплохой результат! +``` +Completed 200 OK in 455ms (Views: 434.0ms | ActiveRecord: 17.1ms) +Completed 200 OK in 473ms (Views: 449.4ms | ActiveRecord: 15.2ms) +``` + +Заменили тип `start_time` на `time` в БД, стало немного лучше: +``` +Completed 200 OK in 397ms (Views: 381.6ms | ActiveRecord: 11.5ms) +``` + +Если бы стояла задача улучшать дальше (например, добраться-таки до 50 ms), я бы в первую очередь смотрел в сторону пагинации для этой страницы, наверное - но это в рамках данной задачи нам вряд ли нужно. diff --git a/config/database.yml b/config/database.yml index e116cfa6..e56ec196 100644 --- a/config/database.yml +++ b/config/database.yml @@ -20,10 +20,12 @@ default: &default # For details on connection pooling, see Rails configuration guide # http://guides.rubyonrails.org/configuring.html#database-pooling pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + username: task_4 + password: task_4 development: <<: *default - database: task-4_development + database: task_4_development # The specified database role being used to connect to postgres. # To create additional roles in postgres see `$ createuser --help`. @@ -57,7 +59,7 @@ development: # Do not set this db to the same as development or production. test: <<: *default - database: task-4_test + database: task_4_test # As with config/secrets.yml, you never want to store sensitive information, # like your database password, in your source code. If your source code is diff --git a/config/environments/development.rb b/config/environments/development.rb index 1311e3e4..8468e80a 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,4 +1,13 @@ Rails.application.configure do + # config.after_initialize do + # Bullet.enable = true + # Bullet.alert = true + # Bullet.bullet_logger = true + # Bullet.console = true + # Bullet.rails_logger = true + # Bullet.add_footer = true + # end + # Settings specified here will take precedence over those in config/application.rb. # In the development environment your application's code is reloaded on diff --git a/config/environments/test.rb b/config/environments/test.rb index 0a38fd3c..bc5971ab 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,4 +1,10 @@ Rails.application.configure do + config.after_initialize do + Bullet.enable = true + Bullet.bullet_logger = true + Bullet.raise = true # raise an error if n+1 query occurs + end + # Settings specified here will take precedence over those in config/application.rb. # The test environment is used exclusively to run your application's diff --git a/config/routes.rb b/config/routes.rb index a2da6a7b..371b78fb 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,7 @@ Rails.application.routes.draw do # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html - get "/" => "statistics#index" + mount PgHero::Engine, at: "pghero" + + # get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" end diff --git a/db/migrate/20231028140206_create_pghero_query_stats.rb b/db/migrate/20231028140206_create_pghero_query_stats.rb new file mode 100644 index 00000000..fbf41263 --- /dev/null +++ b/db/migrate/20231028140206_create_pghero_query_stats.rb @@ -0,0 +1,15 @@ +class CreatePgheroQueryStats < ActiveRecord::Migration[5.2] + def change + create_table :pghero_query_stats do |t| + t.text :database + t.text :user + t.text :query + t.integer :query_hash, limit: 8 + t.float :total_time + t.integer :calls, limit: 8 + t.timestamp :captured_at + end + + add_index :pghero_query_stats, [:database, :captured_at] + end +end diff --git a/db/migrate/20231030061817_add_name_index_to_cities.rb b/db/migrate/20231030061817_add_name_index_to_cities.rb new file mode 100644 index 00000000..abf16cc4 --- /dev/null +++ b/db/migrate/20231030061817_add_name_index_to_cities.rb @@ -0,0 +1,5 @@ +class AddNameIndexToCities < ActiveRecord::Migration[5.2] + def change + add_index :cities, :name, unique: true + end +end diff --git a/db/migrate/20231030061925_add_bus_id_index_to_buses_services.rb b/db/migrate/20231030061925_add_bus_id_index_to_buses_services.rb new file mode 100644 index 00000000..e66a0ad5 --- /dev/null +++ b/db/migrate/20231030061925_add_bus_id_index_to_buses_services.rb @@ -0,0 +1,5 @@ +class AddBusIdIndexToBusesServices < ActiveRecord::Migration[5.2] + def change + add_index :buses_services, :bus_id + end +end diff --git a/db/migrate/20231030062020_add_indexes_to_trips.rb b/db/migrate/20231030062020_add_indexes_to_trips.rb new file mode 100644 index 00000000..e85e12d2 --- /dev/null +++ b/db/migrate/20231030062020_add_indexes_to_trips.rb @@ -0,0 +1,5 @@ +class AddIndexesToTrips < ActiveRecord::Migration[5.2] + def change + add_index :trips, [:from_id, :to_id] + end +end diff --git a/db/migrate/20231030150533_change_trips_start_time_type.rb b/db/migrate/20231030150533_change_trips_start_time_type.rb new file mode 100644 index 00000000..1214c212 --- /dev/null +++ b/db/migrate/20231030150533_change_trips_start_time_type.rb @@ -0,0 +1,28 @@ +class ChangeTripsStartTimeType < ActiveRecord::Migration[5.2] + def up + # add a temporary column + add_column :trips, :start_time_time, :time + + # add the the current start_time as datetime to the temporary column for each entry + Trip.all.each do |trip| + trip.update(start_time_time: Time.parse(trip.start_time)) + end + + # drop the old time column + remove_column :trips, :start_time + + # rename the temporary column to start_time + rename_column :trips, :start_time_time, :start_time + end + + def down + add_column :trips, :start_time_time, :time + + Trip.all.each do |trip| + trip.update(start_time_time: Time.parse(trip.start_time).strftime('%H:%M')) + end + + remove_column :trips, :start_time + rename_column :trips, :start_time_time, :start_time + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e45..846bef40 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2023_10_30_150533) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -23,10 +23,23 @@ create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["bus_id"], name: "index_buses_services_on_bus_id" end create_table "cities", force: :cascade do |t| t.string "name" + t.index ["name"], name: "index_cities_on_name", unique: true + end + + create_table "pghero_query_stats", force: :cascade do |t| + t.text "database" + t.text "user" + t.text "query" + t.bigint "query_hash" + t.float "total_time" + t.bigint "calls" + t.datetime "captured_at" + t.index ["database", "captured_at"], name: "index_pghero_query_stats_on_database_and_captured_at" end create_table "services", force: :cascade do |t| @@ -36,10 +49,11 @@ create_table "trips", force: :cascade do |t| t.integer "from_id" t.integer "to_id" - t.string "start_time" t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.time "start_time" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end end diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe871..87e920ed 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,6 +1,11 @@ +def insert_multiple(array) + +end + # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| + time_point = Time.now json = JSON.parse(File.read(args.file_name)) ActiveRecord::Base.transaction do @@ -10,25 +15,61 @@ task :reload_json, [:file_name] => :environment do |_task, args| Trip.delete_all ActiveRecord::Base.connection.execute('delete from buses_services;') + # create cities + cities_names = ['from', 'to'].map { |f| json.pluck(f) }.flatten.uniq + sql = "INSERT INTO cities ( name ) VALUES #{cities_names.map { |n| "( '#{n}' )" }.join(', ') };" + ActiveRecord::Base.connection.execute(sql) + + # create buses + buses = json.map { |trip| [trip['bus']['number'], trip['bus']['model']] }.uniq + sql = "INSERT INTO buses ( number, model ) VALUES #{buses.map { |b| "( '#{b[0]}', '#{b[1]}' )" }.join(', ') };" + ActiveRecord::Base.connection.execute(sql) + + # create services + services_names = json.map { |trip| trip['bus']['services'] }.flatten.uniq + sql = "INSERT INTO services ( name ) VALUES #{services_names.map { |n| "( '#{n}' )" }.join(', ') };" + ActiveRecord::Base.connection.execute(sql) + + # assign services to buses + services_ids_by_name = Service.all.map { |s| [s.name, s.id] }.to_h + buses_ids_by_number = Bus.all.map { |b| [b.number, b.id] }.to_h + with_assigned_services = [] + buses_services = [] json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s + bus = trip['bus'] + bus_number = bus['number'] + next if with_assigned_services.include?(bus_number) + + bus_id = buses_ids_by_number[bus_number] + bus['services'].each do |service_name| + service_id = services_ids_by_name[service_name] + buses_services << [bus_id, service_id] end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) + with_assigned_services << bus_number + end + sql = <<~HEREDOC + INSERT INTO buses_services ( bus_id, service_id ) VALUES + #{buses_services.map { |b_s| "( #{b_s[0]}, #{b_s[1]} )" }.join(', ') }; + HEREDOC + ActiveRecord::Base.connection.execute(sql) - Trip.create!( - from: from, - to: to, - bus: bus, + # create trips + cities_ids_by_name = City.all.map { |s| [s.name, s.id] }.to_h + trips = json.map do |trip| + { + from_id: cities_ids_by_name[trip['from']], + to_id: cities_ids_by_name[trip['to']], + bus_id: buses_ids_by_number[trip['bus']['number']], start_time: trip['start_time'], duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) + price_cents: trip['price_cents'] + } end + sql = <<~HEREDOC + INSERT INTO trips ( from_id, to_id, start_time, duration_minutes, price_cents, bus_id ) VALUES + #{trips.map { |trip| "( #{trip[:from_id]}, #{trip[:to_id]}, '#{trip[:start_time]}', #{trip[:duration_minutes]}, #{trip[:price_cents]}, #{trip[:bus_id]} )" }.join(', ') }; + HEREDOC + ActiveRecord::Base.connection.execute(sql) end + puts "Done! it took: #{Time.now - time_point} sec." end diff --git a/out.html b/out.html new file mode 100644 index 00000000..2222164b --- /dev/null +++ b/out.html @@ -0,0 +1,8100 @@ + + + +pgBadger :: PostgreSQL Log Analyzer + + + + + + + + + + +
    +
    +
    +
    + + pgBadger +
    +
    + +
    +
    +
    + +
    + +
      + +
    • + +
      +

      Global information

      +
      +
      + +
        +
      • Generated on Sun Oct 29 12:14:01 2023
        • +
      • Log file: /Library/PostgreSQL/15/data/log/postgresql-2023-10-29_131256.log
        • +
      • Parsed 60,822 log entries in 5s
        • +
      • Log start from 2023-10-29 13:12:56 to 2023-10-29 13:13:58
        • +
      + +
      +
      +
      +
      • + + +
    • + +

      Overview

      + +
      +

      Global Stats

      +
      + +
      +
      +
        +
      • 11 Number of unique normalized queries
        • +
      • 8,950 Number of queries
        • +
      • 624ms Total query duration
        • +
      • 2023-10-29 13:12:56 First query
        • +
      • 2023-10-29 13:13:58 Last query
        • +
      • 1,428 queries/s at 2023-10-29 13:13:02 Query peak
        • +
      +
      +
      +
        +
      • 624ms Total query duration
        • +
      • 123ms Prepare/parse total duration
        • +
      • 209ms Bind total duration
        • +
      • 291ms Execute total duration
        • +
      +
      +
      +
        +
      • 0 Number of events
        • +
      • 0 Number of unique normalized events
        • +
      • 0 Max number of times the same event was reported
        • +
      • 0 Number of cancellation
        • +
      +
      +
      +
        +
      • 2 Total number of automatic vacuums
        • +
      • 0 Total number of automatic analyzes
        • +
      +
      +
      +
        +
      • 0 Number temporary file
        • +
      • 0 Max size of temporary file
        • +
      • 0.00 B Average size of temporary file
        • +
      +
      +
      +
        +
      • 1 Total number of sessions
        • +
      • 0 sessions at Session peak
        • +
      • 10s174ms Total duration of sessions
        • +
      • 10s174ms Average duration of sessions
        • +
      • 8,950 Average queries per session
        • +
      • 624ms Average queries duration per session
        • +
      • 9s549ms Average idle time per session
        • + +
      +
      +
      +
        +
      • 0 Total number of connections
        • + +
      • 1 Total number of databases
        • +
      +
      +
      +
      +
      + +
      +

      SQL Traffic

      +
      +

      Key values

      +
      +
        +
      • 1,428 queries/s Query Peak
        • +
      • 2023-10-29 13:13:02 Date
        • +
      +
      +
      +
      +
      + + +
      +
      + +
      +

      SELECT Traffic

      +
      +

      Key values

      +
      +
        +
      • 1,092 queries/s Query Peak
        • +
      • 2023-10-29 13:13:02 Date
        • +
      +
      +
      +
      +
      + + +
      +
      + +
      +

      INSERT/UPDATE/DELETE Traffic

      +
      +

      Key values

      +
      +
        +
      • 444 queries/s Query Peak
        • +
      • 2023-10-29 13:12:57 Date
        • +
      +
      +
      +
      +
      + + +
      +
      + +
      +

      Queries duration

      +
      +

      Key values

      +
      +
        +
      • 624ms Total query duration
        • +
      +
      +
      +
      +
      + + +
      +
      + +
      +

      Prepared queries ratio

      +
      +

      Key values

      +
      +
        +
      • 0.00 Ratio of bind vs prepare
        • +
      • 0.00 % Ratio between prepared and "usual" statements
        • +
      +
      +
      +
      +
      + + +
      +
      + +
      +

      General Activity

      +
      + +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      DayHourCountMin durationMax durationAvg durationLatency Percentile(90)Latency Percentile(95)Latency Percentile(99)
      Oct 29138,9500ms0ms0ms171ms171ms171ms
      +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + +
      DayHourSELECTCOPY TOAverage DurationLatency Percentile(90)Latency Percentile(95)Latency Percentile(99)
      Oct 29136,41500ms138ms138ms138ms
      +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      DayHourINSERTUPDATEDELETECOPY FROMAverage DurationLatency Percentile(90)Latency Percentile(95)Latency Percentile(99)
      Oct 29132,5340000ms38ms38ms38ms
      +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      DayHourPrepareBindBind/PreparePercentage of prepare
      Oct 29133,5708,9372.5039.89%
      +
      +
      + + + + + + + + + + + + + + + + + +
      DayHourCountAverage / Second
      Oct 291300.00/s
      +
      +
      + + + + + + + + + + + + + + + + + + + +
      DayHourCountAverage DurationAverage idle time
      Oct 2913110s174ms9s882ms
      +
      +
      + ↑ Back to the top of the General Activity table +
      + +
      + +
      • + + + + + + + + + +
    + + + + +
    + + + +
    + +
    + + + diff --git a/spec/features/trips_spec.rb b/spec/features/trips_spec.rb new file mode 100644 index 00000000..7b5f6b15 --- /dev/null +++ b/spec/features/trips_spec.rb @@ -0,0 +1,23 @@ +require 'rails_helper' + +RSpec.describe 'displays trips' do + let(:example_path) { 'fixtures/example.json' } + let(:example_data) { JSON.parse File.read(example_path) } + let(:good_trips) { example_data.select { |t| t['from'] == 'Самара' && t['to'] == 'Москва' } } + let(:bad_trips) { bad_trips - good_trips } + let(:link) { URI.encode('/автобусы/Самара/Москва') } + + before { `bin/rake reload_json[#{example_path}]` } + + it 'displays relevant trips with correct details' do + visit link + expect(page).to have_content("В расписании #{good_trips.count} рейсов") + good_trips.each do |trip| + expect(page).to have_content("Отправление: #{trip['start_time']}") + expect(page).to have_content("Прибытие: #{(Time.parse(trip['start_time']) + trip['duration_minutes'].minutes).strftime('%H:%M')}") + expect(page).to have_content("В пути: #{trip['duration_minutes'] / 60}ч. #{trip['duration_minutes'] % 60}мин.") + expect(page).to have_content("Цена: #{trip['price_cents'] / 100}р. #{trip['price_cents'] % 100}коп.") + expect(page).to have_content("Автобус: #{trip['bus']['model']} №#{trip['bus']['number']}") + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb new file mode 100644 index 00000000..b6317b5a --- /dev/null +++ b/spec/rails_helper.rb @@ -0,0 +1,64 @@ +# This file is copied to spec/ when you run 'rails generate rspec:install' +require 'spec_helper' +ENV['RAILS_ENV'] ||= 'test' +require_relative '../config/environment' +# Prevent database truncation if the environment is production +abort("The Rails environment is running in production mode!") if Rails.env.production? +require 'rspec/rails' +# Add additional requires below this line. Rails is not loaded until this point! + +# Requires supporting ruby files with custom matchers and macros, etc, in +# spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are +# run as spec files by default. This means that files in spec/support that end +# in _spec.rb will both be required and run as specs, causing the specs to be +# run twice. It is recommended that you do not name files matching this glob to +# end with _spec.rb. You can configure this pattern with the --pattern +# option on the command line or in ~/.rspec, .rspec or `.rspec-local`. +# +# The following line is provided for convenience purposes. It has the downside +# of increasing the boot-up time by auto-requiring all files in the support +# directory. Alternatively, in the individual `*_spec.rb` files, manually +# require only the support files necessary. +# +# Dir[Rails.root.join('spec', 'support', '**', '*.rb')].sort.each { |f| require f } + +# Checks for pending migrations and applies them before tests are run. +# If you are not using ActiveRecord, you can remove these lines. +begin + ActiveRecord::Migration.maintain_test_schema! +rescue ActiveRecord::PendingMigrationError => e + puts e.to_s.strip + exit 1 +end +RSpec.configure do |config| + # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures + config.fixture_path = "#{::Rails.root}/spec/fixtures" + + # If you're not using ActiveRecord, or you'd prefer not to run each of your + # examples within a transaction, remove the following line or assign false + # instead of true. + config.use_transactional_fixtures = true + + # You can uncomment this line to turn off ActiveRecord support entirely. + # config.use_active_record = false + + # RSpec Rails can automatically mix in different behaviours to your tests + # based on their file location, for example enabling you to call `get` and + # `post` in specs under `spec/controllers`. + # + # You can disable this behaviour by removing the line below, and instead + # explicitly tag your specs with their type, e.g.: + # + # RSpec.describe UsersController, type: :controller do + # # ... + # end + # + # The different available types are documented in the features, such as in + # https://relishapp.com/rspec/rspec-rails/docs + config.infer_spec_type_from_file_location! + + # Filter lines from Rails gems in backtraces. + config.filter_rails_from_backtrace! + # arbitrary gems may also be filtered via: + # config.filter_gems_from_backtrace("gem name") +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..6f3422b5 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,97 @@ +# This file was generated by the `rails generate rspec:install` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration + +require 'capybara/rspec' + +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # https://rspec.info/features/3-12/rspec-core/configuration/zero-monkey-patching-mode/ + config.disable_monkey_patching! + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end