diff --git a/.env.sample b/.env.sample new file mode 100644 index 00000000..38182410 --- /dev/null +++ b/.env.sample @@ -0,0 +1,3 @@ +DB_HOST=db +DB_USER=postgres +DB_PASSWORD=1q2w3e4r \ No newline at end of file diff --git a/.gitignore b/.gitignore index 59c74047..354f1c08 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /tmp /log /public +.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..239f76bd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM ruby:2.6.3 + +RUN gem install bundler:2.0.2 + +WORKDIR /app + +COPY Gemfile Gemfile.lock ./ + +RUN bundle install + +COPY . . + +CMD ["rails", "server", "-b", "0.0.0.0"] \ No newline at end of file diff --git a/Gemfile b/Gemfile index e20b1260..e807ef26 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,8 @@ gem 'rails', '~> 5.2.3' gem 'pg', '>= 0.18', '< 2.0' gem 'puma', '~> 3.11' gem 'bootsnap', '>= 1.1.0', require: false +gem 'activerecord-import' +gem 'rack-mini-profiler' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console @@ -17,6 +19,9 @@ group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + gem 'ruby-prof' + gem 'stackprof' + gem 'meta_request' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index fccf6f5f..befbeeaf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,145 +1,176 @@ GEM remote: https://rubygems.org/ specs: - actioncable (5.2.3) - actionpack (= 5.2.3) + actioncable (5.2.8.1) + actionpack (= 5.2.8.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) + actionmailer (5.2.8.1) + actionpack (= 5.2.8.1) + actionview (= 5.2.8.1) + activejob (= 5.2.8.1) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.3) - actionview (= 5.2.3) - activesupport (= 5.2.3) - rack (~> 2.0) + actionpack (5.2.8.1) + actionview (= 5.2.8.1) + activesupport (= 5.2.8.1) + rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.3) - activesupport (= 5.2.3) + actionview (5.2.8.1) + activesupport (= 5.2.8.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.3) - activesupport (= 5.2.3) + activejob (5.2.8.1) + activesupport (= 5.2.8.1) globalid (>= 0.3.6) - activemodel (5.2.3) - activesupport (= 5.2.3) - activerecord (5.2.3) - activemodel (= 5.2.3) - activesupport (= 5.2.3) + activemodel (5.2.8.1) + activesupport (= 5.2.8.1) + activerecord (5.2.8.1) + activemodel (= 5.2.8.1) + activesupport (= 5.2.8.1) arel (>= 9.0) - activestorage (5.2.3) - actionpack (= 5.2.3) - activerecord (= 5.2.3) - marcel (~> 0.3.1) - activesupport (5.2.3) + activerecord-import (1.6.0) + activerecord (>= 4.2) + activestorage (5.2.8.1) + actionpack (= 5.2.8.1) + activerecord (= 5.2.8.1) + marcel (~> 1.0.0) + activesupport (5.2.8.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) arel (9.0.0) - bindex (0.6.0) - bootsnap (1.4.2) - msgpack (~> 1.0) - builder (3.2.3) - byebug (11.0.1) - concurrent-ruby (1.1.5) - crass (1.0.4) - erubi (1.8.0) - ffi (1.10.0) - globalid (0.4.2) - activesupport (>= 4.2.0) - i18n (1.6.0) + bindex (0.8.1) + bootsnap (1.18.3) + msgpack (~> 1.2) + builder (3.2.4) + byebug (11.1.3) + concurrent-ruby (1.2.3) + crass (1.0.6) + date (3.3.4) + erubi (1.12.0) + ffi (1.16.3) + globalid (1.1.0) + activesupport (>= 5.0) + i18n (1.14.5) concurrent-ruby (~> 1.0) listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) ruby_dep (~> 1.2) - loofah (2.2.3) + loofah (2.22.0) crass (~> 1.0.2) - nokogiri (>= 1.5.9) - mail (2.7.1) + nokogiri (>= 1.12.0) + mail (2.8.1) mini_mime (>= 0.1.1) - marcel (0.3.3) - mimemagic (~> 0.3.2) - method_source (0.9.2) - mimemagic (0.3.3) - mini_mime (1.0.1) - mini_portile2 (2.4.0) - minitest (5.11.3) - msgpack (1.2.9) - nio4r (2.3.1) - nokogiri (1.10.2) - mini_portile2 (~> 2.4.0) - pg (1.1.4) - puma (3.12.1) - rack (2.0.6) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (5.2.3) - actioncable (= 5.2.3) - actionmailer (= 5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) - activemodel (= 5.2.3) - activerecord (= 5.2.3) - activestorage (= 5.2.3) - activesupport (= 5.2.3) + net-imap + net-pop + net-smtp + marcel (1.0.4) + meta_request (0.8.2) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 8) + method_source (1.1.0) + mini_mime (1.1.5) + mini_portile2 (2.8.6) + minitest (5.22.3) + msgpack (1.7.2) + net-imap (0.3.7) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.5.0) + net-protocol + nio4r (2.7.3) + nokogiri (1.13.10) + mini_portile2 (~> 2.8.0) + racc (~> 1.4) + pg (1.5.6) + puma (3.12.6) + racc (1.7.3) + rack (2.2.9) + rack-contrib (2.4.0) + rack (< 4) + rack-mini-profiler (3.1.1) + rack (>= 1.2.0) + rack-test (2.1.0) + rack (>= 1.3) + rails (5.2.8.1) + actioncable (= 5.2.8.1) + actionmailer (= 5.2.8.1) + actionpack (= 5.2.8.1) + actionview (= 5.2.8.1) + activejob (= 5.2.8.1) + activemodel (= 5.2.8.1) + activerecord (= 5.2.8.1) + activestorage (= 5.2.8.1) + activesupport (= 5.2.8.1) bundler (>= 1.3.0) - railties (= 5.2.3) + railties (= 5.2.8.1) sprockets-rails (>= 2.0.0) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.2) - railties (5.2.3) - actionpack (= 5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (1.5.0) + loofah (~> 2.19, >= 2.19.1) + railties (5.2.8.1) + actionpack (= 5.2.8.1) + activesupport (= 5.2.8.1) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) - rake (12.3.2) - rb-fsevent (0.10.3) - rb-inotify (0.10.0) + rake (13.2.1) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) ffi (~> 1.0) + ruby-prof (1.4.3) ruby_dep (1.5.0) - sprockets (3.7.2) + sprockets (4.2.1) concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.2.1) - actionpack (>= 4.0) - activesupport (>= 4.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) sprockets (>= 3.0.0) - thor (0.20.3) + stackprof (0.2.26) + thor (1.3.1) thread_safe (0.3.6) - tzinfo (1.2.5) + timeout (0.4.1) + tzinfo (1.2.11) thread_safe (~> 0.1) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) bindex (>= 0.4.0) railties (>= 5.0) - websocket-driver (0.7.0) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.3) + websocket-extensions (0.1.5) PLATFORMS ruby DEPENDENCIES + activerecord-import bootsnap (>= 1.1.0) byebug listen (>= 3.0.5, < 3.2) + meta_request pg (>= 0.18, < 2.0) puma (~> 3.11) + rack-mini-profiler rails (~> 5.2.3) + ruby-prof + stackprof tzinfo-data web-console (>= 3.3.0) diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..2d60a408 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.where(from: @from, to: @to).order(:start_time).preload(bus: :services).load end end diff --git a/app/models/buses_service.rb b/app/models/buses_service.rb new file mode 100644 index 00000000..6219d44e --- /dev/null +++ b/app/models/buses_service.rb @@ -0,0 +1,4 @@ +class BusesService < ApplicationRecord + belongs_to :bus + belongs_to :service +end diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad0..00000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c0..00000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639fc..00000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • - diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb deleted file mode 100644 index fa1de9aa..00000000 --- a/app/views/trips/_trip.html.erb +++ /dev/null @@ -1,5 +0,0 @@ -
  • <%= "Отправление: #{trip.start_time}" %>
    • -
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • -
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • -
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • -
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..019edb52 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -2,15 +2,24 @@ <%= "Автобусы #{@from.name} – #{@to.name}" %>

    - <%= "В расписании #{@trips.count} рейсов" %> + <%= "В расписании #{@trips.size} рейсов" %>

    <% @trips.each do |trip| %> - <%= render "delimiter" %> + ==================================================== <% end %> diff --git a/case-study.md b/case-study.md new file mode 100644 index 00000000..a8d5448d --- /dev/null +++ b/case-study.md @@ -0,0 +1,75 @@ +# Оптимизация импорта данных + +## Задача + +Нужно оптимизировать механизм перезагрузки расписания из файла так, чтобы он импортировал файл `large.json` в пределах +минуты. + +## Рабочий процесс + +Для начала я решил узнать за какое время выполняется импорт данных в текущей реализации. +Получилось что импорт файла `medium.json` выполняется 58 секунд. +Соответственно импорт файла `large.json` будет выполняться сильно дольше и мы не уложимся в бюджет, следовательно +необходимо оптимизировать этот процесс. + +Перед внесением изменений я решил покрыть имеющийся функционал тестами чтобы ничего не сломать (см +test/integration/bus_schedule_test.rb). + +Далее, после изучения отчетов профилировщиков стало понятно что основное время тратится на методах относящихся к +соединению с БД. +Следовательно можно уменьшить время выполнения программы сократив количесво транзакций с БД, с чем может помочь +гем `activerecord-import`. + +После переписывания программы с использованием гема импорт файла `medium.json` занимает менее 4 секунд: + +```bash +# bin/rake reload_json[fixtures/medium.json] +Finish in 3.53 +``` + +а импорт файла `large.json` менее 30 секунд: + +```bash +# bin/rake reload_json[fixtures/large.json] +Finish in 28.93 +``` + +Метрика укладывается в бюджет, задача выполнена. + +# Оптимизация отображения расписаний + +## Задача + +Нужно найти и устранить проблемы, замедляющие формирование этих страниц (какое время рендеринга страницы считать +удовлетворительным???). + +## Рабочий процесс + +Из логов можно увидеть что основное время тратится на рендеринг вьюх: + +``` +Completed 200 OK in 457ms (Views: 367.1ms | ActiveRecord: 84.5ms) +``` + +это время можно сократить избавившись от паршалов. После переработки имеем: + +``` +Completed 200 OK in 197ms (Views: 130.7ms | ActiveRecord: 63.8ms) +``` + +Так же в логах наглядно видна N+1 проблема при получении автобуса и его сервисов. +После её исправления имеем: + +``` +Completed 200 OK in 63ms (Views: 50.5ms | ActiveRecord: 9.5ms) +``` + +`rack-mini-profiler` показывает что наибоьшее время тратится на запрос трипов (выполняется Seq Scan), +ускорить этот запрос можно добавлением индексов на поля `from_id` и `to_id`. +После чего выполнение запроса сократилось с 30 мс до 11 мс. + +После загрузки файла `large.json` время рендеринга страницы `автобусы/Самара/Москва`: + +``` +Completed 200 OK in 204ms (Views: 69.6ms | ActiveRecord: 9.1ms) +``` \ No newline at end of file diff --git a/config/database.yml b/config/database.yml index e116cfa6..146f4e1b 100644 --- a/config/database.yml +++ b/config/database.yml @@ -20,6 +20,9 @@ default: &default # For details on connection pooling, see Rails configuration guide # http://guides.rubyonrails.org/configuring.html#database-pooling pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + host: <%= ENV.fetch("DB_HOST") %> + username: <%= ENV.fetch("DB_USER") %> + password: <%= ENV.fetch("DB_PASSWORD") %> development: <<: *default diff --git a/db/migrate/20240514113739_add_index_to_trips.rb b/db/migrate/20240514113739_add_index_to_trips.rb new file mode 100644 index 00000000..b2644b05 --- /dev/null +++ b/db/migrate/20240514113739_add_index_to_trips.rb @@ -0,0 +1,5 @@ +class AddIndexToTrips < ActiveRecord::Migration[5.2] + def change + add_index :trips, [:from_id, :to_id] + end +end diff --git a/db/migrate/20240514155955_add_index_to_buses_services.rb b/db/migrate/20240514155955_add_index_to_buses_services.rb new file mode 100644 index 00000000..5d832d93 --- /dev/null +++ b/db/migrate/20240514155955_add_index_to_buses_services.rb @@ -0,0 +1,5 @@ +class AddIndexToBusesServices < ActiveRecord::Migration[5.2] + def change + add_index :buses_services, :bus_id + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e45..dd54fb8a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,9 +10,10 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2024_05_14_155955) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" enable_extension "plpgsql" create_table "buses", force: :cascade do |t| @@ -23,6 +24,7 @@ create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["bus_id"], name: "index_buses_services_on_bus_id" end create_table "cities", force: :cascade do |t| @@ -40,6 +42,7 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end end diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..5fbb9777 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +services: + db: + image: postgres:16 + command: postgres -c shared_preload_libraries='pg_stat_statements' -c pg_stat_statements.track=all + volumes: + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: $DB_PASSWORD + + app: + build: . + volumes: + - .:/app + ports: + - "3000:3000" + env_file: .env + tty: true + stdin_open: true + depends_on: + - db + + adminer: + image: adminer + ports: + - ${ADMINER_PORT:-8080}:8080 + + pghero: + image: ankane/pghero + ports: + - 8085:8080 + environment: + DATABASE_URL: postgres://$DB_USER:$DB_PASSWORD@$DB_HOST:5432/task-4_development + +volumes: + postgres_data: \ No newline at end of file diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe871..6cc5fd41 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -10,18 +10,24 @@ task :reload_json, [:file_name] => :environment do |_task, args| Trip.delete_all ActiveRecord::Base.connection.execute('delete from buses_services;') + cities = {} + services = {} + buses_services = {} + buses = {} + trips = [] + json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] + from = cities[trip['from']] ||= City.new(name: trip['from']) + to = cities[trip['to']] ||= City.new(name: trip['to']) + + bus = buses[trip['bus']['number']] ||= Bus.new(number: trip['bus']['number'], model: trip['bus']['model']) + trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s + s = services[service] ||= Service.new(name: service) + buses_services[[bus, s]] ||= BusesService.new(bus: bus, service: s) end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - Trip.create!( + trips << Trip.new( from: from, to: to, bus: bus, @@ -30,5 +36,11 @@ task :reload_json, [:file_name] => :environment do |_task, args| price_cents: trip['price_cents'], ) end + + City.import! cities.values + Bus.import! buses.values + Service.import! services.values + BusesService.import! buses_services.values + Trip.import! trips end end diff --git a/test/integration/bus_schedule_test.rb b/test/integration/bus_schedule_test.rb new file mode 100644 index 00000000..f8b3823e --- /dev/null +++ b/test/integration/bus_schedule_test.rb @@ -0,0 +1,35 @@ +require 'test_helper' + +class BusScheduleTest < ActionDispatch::IntegrationTest + def setup + `rake reload_json[fixtures/example.json]` + end + + test "retrieve and check bus schedule page" do + get URI.encode('/автобусы/Самара/Москва') + assert_response :success + + assert_includes @response.body, 'Автобусы Самара – Москва' + assert_includes @response.body, 'В расписании 5 рейсов' + + schedule = [ + { departure: '17:30', arrival: '18:07', duration: '0ч. 37мин.', price: '1р. 73коп.', bus: 'Икарус №123', services: ['Туалет', 'WiFi'] }, + { departure: '18:30', arrival: '23:45', duration: '5ч. 15мин.', price: '9р. 69коп.', bus: 'Икарус №123', services: ['Туалет', 'WiFi'] }, + { departure: '19:30', arrival: '19:51', duration: '0ч. 21мин.', price: '6р. 63коп.', bus: 'Икарус №123', services: ['Туалет', 'WiFi'] }, + { departure: '20:30', arrival: '01:22', duration: '4ч. 52мин.', price: '0р. 22коп.', bus: 'Икарус №123', services: ['Туалет', 'WiFi'] }, + { departure: '21:30', arrival: '00:33', duration: '3ч. 3мин.', price: '8р. 46коп.', bus: 'Икарус №123', services: ['Туалет', 'WiFi'] } + ] + + schedule.each do |entry| + assert_includes @response.body, "Отправление: #{entry[:departure]}" + assert_includes @response.body, "Прибытие: #{entry[:arrival]}" + assert_includes @response.body, "В пути: #{entry[:duration]}" + assert_includes @response.body, "Цена: #{entry[:price]}" + assert_includes @response.body, "Автобус: #{entry[:bus]}" + entry[:services].each do |service| + assert_includes @response.body, service + end + assert_includes @response.body, "====================" + end + end +end \ No newline at end of file