diff --git a/.gitignore b/.gitignore index ad2933f..1cd0923 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,5 @@ /coverage /config/*.local.yml /vendor + +/test/shipments/* diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..c99d2e7 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/Dockerfile b/Dockerfile index e37825a..c98c9de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ################################################################################ # BASE ################################################################################ -FROM ruby:3.4-bullseye AS base +FROM ruby:3.4-bookworm AS base ARG KAKADU_FILE=KDU841_Demo_Apps_for_Linux-x86-64_231117.zip ARG FEED_VERSION=feed_v1.14.1 diff --git a/Gemfile b/Gemfile index ab11a60..3a8c8a8 100644 --- a/Gemfile +++ b/Gemfile @@ -7,10 +7,13 @@ gem "byebug" gem "dotenv" gem "minitest" gem "luhn-ruby" +gem "ostruct" gem "rake" gem "simplecov" gem "standardrb" gem "rubycritic" +gem "rspec" +gem "rspec-temp_dir" # This is to deal with warning in /usr/local/lib/ruby/3.4.0/readline.rb:4: # with the 3.4.5 ruby image. We should be able to take this out when we diff --git a/Gemfile.lock b/Gemfile.lock index edc5969..b72d0d7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,6 +17,7 @@ GEM concurrent-ruby (1.3.5) descendants_tracker (0.0.4) thread_safe (~> 0.3, >= 0.3.1) + diff-lcs (1.6.2) docile (1.4.1) dotenv (3.2.0) dry-configurable (1.3.0) @@ -70,6 +71,7 @@ GEM logger (1.7.0) luhn-ruby (1.0.0) minitest (5.26.2) + ostruct (0.6.1) parallel (1.27.0) parser (3.3.10.0) ast (~> 2.4.1) @@ -90,6 +92,21 @@ GEM reline (0.6.3) io-console (~> 0.5) rexml (3.4.4) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rspec-temp_dir (1.1.2) + rspec (>= 3.0) rubocop (1.81.7) json (~> 2.3) language_server-protocol (~> 3.17.0.2) @@ -165,8 +182,11 @@ DEPENDENCIES dotenv luhn-ruby minitest + ostruct rake reline + rspec + rspec-temp_dir rubycritic simplecov standardrb @@ -181,6 +201,7 @@ CHECKSUMS coercible (1.0.0) sha256=5081ad24352cc8435ce5472bc2faa30260c7ea7f2102cc6a9f167c4d9bffaadc concurrent-ruby (1.3.5) sha256=813b3e37aca6df2a21a3b9f1d497f8cbab24a2b94cab325bffe65ee0f6cbebc6 descendants_tracker (0.0.4) sha256=e9c41dd4cfbb85829a9301ea7e7c48c2a03b26f09319db230e6479ccdc780897 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e dotenv (3.2.0) sha256=e375b83121ea7ca4ce20f214740076129ab8514cd81378161f11c03853fe619d dry-configurable (1.3.0) sha256=882d862858567fc1210d2549d4c090f34370fc1bb7c5c1933de3fe792e18afa8 @@ -202,6 +223,7 @@ CHECKSUMS logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 luhn-ruby (1.0.0) sha256=bc9979c2b37eb14c0c15463aa6a9972a4fd07ddd272b8abefa2877fe5c43e4a8 minitest (5.26.2) sha256=f021118a6185b9ba9f5af71f2ba103ad770c75afde9f2ab8da512677c550cde3 + ostruct (0.6.1) sha256=09a3fb7ecc1fa4039f25418cc05ae9c82bd520472c5c6a6f515f03e4988cb817 parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 path_expander (1.1.3) sha256=bea16440dea5a770b9765312c8037931cc576f4f2872d71133a3e480028f9f67 @@ -214,6 +236,12 @@ CHECKSUMS regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rspec-temp_dir (1.1.2) sha256=92ccd30b8f4c80632a67cc4bbd6701b7b9d34b410d5bfc1fb6a095c5e6d8ce0a rubocop (1.81.7) sha256=6fb5cc298c731691e2a414fe0041a13eb1beed7bab23aec131da1bcc527af094 rubocop-ast (1.48.0) sha256=22df9bbf3f7a6eccde0fad54e68547ae1e2a704bf8719e7c83813a99c05d2e76 rubocop-performance (1.25.0) sha256=6f7d03568a770054117a78d0a8e191cefeffb703b382871ca7743831b1a52ec1 diff --git a/README.md b/README.md index 3451849..83af28d 100644 --- a/README.md +++ b/README.md @@ -41,13 +41,15 @@ $ docker-compose build ### 3. Running tests ``` -$ docker-compose run --rm test -$ docker-compose run --rm test bundle exec rubocop +$ docker-compose run --rm app bundle exec rake test +$ docker-compose run --rm app bundle exec rspec +$ docker-compose run --rm app bundle exec standardrb ``` or ``` $ bundle exec rake test -$ bundle exec rubocop +$ bundle exec rspec +$ bundle exec standardrb ``` diff --git a/bin/jhove b/bin/jhove index 4f31b1b..f220fd7 100755 --- a/bin/jhove +++ b/bin/jhove @@ -53,7 +53,7 @@ statuses = {} err_file = ['jhove', Time.now.strftime('%Y%m%d_%H%M%S'), 'errors.txt'].join '_' outfile = File.open err_file, 'w' -args.each do |arg| # rubocop:disable Metrics/BlockLength +args.each do |arg| dir = Pathname.new(arg).cleanpath.to_s unless File.exist? dir statuses[arg] = 'No such directory'.red diff --git a/bin/process b/bin/process index 48a1f1d..e89b67e 100755 --- a/bin/process +++ b/bin/process @@ -55,7 +55,7 @@ if ARGV.count.zero? exit 1 end -ARGV.each do |arg| # rubocop:disable Metrics/BlockLength +ARGV.each do |arg| dir = Pathname.new(arg).realpath.to_s unless File.exist?(dir) && File.directory?(dir) puts "Shipment directory #{dir.bold} does not exist, skipping".red diff --git a/config/config.dlxs.yml b/config/config.dlxs.yml index f709ab6..322434a 100644 --- a/config/config.dlxs.yml +++ b/config/config.dlxs.yml @@ -12,9 +12,9 @@ stages: - name: Tagger class: Tagger file: tagger - - name: Compressor - class: Compressor - file: compressor + - name: Compression + class: Compression + file: compression - name: DLXSCompressor class: DLXSCompressor file: dlxs_compressor diff --git a/config/config.yml b/config/config.yml index 0ca7a57..104643c 100644 --- a/config/config.yml +++ b/config/config.yml @@ -12,9 +12,9 @@ stages: - name: Tagger class: Tagger file: tagger - - name: Compressor - class: Compressor - file: compressor + - name: Compression + class: Compression + file: compression - name: Postflight class: Postflight file: postflight diff --git a/lib/agenda.rb b/lib/agenda.rb index 6067013..5aae7d4 100755 --- a/lib/agenda.rb +++ b/lib/agenda.rb @@ -15,7 +15,7 @@ def to_s end # Propagate stage errors onto subsequent stages. - def update(source_stage) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def update(source_stage) @stages.each do |stage| next if @stage_to_index[stage.name.to_sym] <= @stage_to_index[source_stage.name.to_sym] diff --git a/lib/compressor.rb b/lib/compressor.rb new file mode 100644 index 0000000..a2fa70f --- /dev/null +++ b/lib/compressor.rb @@ -0,0 +1,310 @@ +require "tiff" + +module Kakadu + JP2_LEVEL_MIN = 5 + JP2_LAYERS = 8 + JP2_ORDER = "RLCP" + JP2_USE_SOP = "yes" + JP2_USE_EPH = "yes" + JP2_MODES = '"RESET|RESTART|CAUSAL|ERTERM|SEGMARK"' + JP2_SLOPE = 42_988 + def self.compress(source, destination, tiffinfo) + clevels = jp2_clevels(tiffinfo) + cmd = "kdu_compress -quiet -i #{source} -o #{destination}" \ + " 'Clevels=#{clevels}'" \ + " 'Clayers=#{JP2_LAYERS}'" \ + " 'Corder=#{JP2_ORDER}'" \ + " 'Cuse_sop=#{JP2_USE_SOP}'" \ + " 'Cuse_eph=#{JP2_USE_EPH}'" \ + " Cmodes=#{JP2_MODES}" \ + " -no_weights -slope '#{JP2_SLOPE}'" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.jp2_clevels(tiffinfo) + # Get the width and height, figure out which is larger. + size = [tiffinfo[:width], tiffinfo[:height]].max + # Calculate appropriate Clevels. + clevels = (Math.log(size.to_i / 100.0) / Math.log(2)).to_i + (clevels < JP2_LEVEL_MIN) ? JP2_LEVEL_MIN : clevels + end +end + +module ExifTool + def self.remove_tiff_metadata(source:, destination:) + cmd = "exiftool -XMP:All= -MakerNotes:All= #{source} -o #{destination}" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.copy_jp2_metadata(source, destination, document_name, tiffinfo) + # If the original image has a date, we want it. If not, we + # want to add the current date. + # date "%Y-%m-%dT%H:%M:%S" + datetime = if tiffinfo[:date_time] + "-IFD0:ModifyDate>XMP-tiff:DateTime" + else + "-XMP-tiff:DateTime=#{Time.now.strftime("%FT%T")}" + end + cmd = "exiftool -tagsFromFile #{source}" \ + " '-XMP-dc:source=#{document_name}'" \ + " '-XMP-tiff:Compression=JPEG 2000'" \ + " '-IFD0:ImageWidth>XMP-tiff:ImageWidth'" \ + " '-IFD0:ImageHeight>XMP-tiff:ImageHeight'" \ + " '-IFD0:BitsPerSample>XMP-tiff:BitsPerSample'" \ + " '-IFD0:PhotometricInterpretation>XMP-tiff:" \ + "PhotometricInterpretation'" \ + " '-IFD0:Orientation>XMP-tiff:Orientation'" \ + " '-IFD0:SamplesPerPixel>XMP-tiff:SamplesPerPixel'" \ + " '-IFD0:XResolution>XMP-tiff:XResolution'" \ + " '-IFD0:YResolution>XMP-tiff:YResolution'" \ + " '-IFD0:ResolutionUnit>XMP-tiff:ResolutionUnit'" \ + " '-IFD0:Artist>XMP-tiff:Artist'" \ + " '-IFD0:Make>XMP-tiff:Make'" \ + " '-IFD0:Model>XMP-tiff:Model'" \ + " '-IFD0:Software>XMP-tiff:Software'" \ + " '#{datetime}'" \ + " -overwrite_original #{destination}" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.copy_jp2_alphaless_metadata(source, destination) + cmd = "exiftool -tagsFromFile #{source}" \ + " '-IFD0:BitsPerSample>XMP-tiff:BitsPerSample'" \ + " '-IFD0:SamplesPerPixel>XMP-tiff:SamplesPerPixel'" \ + " '-IFD0:PhotometricInterpretation>XMP-tiff:" \ + "PhotometricInterpretation'" \ + " -overwrite_original '#{destination}'" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.copy_tiff_metadata(source, destination) + cmd = "exiftool -tagsFromFile #{source}" \ + " '-IFD0:DocumentName'" \ + " '-IFD0:ImageDescription='" \ + " '-IFD0:Orientation'" \ + " '-IFD0:XResolution'" \ + " '-IFD0:YResolution'" \ + " '-IFD0:ResolutionUnit'" \ + " '-IFD0:ModifyDate'" \ + " '-IFD0:Artist'" \ + " '-IFD0:Make'" \ + " '-IFD0:Model'" \ + " '-IFD0:Software'" \ + " -overwrite_original '#{destination}'" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.clear_software_tag(path) + cmd = "exiftool -IFD0:Software= -overwrite_original #{path}" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end +end + +module ImageMagick + def self.remove_tiff_alpha(path) + tmp = path + ".alphaoff" + cmd = "convert #{path} -alpha off #{tmp}" + status = Command.new(cmd).run + FileUtils.mv(tmp, path) + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.strip_tiff_profiles(path) + tmp = path + ".stripped" + cmd = "convert #{path} -strip #{tmp}" + begin + status = Command.new(cmd).run + rescue => e + warning = "couldn't remove ICC profile (#{cmd}) (#{e.message})" + LogEntry.warning(error: Error.new(warning, nil, path)) + else + FileUtils.mv(tmp, path) + LogEntry.info(command: cmd, time: status[:time]) + end + end +end + +module TiffTools + TIFFTAGS = { + document_name: 269, + date_time: 306, + software: 305 + } + + def self.date_time_format(datetime) + datetime.strftime("%Y:%m:%d %H:%M:%S") + end + + def self.compress(source, destination) + cmd = "tifftopnm #{source} | pnmtotiff -g4 -rowsperstrip" \ + " 196136698 > #{destination}" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.copy_page_1(source, destination) + cmd = "tiffcp #{source},0 #{destination}" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end + + def self.set_tag(path:, tag:, value:) + cmd = "tiffset -s #{TIFFTAGS[tag]}, '#{value}' #{path}" + status = Command.new(cmd).run + LogEntry.info(command: cmd, time: status[:time]) + end +end + +class Compressor + attr_reader :tiffinfo, :image_file, :tmpdir + + def self.for(image_file:, tmpdir:, log: "whatever", now: Time.now) + tiffinfo = TIFF.new(image_file.path).info + klass = case tiffinfo[:bps] + when 8 + Compressor::Contone + when 1 + Compressor::Bitonal + else + raise "invalid source TIFF BPS #{tiffinfo[:bps]}" + end + klass.new(image_file:, tmpdir:, log: log, now: now) + end + + def initialize(image_file:, tmpdir:, log:, now:) + @image_file = image_file + @tiffinfo = TIFF.new(image_file.path).info + @tmpdir = tmpdir + @log = log + @now = now + end + + def run + raise NotImplementedError + end + + def compression_type + raise NotImplementedError + end + + def bps + @tiffinfo[:bps] + end + + def final_image_path + raise NotImplementedError + end + + private + + def log_it(log_entry) + @log.log_it log_entry + end +end + +class Compressor::Contone < Compressor + def compression_type + "JP2" + end + + def run(compression_tool = Kakadu) + # We don't want any XMP metadata to be copied over on its own. If + # it's been a while since we last ran exiftool, this might take a sec. + log_it ExifTool.remove_tiff_metadata(source: image_file.path, destination: sparse_path) + log_it ImageMagick.remove_tiff_alpha(sparse_path) if tiffinfo[:alpha] + log_it ImageMagick.strip_tiff_profiles(sparse_path) if tiffinfo[:icc] + + # mrio: copying this note over from Compression.rb. Not sure what it means + # or implies yet. + # + # FIXME: process-tiffs.sh defines this variable but does not + # use it. Check the original on tang. + # if /Samples\/Pixel:\s3/.match? metadata + # jp2_space = 'sRGB' + # else + # jp2_space = 'sLUM' + # end + + # We have a TIFF with no XMP now. We try to convert it to JP2. + # This will always take a second. Other than the initial loading + # of exiftool libraries, this is the only JP2 step that takes + # noticeable time. + log_it compression_tool.compress(sparse_path, output_path, tiffinfo) + + # We have our JP2; we can remove the middle TIFF. Then we try + # to grab metadata from the original TIFF. This should be very + # quick since we just used exiftool a few lines back. + log_it ExifTool.copy_jp2_metadata(image_file.path, output_path, document_name, tiffinfo) + + # If our image had an alpha channel, it'll be gone now, and + # the XMP data needs to reflect that (previously, we were + # taking that info from the original image). + log_it ExifTool.copy_jp2_alphaless_metadata(sparse_path, output_path) if tiffinfo[:alpha] + end + + def final_image_path + File.join(File.dirname(image_file.path), final_image_name) + end + + def document_name + objid_file_parts = image_file.objid_file.split("/") + objid_file_parts[-1] = final_image_name + File.join(objid_file_parts) + end + + def final_image_name + File.basename(image_file.file, ".*") + ".jp2" + end + + def sparse_path + @sparse_path ||= File.join(tmpdir, "sparse.tif") + end + + def output_path + @output_path ||= File.join(tmpdir, "output.jp2") + end +end + +class Compressor::Bitonal < Compressor + def compression_type + "G4" + end + + def run + # Try to compress the image. This is the only part of this step + # that should take any time. It should take a second or so. + log_it TiffTools.compress(image_file.path, compressed_path) + + log_it ExifTool.copy_tiff_metadata(image_file.path, compressed_path) + + log_it TiffTools.copy_page_1(compressed_path, output_path) + log_it TiffTools.set_tag(path: output_path, tag: :date_time, value: TiffTools.date_time_format(@now)) unless tiffinfo[:date_time] + + # Set the document name with objid/image.tif + log_it TiffTools.set_tag(path: output_path, tag: :document_name, value: image_file.objid_file) + if tiffinfo[:software] + log_it ExifTool.clear_software_tag(output_path) + log_it TiffTools.set_tag(path: output_path, tag: :software, value: tiffinfo[:software]) + else + log_it LogEntry.warning(error: Error.new("could not extract software", image_file.objid, image_file.path)) + end + end + + def compressed_path + @compressed_path ||= File.join(tmpdir, "#{File.basename(image_file.path)}-compressed") + end + + def final_image_path + image_file.path + end + + def output_path + @output_path ||= File.join(tmpdir, "output.tif") + end +end diff --git a/lib/jhove.rb b/lib/jhove.rb index c75755b..3c1752f 100755 --- a/lib/jhove.rb +++ b/lib/jhove.rb @@ -8,7 +8,7 @@ require "symbolize" # Wrapper for feed validate Perl script which invokes JHOVE -class JHOVE # rubocop:disable Metrics/ClassLength +class JHOVE attr_reader :errors, :raw_output UNUSED_FIELDS = %i[description file namespace objid diff --git a/lib/jp2.rb b/lib/jp2.rb index b7289ed..e391715 100755 --- a/lib/jp2.rb +++ b/lib/jp2.rb @@ -10,7 +10,7 @@ def initialize(path) end # Run tiffinfo command and return output text block - def info # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def info cmd = "exiftool #{@path}" status = Command.new(cmd).run jp2info = extract_fields(status[:stdout]) diff --git a/lib/log.rb b/lib/log.rb new file mode 100644 index 0000000..fc5fc0a --- /dev/null +++ b/lib/log.rb @@ -0,0 +1,129 @@ +class Log + include Enumerable + + def initialize(log: nil, objids: [], warnings: Warnings.new(objids: objids)) + @log = log || [] + @warnings = warnings + end + + def each + @log.each do |line| + yield line + end + end + + def entries + @log + end + + def warnings + @warnings.list + end + + def log(entry, time) + entry += format(" (%.3f sec)", time) unless time.nil? + @log << entry + end + + def log_it(data) + case data.level + when :info + log(data.command, data.time) + when :warning + add_warning(data.error) + end + end + + def add_warning(warning) + @warnings.add(warning) + end + + def to_json(state = nil, *) + JSON::State.from_state(state).generate(@log) + end +end + +class Exceptions + include Enumerable + + attr_reader :list, :objids + + def initialize(bar: SilentProgressBar.new, objids: [], list: nil) + @list = list || [] + @bar = bar + @objids = objids + end + + def each + @list.each do |line| + yield line + end + end + + def [](index) + @list[index] + end + + def []=(index, value) + @list[index] = value + end + + def to_json(state = nil, *) + JSON::State.from_state(state).generate(@list) + end + + def add(err) + unless err.objid.nil? || objids.member?(err.objid) + raise "unknown #{kind} objid #{err.objid}" + end + set_bar + @list << err + end + + def kind + raise NotImplementedError + end + + def set_bar + raise NotImplementedError + end +end + +class Errors < Exceptions + def kind + :error + end + + def set_bar + @bar.error = true + end +end + +class Warnings < Exceptions + def kind + :warning + end + + def set_bar + @bar.warning = true + end +end + +class LogEntry + def self.info(command:, time:) + new(level: :info, command: command, time: time) + end + + def self.warning(error:) + new(level: :warning, error: error) + end + + attr_reader :level, :command, :time, :error + + def initialize(level:, command: nil, time: nil, error: nil) + @level = level + @command = command + @time = time + @error = error + end +end diff --git a/lib/processor.rb b/lib/processor.rb index 61d0b4f..1b0c81c 100755 --- a/lib/processor.rb +++ b/lib/processor.rb @@ -18,7 +18,7 @@ Dir[File.join(__dir__, "shipment", "*.rb")].sort.each { |file| require file } # Processor -class Processor # rubocop:disable Metrics/ClassLength +class Processor attr_reader :dir, :config, :shipment # Can take a Shipment instead of a directory. @@ -36,7 +36,7 @@ def shipment_class Object.const_get(@config[:shipment_class] || "Shipment") end - def run # rubocop:disable Metrics/MethodLength + def run raise FinalizedShipmentError if shipment.finalized? if config[:restart_all] @@ -174,7 +174,7 @@ def run_stages end end - def run_stage(stage) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def run_stage(stage) stage_agenda = agenda.for_stage stage return if stage_agenda.none? @@ -194,14 +194,12 @@ def run_stage(stage) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength end end - def init_status_file # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def init_status_file return false unless File.exist? status_file # In order to reconstitute Error objects we need to use JSON.load # instead of JSON.parse. So we explicitly symbolize the output. - # rubocop:disable Security/JSONLoad status = Symbolize.symbolize JSON.unsafe_load File.new(status_file) - # rubocop:enable Security/JSONLoad raise JSON::ParserError, "unable to parse #{status_file}" if status.nil? unless status.key?(:shipment) && status[:shipment].is_a?(Shipment) diff --git a/lib/query_tool.rb b/lib/query_tool.rb index 61b593c..cdc4650 100755 --- a/lib/query_tool.rb +++ b/lib/query_tool.rb @@ -8,14 +8,14 @@ require "processor" # Facility for running command-line processor/shipment queries and commands -class QueryTool # rubocop:disable Metrics/ClassLength +class QueryTool attr_reader :processor def initialize(processor) @processor = processor end - def agenda_cmd(*_args) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def agenda_cmd(*_args) if processor.agenda.any? processor.stages.each do |stage| puts stage.name.bold @@ -43,7 +43,7 @@ def objids_cmd end end - def errors_cmd(*args) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength + def errors_cmd(*args) errs = processor.errors_by_objid_by_stage errs.each_key do |objid| next if args.count.positive? && !args.include?(objid) @@ -63,7 +63,7 @@ def errors_cmd(*args) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticCompl end end - def warnings_cmd(*args) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength + def warnings_cmd(*args) warnings = processor.warnings_by_objid_by_stage warnings.each_key do |objid| next if args.count.positive? && !args.include?(objid) @@ -91,7 +91,7 @@ def status_cmd end end - def fixity_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def fixity_cmd unless File.directory? processor.shipment.source_directory puts "Source directory not yet populated" end @@ -141,7 +141,7 @@ def status_status(stage) end end - def status_detail(stage) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def status_detail(stage) statuses = [] bad = stage.error_objids.count total = stage.objids.count diff --git a/lib/shipment.rb b/lib/shipment.rb index 9f3d279..3c19e20 100755 --- a/lib/shipment.rb +++ b/lib/shipment.rb @@ -12,7 +12,7 @@ class FinalizedShipmentError < StandardError ImageFile = Struct.new(:objid, :path, :objid_file, :file) # Shipment directory class -class Shipment # rubocop:disable Metrics/ClassLength +class Shipment PATH_COMPONENTS = 1 OBJID_SEPARATOR = "/" attr_reader :metadata @@ -121,7 +121,7 @@ def validate_objid(objid) Luhn.valid?(objid) ? nil : "Luhn checksum failed" end - def image_files(type = "tif", dir = @dir) # rubocop:disable Metrics/MethodLength + def image_files(type = "tif", dir = @dir) files = [] find_objids(dir).each do |objid| objid_path = objid_to_path objid @@ -147,7 +147,7 @@ def source_image_files(type = "tif") # every other directory in @dir into it. # We will potentially remove and re-copy directories from source/ # but that depends on the options passed to the processor. - def setup_source_directory # rubocop:disable Metrics/AbcSize + def setup_source_directory raise FinalizedShipmentError if finalized? return if File.exist? source_directory @@ -164,7 +164,7 @@ def setup_source_directory # rubocop:disable Metrics/AbcSize # Copy clean or remediated objid directories from source. # Called with nil to replaces all objids, or an Array of objids. - def restore_from_source_directory(objid_array = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def restore_from_source_directory(objid_array = nil) raise FinalizedShipmentError if finalized? unless File.directory? source_directory raise Errno::ENOENT, "source directory #{source_directory} not found" @@ -213,7 +213,7 @@ def checksum_source_directory end # Returns Hash with keys {added, changed, removed} -> Array of ImageFile - def fixity_check # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def fixity_check fixity = {added: [], removed: [], changed: []} return fixity if metadata[:checksums].nil? @@ -250,7 +250,7 @@ def find_objids(dir = @dir) bars.sort end - def find_objids_with_components(dir, components) # rubocop:disable Metrics/MethodLength + def find_objids_with_components(dir, components) bars = [] if components.count < self.class::PATH_COMPONENTS subdir = File.join(dir, components) diff --git a/lib/stage.rb b/lib/stage.rb index 3b63c85..c7ff51f 100755 --- a/lib/stage.rb +++ b/lib/stage.rb @@ -9,9 +9,10 @@ require "error" require "progress_bar" require "symbolize" +require "log" # Base class for conversion stages -class Stage # rubocop:disable Metrics/ClassLength +class Stage attr_reader :errors, :warnings, :start, :end attr_accessor :name, :config, :shipment @@ -38,7 +39,7 @@ def to_json(*args) # Can be created without a shipment, but that field needs to be set # before the #run method can be called. - def initialize(shipment, **args) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def initialize(shipment, **args) unless shipment.nil? || shipment.is_a?(Shipment) raise StandardError, "unknown shipment class #{shipment.class}" end @@ -46,9 +47,18 @@ def initialize(shipment, **args) # rubocop:disable Metrics/AbcSize, Metrics/Cycl @shipment = shipment @name = args[:name] || self.class.to_s @config = args[:config] || {} # Top-level Config object - @errors = args[:errors] || [] # Fatal conditions, Array of Error - @warnings = args[:warnings] || [] # Nonfatal conditions, Array of Error - @data = args[:data] || {} # Misc data structure including log + @bar = if config[:no_progress] + SilentProgressBar.new(self.class) + else + ProgressBar.new(self.class) + end + @errors = Errors.new(bar: @bar, objids: objids, list: args[:errors]) + @warnings = Warnings.new(bar: @bar, objids: objids, list: args[:errors]) + + @data = args[:data] || {log: Log.new(warnings: @warnings)} # Misc data structure including log + if @data[:log].instance_of?(::Array) || @data[:log].nil? + @data[:log] = Log.new(log: @data[:log], warnings: @warnings) + end # Time the stage was last run @start = if args[:start].to_s == "" nil @@ -62,17 +72,12 @@ def initialize(shipment, **args) # rubocop:disable Metrics/AbcSize, Metrics/Cycl else Time.parse args[:end] end - @bar = if config[:no_progress] - SilentProgressBar.new(self.class) - else - ProgressBar.new(self.class) - end end # Get rid of errors, warnings, and anything that may have been memoized def reinitialize! - @errors = [] - @warnings = [] + @errors = Errors.new(bar: @bar, objids: objids) + @warnings = Warnings.new(bar: @bar, objids: objids) @bar.done = nil end @@ -92,23 +97,24 @@ def run(_agenda) end def add_error(err) - raise "#{err.class} passed to add_error" unless err.is_a? Error - unless err.objid.nil? || objids.member?(err.objid) - raise "unknown error objid #{err.objid}" - end + @errors.add(err) + # raise "#{err.class} passed to add_error" unless err.is_a? Error + # unless err.objid.nil? || objids.member?(err.objid) + # raise "unknown error objid #{err.objid}" + # end - @bar.error = true - @errors << err + # @bar.error = true + # @errors << err end def add_warning(err) - raise "#{err.class} passed to add_warning" unless err.is_a? Error - unless err.objid.nil? || objids.member?(err.objid) - raise "unknown warning objid #{err.objid}" - end + @warnings.add(err) + # unless err.objid.nil? || objids.member?(err.objid) + # raise "unknown warning objid #{err.objid}" + # end - @bar.warning = true - @warnings << err + # @bar.warning = true + # @warnings << err end # Map of objids + nil -> [Errors] @@ -136,14 +142,6 @@ def fatal_error? errors_by_objid.key? nil end - def delete_errors_for_objid(objid) - @errors.delete_if { |err| err.objid == objid } - end - - def delete_warnings_for_objid(objid) - @warnings.delete_if { |err| err.objid == objid } - end - # OK to make destructive changes to the shipment for this objid? # With nil objid checks for presence of any error. def make_changes?(objid = nil) @@ -219,12 +217,15 @@ def delete_on_success(path, objid = nil) end def objids - @objids ||= shipment.objids + @objids ||= shipment&.objids || [] + end + + def log_collection + @data[:log] end def log(entry, time = nil) - entry += format(" (%.3f sec)", time) unless time.nil? - (@data[:log] ||= []) << entry + log_collection.log(entry, time) end def shipment_directory diff --git a/lib/stage/compression.rb b/lib/stage/compression.rb new file mode 100755 index 0000000..dcd3a87 --- /dev/null +++ b/lib/stage/compression.rb @@ -0,0 +1,34 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "stage" +require "tiff" +require "compressor" + +# TIFF to JP2/TIFF compression stage +class Compression < Stage + def run(agenda) + return unless agenda.any? + files = image_files.select { |file| agenda.include? file.objid } + @bar.steps = files.count + files.each_with_index do |image_file, i| + compressor = Compressor.for(image_file: image_file, tmpdir: create_tempdir, log: log_collection) + + @bar.step! i, "#{image_file.objid_file} #{compressor.compression_type}" + + compressor.run + + on_disk_temp_image_path = compressor.final_image_path.sub(shipment.directory, shipment.tmp_directory) + system("mkdir -p #{File.dirname(on_disk_temp_image_path)}") + system("cp #{compressor.output_path} #{on_disk_temp_image_path}") + + copy_on_success on_disk_temp_image_path, compressor.final_image_path, compressor.image_file.objid + delete_on_success compressor.image_file.path, compressor.image_file.objid if compressor.bps == 8 + + system("rm -r #{compressor.tmpdir}/*") + rescue => e + add_error Error.new(e.message, image_file.objid, image_file.file) + next + end + end +end diff --git a/lib/stage/compressor.rb b/lib/stage/compressor.rb deleted file mode 100755 index 3f3d9b1..0000000 --- a/lib/stage/compressor.rb +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require "stage" -require "tiff" - -JP2_LEVEL_MIN = 5 -JP2_LAYERS = 8 -JP2_ORDER = "RLCP" -JP2_USE_SOP = "yes" -JP2_USE_EPH = "yes" -JP2_MODES = '"RESET|RESTART|CAUSAL|ERTERM|SEGMARK"' -JP2_SLOPE = 42_988 - -TIFF_DATE_FORMAT = "%Y:%m:%d %H:%M:%S" - -# TIFF to JP2/TIFF compression stage -class Compressor < Stage # rubocop:disable Metrics/ClassLength - def run(agenda) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength - return unless agenda.any? - - files = image_files.select { |file| agenda.include? file.objid } - @bar.steps = files.count - files.each_with_index do |image_file, i| - begin - tiffinfo = TIFF.new(image_file.path).info - rescue => e - add_error Error.new(e.message, image_file.objid, image_file.file) - next - end - case tiffinfo[:bps] - when 8 - # It's a contone, so we convert to JP2. - @bar.step! i, "#{image_file.objid_file} JP2" - begin - handle_8_bps_conversion(image_file, tiffinfo) - rescue => e - add_error Error.new(e.message, image_file.objid, image_file.file) - end - when 1 - # It's bitonal, so we G4 compress it. - @bar.step! i, "#{image_file.objid_file} G4" - begin - handle_1_bps_conversion(image_file, tiffinfo) - rescue => e - add_error Error.new(e.message, image_file.objid, image_file.file) - end - else - add_error Error.new("invalid source TIFF BPS #{tiffinfo[:bps]}", - image_file.objid, image_file.file) - end - end - end - - private - - def handle_8_bps_conversion(image_file, tiffinfo) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - tmpdir = create_tempdir - sparse = File.join(tmpdir, "sparse.tif") - new_image = File.join(tmpdir, "new.jp2") - final_image_name = File.basename(image_file.path, ".*") + ".jp2" - final_image = File.join(File.dirname(image_file.path), final_image_name) - document_name = File.join(shipment.objid_to_path(image_file.objid), - final_image_name) - on_disk_temp_image = final_image.sub(shipment.directory, shipment.tmp_directory) - system("mkdir -p #{File.dirname(on_disk_temp_image)}") - - # We don't want any XMP metadata to be copied over on its own. If - # it's been a while since we last ran exiftool, this might take a sec. - remove_tiff_metadata(image_file.path, sparse) - - remove_tiff_alpha(sparse) if tiffinfo[:alpha] - - strip_tiff_profiles(sparse) if tiffinfo[:icc] - - # FIXME: process-tiffs.sh defines this variable but does not - # use it. Check the original on tang. - # if /Samples\/Pixel:\s3/.match? metadata - # jp2_space = 'sRGB' - # else - # jp2_space = 'sLUM' - # end - - # We have a TIFF with no XMP now. We try to convert it to JP2. - # This will always take a second. Other than the initial loading - # of exiftool libraries, this is the only JP2 step that takes - # noticeable time. - compress_jp2(sparse, new_image, tiffinfo) - # We have our JP2; we can remove the middle TIFF. Then we try - # to grab metadata from the original TIFF. This should be very - # quick since we just used exiftool a few lines back. - copy_jp2_metadata(image_file.path, new_image, document_name, tiffinfo) - # If our image had an alpha channel, it'll be gone now, and - # the XMP data needs to reflect that (previously, we were - # taking that info from the original image). - copy_jp2_alphaless_metadata(sparse, new_image) if tiffinfo[:alpha] - system("cp #{new_image} #{on_disk_temp_image}") - system("rm -r #{tmpdir}/*") - copy_on_success on_disk_temp_image, final_image, image_file.objid - delete_on_success image_file.path, image_file.objid - end - - def jp2_clevels(tiffinfo) - # Get the width and height, figure out which is larger. - size = [tiffinfo[:width], tiffinfo[:height]].max - # Calculate appropriate Clevels. - clevels = (Math.log(size.to_i / 100.0) / Math.log(2)).to_i - (clevels < JP2_LEVEL_MIN) ? JP2_LEVEL_MIN : clevels - end - - def remove_tiff_metadata(path, destination) - cmd = "exiftool -XMP:All= -MakerNotes:All= #{path} -o #{destination}" - status = Command.new(cmd).run - log cmd, status[:time] - end - - def remove_tiff_alpha(path) - tmp = path + ".alphaoff" - cmd = "convert #{path} -alpha off #{tmp}" - status = Command.new(cmd).run - log cmd, status[:time] - FileUtils.mv(tmp, path) - end - - def strip_tiff_profiles(path) # rubocop:disable Metrics/MethodLength - tmp = path + ".stripped" - cmd = "convert #{path} -strip #{tmp}" - begin - status = Command.new(cmd).run - rescue => e - warning = "couldn't remove ICC profile (#{cmd}) (#{e.message})" - add_warning Error.new(warning, objid_from_path(path), path) - else - log cmd, status[:time] - FileUtils.mv(tmp, path) - end - end - - def compress_jp2(source, destination, tiffinfo) # rubocop:disable Metrics/MethodLength - clevels = jp2_clevels(tiffinfo) - cmd = "kdu_compress -quiet -i #{source} -o #{destination}" \ - " 'Clevels=#{clevels}'" \ - " 'Clayers=#{JP2_LAYERS}'" \ - " 'Corder=#{JP2_ORDER}'" \ - " 'Cuse_sop=#{JP2_USE_SOP}'" \ - " 'Cuse_eph=#{JP2_USE_EPH}'" \ - " Cmodes=#{JP2_MODES}" \ - " -no_weights -slope '#{JP2_SLOPE}'" - status = Command.new(cmd).run - log cmd, status[:time] - end - - def copy_jp2_metadata(source, destination, document_name, tiffinfo) # rubocop:disable Metrics/MethodLength - # If the original image has a date, we want it. If not, we - # want to add the current date. - # date "%Y-%m-%dT%H:%M:%S" - datetime = if tiffinfo[:date_time] - "-IFD0:ModifyDate>XMP-tiff:DateTime" - else - "-XMP-tiff:DateTime=#{Time.now.strftime("%FT%T")}" - end - cmd = "exiftool -tagsFromFile #{source}" \ - " '-XMP-dc:source=#{document_name}'" \ - " '-XMP-tiff:Compression=JPEG 2000'" \ - " '-IFD0:ImageWidth>XMP-tiff:ImageWidth'" \ - " '-IFD0:ImageHeight>XMP-tiff:ImageHeight'" \ - " '-IFD0:BitsPerSample>XMP-tiff:BitsPerSample'" \ - " '-IFD0:PhotometricInterpretation>XMP-tiff:" \ - "PhotometricInterpretation'" \ - " '-IFD0:Orientation>XMP-tiff:Orientation'" \ - " '-IFD0:SamplesPerPixel>XMP-tiff:SamplesPerPixel'" \ - " '-IFD0:XResolution>XMP-tiff:XResolution'" \ - " '-IFD0:YResolution>XMP-tiff:YResolution'" \ - " '-IFD0:ResolutionUnit>XMP-tiff:ResolutionUnit'" \ - " '-IFD0:Artist>XMP-tiff:Artist'" \ - " '-IFD0:Make>XMP-tiff:Make'" \ - " '-IFD0:Model>XMP-tiff:Model'" \ - " '-IFD0:Software>XMP-tiff:Software'" \ - " '#{datetime}'" \ - " -overwrite_original #{destination}" - status = Command.new(cmd).run - log cmd, status[:time] - end - - def copy_jp2_alphaless_metadata(source, destination) - cmd = "exiftool -tagsFromFile #{source}" \ - " '-IFD0:BitsPerSample>XMP-tiff:BitsPerSample'" \ - " '-IFD0:SamplesPerPixel>XMP-tiff:SamplesPerPixel'" \ - " '-IFD0:PhotometricInterpretation>XMP-tiff:" \ - "PhotometricInterpretation'" \ - " -overwrite_original '#{destination}'" - status = Command.new(cmd).run - log cmd, status[:time] - end - - def handle_1_bps_conversion(image_file, tiffinfo) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - tmpdir = create_tempdir - compressed = File.join(tmpdir, - "#{File.basename(image_file.path)}-compressed") - page1 = File.join(tmpdir, "#{File.basename(image_file.path)}-page1") - compress_tiff(image_file.path, compressed) - copy_tiff_metadata(image_file.path, compressed) - copy_tiff_page1(compressed, page1) - FileUtils.rm(compressed) - write_tiff_date_time page1 unless tiffinfo[:date_time] - write_tiff_document_name(image_file, page1) - if tiffinfo[:software] - write_tiff_software(page1, tiffinfo[:software]) - else - add_warning Error.new("could not extract software", image_file.objid, - image_file.path) - end - copy_on_success page1, image_file.path, image_file.objid - end - - # Try to compress the image. This is the only part of this step - # that should take any time. It should take a second or so. - def compress_tiff(path, destination) - cmd = "tifftopnm #{path} | pnmtotiff -g4 -rowsperstrip" \ - " 196136698 > #{destination}" - status = Command.new(cmd).run - log cmd, status[:time] - end - - def copy_tiff_metadata(path, destination) # rubocop:disable Metrics/MethodLength - cmd = "exiftool -tagsFromFile #{path}" \ - " '-IFD0:DocumentName'" \ - " '-IFD0:ImageDescription='" \ - " '-IFD0:Orientation'" \ - " '-IFD0:XResolution'" \ - " '-IFD0:YResolution'" \ - " '-IFD0:ResolutionUnit'" \ - " '-IFD0:ModifyDate'" \ - " '-IFD0:Artist'" \ - " '-IFD0:Make'" \ - " '-IFD0:Model'" \ - " '-IFD0:Software'" \ - " -overwrite_original '#{destination}'" - status = Command.new(cmd).run - log cmd, status[:time] - end - - def copy_tiff_page1(path, destination) - cmd = "tiffcp #{path},0 #{destination}" - status = Command.new(cmd).run - log cmd, status[:time] - end - - # Set the document name with objid/image.tif - def write_tiff_document_name(image_file, destination) - tiff = TIFF.new(destination) - tiffset = tiff.set(TIFF::TIFFTAG_DOCUMENTNAME, image_file.objid_file) - log tiffset[:cmd], tiffset[:time] - end - - # Remove ImageMagick software tag (if it exists) and replace with original - def write_tiff_software(path, software) - cmd = "exiftool -IFD0:Software= -overwrite_original #{path}" - status = Command.new(cmd).run - log cmd, status[:time] - cmd = "tiffset -s 305 '#{software}' #{path}" - status = Command.new(cmd).run - log cmd, status[:time] - end - - def write_tiff_date_time(path) - date = Time.now.strftime(TIFF_DATE_FORMAT) - cmd = "tiffset -s 306 '#{date}' #{path}" - status = Command.new(cmd).run - log cmd, status[:time] - end -end diff --git a/lib/stage/dlxs_compressor.rb b/lib/stage/dlxs_compressor.rb index 1c27b7f..6c8a333 100755 --- a/lib/stage/dlxs_compressor.rb +++ b/lib/stage/dlxs_compressor.rb @@ -5,8 +5,8 @@ require "stage" # JP2-to-TIFF conversion stage for DLXS -class DLXSCompressor < Stage # rubocop:disable Metrics/ClassLength - def run(agenda) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength +class DLXSCompressor < Stage + def run(agenda) return unless agenda.any? files = image_files("jp2").select { |file| agenda.include? file.objid } @@ -23,7 +23,7 @@ def run(agenda) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength private - def handle_conversion(image_file) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def handle_conversion(image_file) basename = File.basename(image_file.path, ".*") tmpdir = create_tempdir basename source = File.join(tmpdir, "source.tif") @@ -42,7 +42,7 @@ def handle_conversion(image_file) # rubocop:disable Metrics/AbcSize, Metrics/Met delete_on_success image_file.path end - def copy_metadata(source, destination, source_image) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def copy_metadata(source, destination, source_image) cmd = <<~CMD.tr("\n", " ") exiftool -tagsFromFile #{source} '-IFD0:DocumentName=#{source_image}' @@ -87,7 +87,7 @@ def get_y_resolution(path) end # Converts 'source.tif' to 'source.pgm' in temporary directory - def tiff_to_pgm(dir) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def tiff_to_pgm(dir) source = File.join(dir, "source.tif") pnm = File.join(dir, "source.pnm") pgm = File.join(dir, "source.pgm") diff --git a/lib/stage/image_validator.rb b/lib/stage/image_validator.rb index cd1213c..678ddae 100755 --- a/lib/stage/image_validator.rb +++ b/lib/stage/image_validator.rb @@ -11,7 +11,7 @@ class ImageValidator < Stage BITONAL_RES = 600 CONTONE_RES = 400 - def run(agenda) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength + def run(agenda) return unless agenda.any? tiff_files = image_files.select { |file| agenda.include? file.objid } @@ -36,7 +36,7 @@ def run(agenda) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, private # Run tiffinfo command and return output text block - def run_tiffinfo(image_file) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def run_tiffinfo(image_file) begin info = TIFF.new(image_file.path).info rescue => e @@ -58,7 +58,7 @@ def run_tiffinfo(image_file) # rubocop:disable Metrics/AbcSize, Metrics/MethodLe # 'Samples/Pixel' line -> spp # bps of 1 requires spp=1 and xres=BITONAL_RES and yres=BITONAL_RES # bps of 8 requires spp in [1,3,4] and xres=CONTONE_RES and yres=CONTONE_RES - def evaluate_tiff(image_file, info) # rubocop:disable Metrics/MethodLength + def evaluate_tiff(image_file, info) if info[:res_unit] != "pixels/inch" image_error image_file, "must have pixels/inch, not #{info[:res_unit]}" end @@ -91,7 +91,7 @@ def evaluate_tiff_8_bps(image_file, info) end end - def run_jp2info(image_file) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def run_jp2info(image_file) begin info = JP2.new(image_file.path).info rescue => e diff --git a/lib/stage/pagination_check.rb b/lib/stage/pagination_check.rb index 8151a7d..20febeb 100755 --- a/lib/stage/pagination_check.rb +++ b/lib/stage/pagination_check.rb @@ -41,7 +41,7 @@ def pages_in_dir(dir) # Given sorted pages array of integers from 1 to n inclusive, # returns array of integer and integer range strings missing from list - def missing_pages(pages) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def missing_pages(pages) missing = (1..pages.max).to_a - pages ranges = [] missing.each do |m| diff --git a/lib/stage/postflight.rb b/lib/stage/postflight.rb index cda3232..da0d4df 100755 --- a/lib/stage/postflight.rb +++ b/lib/stage/postflight.rb @@ -29,7 +29,7 @@ def steps(agenda) shipment.checksums.keys.count end - def check_objid_lists # rubocop:disable Metrics/AbcSize + def check_objid_lists s1 = Set.new shipment.metadata[:initial_barcodes] s2 = Set.new shipment.objids if (s1 - s2).any? @@ -40,7 +40,7 @@ def check_objid_lists # rubocop:disable Metrics/AbcSize add_error Error.new("objids added: #{(s2 - s1).to_a.join(", ")}") end - def verify_source_checksums # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def verify_source_checksums fixity = shipment.fixity_check do |image_file| @bar.next! image_file.objid_file end diff --git a/lib/stage/preflight.rb b/lib/stage/preflight.rb index 4e586af..8f9565d 100755 --- a/lib/stage/preflight.rb +++ b/lib/stage/preflight.rb @@ -31,7 +31,7 @@ def self.ignorable_files IGNORABLE_FILES end - def run(agenda) # rubocop:disable Metrics/AbcSize + def run(agenda) shipment.metadata[:initial_barcodes] = shipment.objids if shipment.metadata[:initial_barcodes].none? add_error Error.new("no objids in #{shipment_directory}") @@ -63,7 +63,7 @@ def validate_objects(agenda) # A shipment directory is valid if it contains only objid directories, # a source directory, and status.json - def validate_shipment_directory # rubocop:disable Metrics/MethodLength + def validate_shipment_directory Dir.entries(shipment_directory).sort.each do |entry| next if %w[. .. source tmp status.json].include? entry @@ -82,7 +82,7 @@ def validate_shipment_directory # rubocop:disable Metrics/MethodLength # objid directory must include one or more TIFF files, # and a few other exceptions grandfathered by just_do_everything.sh # No directories are allowed - def validate_objid_directory(objid) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def validate_objid_directory(objid) have_image = false objid_directory = shipment.objid_directory(objid) Dir.entries(objid_directory).sort.each do |entry| diff --git a/lib/stage/tagger.rb b/lib/stage/tagger.rb index 83f40c0..dce81ab 100755 --- a/lib/stage/tagger.rb +++ b/lib/stage/tagger.rb @@ -25,7 +25,7 @@ def run(agenda) private - def calculate_tags # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def calculate_tags artist = config[:tagger_artist] || "dcu" @artist_tag ||= if TagData::ARTIST[artist].nil? add_warning Error.new("using custom artist '#{artist}'") @@ -58,7 +58,7 @@ def calculate_tags # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexi end end - def tag(image_file) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def tag(image_file) tagged_name = image_file.path.split(File::SEPARATOR)[-1] + ".tagged" tagged_path = File.join(tempdir_for_file(image_file), tagged_name) tagged = ImageFile.new(image_file.objid, tagged_path, @@ -101,7 +101,7 @@ def tempdir_for_file(image_file) @objid_to_tempdir[image_file.objid] = create_tempdir end - def run_tiffset(image_file, tag, value) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def run_tiffset(image_file, tag, value) begin info = TIFF.new(image_file.path).set(tag, value) rescue => e diff --git a/lib/string_color.rb b/lib/string_color.rb index 8541726..811ca9e 100755 --- a/lib/string_color.rb +++ b/lib/string_color.rb @@ -4,34 +4,92 @@ # String color methods # https://github.com/puppetlabs/quest/blob/master/lib/quest/colorization.rb -# rubocop:disable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs class String - def black; "\e[30m#{self}\e[0m" end - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def brown; "\e[33m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end - def magenta; "\e[35m#{self}\e[0m" end - def cyan; "\e[36m#{self}\e[0m" end - def gray; "\e[37m#{self}\e[0m" end - - def bg_black; "\e[40m#{self}\e[0m" end - def bg_red; "\e[41m#{self}\e[0m" end - def bg_green; "\e[42m#{self}\e[0m" end - def bg_brown; "\e[43m#{self}\e[0m" end - def bg_blue; "\e[44m#{self}\e[0m" end - def bg_magenta; "\e[45m#{self}\e[0m" end - def bg_cyan; "\e[46m#{self}\e[0m" end - def bg_gray; "\e[47m#{self}\e[0m" end - - def bold; "\e[1m#{self}\e[22m" end - def italic; "\e[3m#{self}\e[23m" end - def underline; "\e[4m#{self}\e[24m" end - def blink; "\e[5m#{self}\e[25m" end - def reverse_color; "\e[7m#{self}\e[27m" end + def black + "\e[30m#{self}\e[0m" + end + + def red + "\e[31m#{self}\e[0m" + end + + def green + "\e[32m#{self}\e[0m" + end + + def brown + "\e[33m#{self}\e[0m" + end + + def blue + "\e[34m#{self}\e[0m" + end + + def magenta + "\e[35m#{self}\e[0m" + end + + def cyan + "\e[36m#{self}\e[0m" + end + + def gray + "\e[37m#{self}\e[0m" + end + + def bg_black + "\e[40m#{self}\e[0m" + end + + def bg_red + "\e[41m#{self}\e[0m" + end + + def bg_green + "\e[42m#{self}\e[0m" + end + + def bg_brown + "\e[43m#{self}\e[0m" + end + + def bg_blue + "\e[44m#{self}\e[0m" + end + + def bg_magenta + "\e[45m#{self}\e[0m" + end + + def bg_cyan + "\e[46m#{self}\e[0m" + end + + def bg_gray + "\e[47m#{self}\e[0m" + end + + def bold + "\e[1m#{self}\e[22m" + end + + def italic + "\e[3m#{self}\e[23m" + end + + def underline + "\e[4m#{self}\e[24m" + end + + def blink + "\e[5m#{self}\e[25m" + end + + def reverse_color + "\e[7m#{self}\e[27m" + end def decolorize gsub(/\e\[\d\d?(;\d\d?)?m/, "") end end -# rubocop:enable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs diff --git a/lib/symbolize.rb b/lib/symbolize.rb index 54f723f..440c59c 100755 --- a/lib/symbolize.rb +++ b/lib/symbolize.rb @@ -3,7 +3,7 @@ # Based on https://gist.github.com/Integralist/9503099 module Symbolize - def self.symbolize(obj) # rubocop:disable Metrics/MethodLength + def self.symbolize(obj) case obj when Hash return obj.each_with_object({}) do |(k, v), memo| diff --git a/lib/tiff.rb b/lib/tiff.rb index bb08103..e714cd7 100755 --- a/lib/tiff.rb +++ b/lib/tiff.rb @@ -17,7 +17,7 @@ def initialize(path) end # Run tiffinfo command and return output text block - def info # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def info cmd = "tiffinfo #{@path}" status = Command.new(cmd).run tiffinfo = extract_fields(status[:stdout]) @@ -35,7 +35,7 @@ def info # rubocop:disable Metrics/AbcSize, Metrics/MethodLength tiffinfo end - def set(tag, value) # rubocop:disable Metrics/MethodLength + def set(tag, value) cmd = "tiffset -s #{tag} '#{value}' #{@path}" status = Command.new(cmd).run tiffset = {cmd: cmd, @@ -54,7 +54,7 @@ def set(tag, value) # rubocop:disable Metrics/MethodLength private - def extract_fields(info) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def extract_fields(info) h = {} m = info.match(/Resolution:\s(\d+(?:\.\d+)?),\s*(\d+(?:\.\d+)?)\s+(.*)/) unless m.nil? diff --git a/spec/compressor_spec.rb b/spec/compressor_spec.rb new file mode 100644 index 0000000..65ade81 --- /dev/null +++ b/spec/compressor_spec.rb @@ -0,0 +1,181 @@ +class FakeCompressionTool + def self.compress(sparse_path, new_path, tiffinfo) + FileUtils.cp(File.join("spec/fixtures/10_10_8_400.jp2"), new_path) + LogEntry.info(command: nil, time: nil) + end +end + +describe Compressor do + include_context "uses temp dir" + def tiffinfo(path) + `tiffinfo #{path}` + end + let(:log) { Log.new(objids: [objid]) } + let(:compression_tool) { FakeCompressionTool } + # image file has path, objid, objid_file, file + # objid="omzhx8s5.0074.149" + # path="/usr/src/app/test/shipments/DLXSCompressorTest_test_run_DLXS/omzhx8s5/0074/149/00000001.tif" + # objid_file="omzhx8s5/0074/149/00000001.tif" + # file="00000001.tif" + let(:path) { File.join("spec/fixtures", @image_file) } + let(:objid) { "some_barcode" } + let(:objid_file) { File.join(objid, @image_file) } + let(:image_file) { double("image_file", path: path, objid: objid, objid_file: objid_file, file: @image_file) } + let(:now) { Time.now } + let(:compressor) do + Compressor.for(image_file: image_file, tmpdir: Pathname(temp_dir), log: log, now: now) + end + context "color tif" do + before(:each) do + @image_file = "10_10_8_400.tif" + @log = Log.new + end + it "generates final document name" do + expect(compressor.document_name).to eq("some_barcode/10_10_8_400.jp2") + end + + it "is a Contone compressor when initialized with an 8bps image" do + expect(compressor.class.to_s).to eq("Compressor::Contone") + end + + context "#run" do + it "removes alpha when it exists" do + @image_file = "10_10_8_400_alpha.tif" + compressor.run(compression_tool) + expect(log.entries).to include(match("-alpha off")) + end + + it "ignores alpha when it doesn't exist" do + compressor.run(compression_tool) + expect(log.entries).not_to include(match("-alpha off")) + end + + it "strips tiff profile data when it exists" do + @image_file = "10_10_8_400_icc.tif" + compressor.run(compression_tool) + expect(log.entries).to include(match("-strip")) + end + + it "ignores tiff profile when it doesn't exist" do + compressor.run(compression_tool) + expect(log.entries).not_to include(match("-strip")) + end + + it "runs the compression tool" do + compressor.run + expect(log.entries).to include(match("kdu_compress")) + end + + it "copies original metadata to the jpeg2000" do + compressor.run + expect(log.entries).to include(match("tiff:Compression=JPEG 2000")) + end + + xit "copies original image datetime when present" do + end + + it "copies alphaless metadata to the jp2 when tiff has alpha" do + @image_file = "10_10_8_400_alpha.tif" + compressor.run(compression_tool) + expect(log.entries).to include(match("PhotometricInterpretation>XMP-tiff")).twice + end + end + end + context "bitonal tif" do + before(:each) do + @image_file = "10_10_1_600.tif" + end + it "is a Bitonal compressor when initialized with a 1bps image" do + expect(compressor.class.to_s).to eq("Compressor::Bitonal") + end + context "#run" do + it "runs the compression tool" do + compressor.run + expect(log.entries).to include(match("tifftopnm")) + end + it "runs copies the metadata from the original tiff to the compressed one" do + compressor.run + expect(log.entries).to include(match("exiftool -tagsFromFile #{compressor.image_file.path}")) + end + it "copies the first page of the tiff" do + @image_file = "10_10_1_600_2pg.tif" + starting_image_info = `tiffinfo #{path}` + expect(starting_image_info).to include("directory 1") + compressor.run + result_info = tiffinfo(compressor.output_path) + expect(result_info).not_to include("directory 1") + expect(log.entries).to include(match("tiffcp")) + end + + it "keeps the original datetime when the original image has one" do + tmpdir_image_path = File.join(Pathname(temp_dir), @image_file) + FileUtils.cp(path, tmpdir_image_path) + one_hour_ago = Time.now - 3600 + allow(image_file).to receive(:path).and_return(tmpdir_image_path) + TiffTools.set_tag(path: tmpdir_image_path, tag: :date_time, value: TiffTools.date_time_format(one_hour_ago)) + + compressor.run + result_info = tiffinfo(compressor.output_path) + expect(result_info).to include("DateTime: #{TiffTools.date_time_format(one_hour_ago)}") + end + + it "has now as the datetime when original tiff does not have a datetime" do + compressor.run + result_info = tiffinfo(compressor.output_path) + expect(result_info).to include("DateTime: #{now.strftime("%Y:%m:%d %H:%M:%S")}") + end + + it "has a document name of the original file" do + compressor.run + result_info = tiffinfo(compressor.output_path) + expect(result_info).to include("DocumentName: #{objid_file}") + end + + it "copies software from the original tiff to the output file" do + tmpdir_image_path = File.join(Pathname(temp_dir), @image_file) + FileUtils.cp(path, tmpdir_image_path) + allow(image_file).to receive(:path).and_return(tmpdir_image_path) + TiffTools.set_tag(path: tmpdir_image_path, tag: :software, value: "My Software") + compressor.run + result_software = TIFF.new(compressor.output_path).info[:software] + expect(result_software).to eq("My Software") + expect(log.entries).to include(match("exiftool -IFD0:Software=")) + end + it "logs a warning if there is no software in the original" do + compressor.run + result_software = TIFF.new(compressor.output_path).info[:software] + expect(result_software).to be_nil + expect(log.warnings).to_json include(match("could not extract software")) + end + end + end +end + +describe ImageMagick do + include_context "uses temp dir" + context "#remove_tiff_alpha" do + it "removes the alpha channel if it exists" do + tiff_path = "#{temp_dir}/input.tif" + FileUtils.copy("spec/fixtures/10_10_8_400_alpha.tif", tiff_path) + expect(TIFF.new(tiff_path).info[:alpha]).to eq(true) + ImageMagick.remove_tiff_alpha(tiff_path) + expect(TIFF.new(tiff_path).info[:alpha]).to eq(false) + end + end + + context "#strip_tiff_profiles" do + it "strips tiff profile data" do + tiff_path = "#{temp_dir}/input.tif" + FileUtils.copy("spec/fixtures/10_10_8_400_icc.tif", tiff_path) + expect(TIFF.new(tiff_path).info[:icc]).to eq(true) + ImageMagick.strip_tiff_profiles(tiff_path) + expect(TIFF.new(tiff_path).info[:icc]).to eq(false) + end + + it "handles warnings" do + tiff_path = "spec/fixtures/10_10_8_400_fake.tif" + log_entry = ImageMagick.strip_tiff_profiles(tiff_path) + expect(log_entry.level).to eq(:warning) + end + end +end diff --git a/spec/fixtures/10_10_1_16bps_400.tif b/spec/fixtures/10_10_1_16bps_400.tif new file mode 100644 index 0000000..624997f Binary files /dev/null and b/spec/fixtures/10_10_1_16bps_400.tif differ diff --git a/spec/fixtures/10_10_1_600.tif b/spec/fixtures/10_10_1_600.tif new file mode 100644 index 0000000..10c7f60 Binary files /dev/null and b/spec/fixtures/10_10_1_600.tif differ diff --git a/spec/fixtures/10_10_1_600_2pg.tif b/spec/fixtures/10_10_1_600_2pg.tif new file mode 100644 index 0000000..c313e29 Binary files /dev/null and b/spec/fixtures/10_10_1_600_2pg.tif differ diff --git a/spec/fixtures/10_10_8_400.jp2 b/spec/fixtures/10_10_8_400.jp2 new file mode 100644 index 0000000..0f7d004 Binary files /dev/null and b/spec/fixtures/10_10_8_400.jp2 differ diff --git a/spec/fixtures/10_10_8_400.tif b/spec/fixtures/10_10_8_400.tif new file mode 100644 index 0000000..5be7616 Binary files /dev/null and b/spec/fixtures/10_10_8_400.tif differ diff --git a/spec/fixtures/10_10_8_400_alpha.tif b/spec/fixtures/10_10_8_400_alpha.tif new file mode 100644 index 0000000..f0f471d Binary files /dev/null and b/spec/fixtures/10_10_8_400_alpha.tif differ diff --git a/spec/fixtures/10_10_8_400_compressed.tif b/spec/fixtures/10_10_8_400_compressed.tif new file mode 100644 index 0000000..133ffbb Binary files /dev/null and b/spec/fixtures/10_10_8_400_compressed.tif differ diff --git a/spec/fixtures/10_10_8_400_icc.tif b/spec/fixtures/10_10_8_400_icc.tif new file mode 100644 index 0000000..29560e0 Binary files /dev/null and b/spec/fixtures/10_10_8_400_icc.tif differ diff --git a/spec/fixtures/README.txt b/spec/fixtures/README.txt new file mode 100644 index 0000000..21d75b9 --- /dev/null +++ b/spec/fixtures/README.txt @@ -0,0 +1,12 @@ +Images in this directory: + +GOOD +==== +10_10_1_600.tif 10x10 1bps 600ppi +10_10_8_400.tif 10x10 8bps 400ppi +10_10_8_400_compressed.tif 10x10 8bps 400ppi +10_10_8_400.jp2 10x10 8bps 400ppi + +BAD +=== +10_10_1_16bps_400.tif 10x10 16bps 400 ppi diff --git a/spec/fixtures/sRGB2014.icc b/spec/fixtures/sRGB2014.icc new file mode 100644 index 0000000..49afbfe Binary files /dev/null and b/spec/fixtures/sRGB2014.icc differ diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..dc9367e --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,109 @@ +require "dotenv" +require "byebug" +require "rspec/temp_dir" + +Dotenv.load +require_relative "../rsvp" +TEST_ROOT = File.expand_path(__dir__) +$LOAD_PATH << TEST_ROOT + +require "processor" + +@config = Config.new({no_progress: true}) + +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + + # The settings below are suggested to provide a good initial experience + # with RSpec, but feel free to customize to your heart's content. + # # This allows you to limit a spec run to individual examples or groups + # # you care about by tagging them with `:focus` metadata. When nothing + # # is tagged with `:focus`, all examples get run. RSpec also provides + # # aliases for `it`, `describe`, and `context` that include `:focus` + # # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + # config.filter_run_when_matching :focus + # + # # Allows RSpec to persist some state between runs in order to support + # # the `--only-failures` and `--next-failure` CLI options. We recommend + # # you configure your source control system to ignore this file. + # config.example_status_persistence_file_path = "spec/examples.txt" + # + # # Limits the available syntax to the non-monkey patched syntax that is + # # recommended. For more details, see: + # # https://rspec.info/features/3-12/rspec-core/configuration/zero-monkey-patching-mode/ + # config.disable_monkey_patching! + # + # # This setting enables warnings. It's recommended, but in some cases may + # # be too noisy due to issues in dependencies. + # config.warnings = true + # + # # Many RSpec users commonly either run the entire suite or an individual + # # file, and it's useful to allow more verbose output when running an + # # individual spec file. + # if config.files_to_run.one? + # # Use the documentation formatter for detailed output, + # # unless a formatter has already been configured + # # (e.g. via a command-line flag). + # config.default_formatter = "doc" + # end + # + # # Print the 10 slowest examples and example groups at the + # # end of the spec run, to help surface which specs are running + # # particularly slow. + # config.profile_examples = 10 + # + # # Run specs in random order to surface order dependencies. If you find an + # # order dependency and want to debug it, you can fix the order by providing + # # the seed, which is printed after each run. + # # --seed 1234 + # config.order = :random + # + # # Seed global randomization in this process using the `--seed` CLI option. + # # Setting this allows you to use `--seed` to deterministically reproduce + # # test failures related to randomization by passing the same `--seed` value + # # as the one that triggered the failure. + # Kernel.srand config.seed +end diff --git a/test/agenda_test.rb b/test/agenda_test.rb index 890994e..0ded431 100755 --- a/test/agenda_test.rb +++ b/test/agenda_test.rb @@ -32,7 +32,7 @@ def self.gen_for_stage generate_tests "for_stage", test_proc end - def self.gen_update # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_update test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| spec = "BC T contone 1 BC T contone 1" test_shipment = test_shipment_class.new(dir, spec) @@ -49,7 +49,7 @@ def self.gen_update # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "update", test_proc end - def self.gen_update_fatal_error # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_update_fatal_error test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| spec = "BC T contone 1 BC T contone 1" test_shipment = test_shipment_class.new(dir, spec) diff --git a/test/compressor_test.rb b/test/compressor_test.rb index 5f70e13..b933c02 100755 --- a/test/compressor_test.rb +++ b/test/compressor_test.rb @@ -2,10 +2,11 @@ # frozen_string_literal: true require "minitest/autorun" -require "compressor" +require_relative "test_helper" +require "compression" require "fixtures" -class CompressorTest < Minitest::Test # rubocop:disable Metrics/ClassLength +class CompressionTest < Minitest::Test def setup @config = Config.new({no_progress: true}) end @@ -18,17 +19,17 @@ def self.gen_new test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir) shipment = shipment_class.new(test_shipment.directory) - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) refute_nil stage, "stage successfully created" } generate_tests "new", test_proc end - def self.gen_run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_run test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2") shipment = shipment_class.new(test_shipment.directory) - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! assert_equal(0, stage.errors.count, "stage runs without errors") tiff = File.join(shipment.directory, @@ -43,23 +44,7 @@ def self.gen_run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "run", test_proc end - def self.gen_set_tiff_date_time # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - test_proc = proc { |shipment_class, test_shipment_class, dir, opts| - test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") - shipment = shipment_class.new(test_shipment.directory) - tiff = File.join(shipment.directory, - shipment.objid_to_path(shipment.objids[0]), - "00000001.tif") - stage = Compressor.new(shipment, config: opts.merge(@config)) - stage.send(:write_tiff_date_time, tiff) - tiffinfo = `tiffinfo #{tiff}` - assert_match(/DateTime:\s\d{4}:\d{2}:\d{2}\s\d{2}:\d{2}:\d{2}/, tiffinfo, - "TIFF DateTime in %Y:%m:%d %H:%M:%S format") - } - generate_tests "set_tiff_date_time", test_proc - end - - def self.gen_set_jp2_date_time # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_set_jp2_date_time test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) @@ -67,7 +52,7 @@ def self.gen_set_jp2_date_time # rubocop:disable Metrics/AbcSize, Metrics/Method shipment.objid_to_path(shipment.objids[0]), "00000001.tif") `tiffset -s 306 '2000:11:11 11:11:11' #{tiff}` - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! jp2 = File.join(shipment.directory, shipment.objid_to_path(shipment.objids[0]), @@ -79,11 +64,11 @@ def self.gen_set_jp2_date_time # rubocop:disable Metrics/AbcSize, Metrics/Method generate_tests "set_jp2_date_time", test_proc end - def self.gen_set_jp2_document_name # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_set_jp2_document_name test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! jp2 = File.join(shipment.directory, shipment.objid_to_path(shipment.objids[0]), @@ -101,7 +86,7 @@ def self.gen_16bps_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bad_16bps 1") shipment = shipment_class.new(test_shipment.directory) - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! assert_equal(1, stage.errors.count, "stage fails with 16bps TIFF") assert_match(/invalid source tiff/i, stage.errors[0].description, @@ -114,7 +99,7 @@ def self.gen_zero_length_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC F 00000001.tif") shipment = shipment_class.new(test_shipment.directory) - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! # Error description may be tiffinfo exit code or something more detailed. assert_equal(1, stage.errors.count, "stage fails with zero-length TIFF") @@ -122,7 +107,7 @@ def self.gen_zero_length_fails generate_tests "zero_length_fails", test_proc end - def self.gen_alpha_channel # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_alpha_channel test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) @@ -130,14 +115,14 @@ def self.gen_alpha_channel # rubocop:disable Metrics/AbcSize, Metrics/MethodLeng shipment.objid_to_path(shipment.objids[0]), "00000001.tif") `convert #{tiff} -alpha on #{tiff}` - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! assert_equal(0, stage.errors.count, "stage runs without errors") } generate_tests "alpha_channel", test_proc end - def self.gen_icc_profile # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_icc_profile test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) @@ -146,14 +131,14 @@ def self.gen_icc_profile # rubocop:disable Metrics/AbcSize, Metrics/MethodLength "00000001.tif") profile_path = File.join(Fixtures::TEST_FIXTURES_PATH, "sRGB2014.icc") `convert #{tiff} -profile #{profile_path} #{tiff}` - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! assert_equal(0, stage.errors.count, "stage runs without errors") } generate_tests "icc_profile", test_proc end - def self.gen_software # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_software test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -161,7 +146,7 @@ def self.gen_software # rubocop:disable Metrics/AbcSize, Metrics/MethodLength shipment.objid_to_path(shipment.objids[0]), "00000001.tif") `tiffset -s 305 'BOGUS SOFTWARE v1.0' #{tiff}` - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! assert_equal(0, stage.errors.count, "stage runs without errors") assert_match(/BOGUS\sSOFTWARE/, `tiffinfo #{tiff}`, diff --git a/test/config/config.dlxs.yml b/test/config/config.dlxs.yml index c19d92b..2f7b313 100644 --- a/test/config/config.dlxs.yml +++ b/test/config/config.dlxs.yml @@ -12,9 +12,9 @@ stages: - name: Tagger class: Tagger file: tagger - - name: Compressor - class: Compressor - file: compressor + - name: Compression + class: Compression + file: compression - name: DLXSCompressor class: DLXSCompressor file: dlxs_compressor diff --git a/test/config/config.yml b/test/config/config.yml index ed3d97b..b4963a7 100644 --- a/test/config/config.yml +++ b/test/config/config.yml @@ -12,9 +12,9 @@ stages: - name: Tagger class: Tagger file: tagger - - name: Compressor - class: Compressor - file: compressor + - name: Compression + class: Compression + file: compression - name: Postflight class: Postflight file: postflight diff --git a/test/dlxs_compressor_test.rb b/test/dlxs_compressor_test.rb index 70b47e2..4f793bd 100755 --- a/test/dlxs_compressor_test.rb +++ b/test/dlxs_compressor_test.rb @@ -16,11 +16,11 @@ def self.gen_new generate_tests "new", test_proc end - def self.gen_run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_run test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) - stage = Compressor.new(shipment, config: opts.merge(@config)) + stage = Compression.new(shipment, config: opts.merge(@config)) stage.run! assert_equal(0, stage.errors.count, "compressor runs without errors") stage = DLXSCompressor.new(shipment, config: opts.merge(@config)) diff --git a/test/error_test.rb b/test/error_test.rb index f543f31..f62dee0 100755 --- a/test/error_test.rb +++ b/test/error_test.rb @@ -31,9 +31,7 @@ def test_to_s def test_json err = Error.new("some error", "12345678", "00000001.tif") - # rubocop:disable Security/JSONLoad err2 = JSON.unsafe_load(err.to_json) - # rubocop:enable Security/JSONLoad assert err.description == err2.description && err.objid == err2.objid && err.path == err2.path, diff --git a/test/fixtures.rb b/test/fixtures.rb index f2882fb..9d497a4 100755 --- a/test/fixtures.rb +++ b/test/fixtures.rb @@ -13,7 +13,7 @@ module Fixtures }, bad_16bps: { file: "10_10_1_16bps_400.tif", - description: "16bps image that will fail Image Validator and Compressor" + description: "16bps image that will fail Image Validator and Compression" } }.freeze JP2_FIXTURES = { diff --git a/test/image_validator_test.rb b/test/image_validator_test.rb index 053f2df..46aa2cc 100755 --- a/test/image_validator_test.rb +++ b/test/image_validator_test.rb @@ -4,7 +4,7 @@ require "minitest/autorun" require "image_validator" -class ImageValidatorTest < Minitest::Test # rubocop:disable Metrics/ClassLength +class ImageValidatorTest < Minitest::Test def setup @config = Config.new({no_progress: true}) end @@ -45,7 +45,7 @@ def self.gen_16bps_fails generate_tests "16bps_fails", test_proc end - def self.gen_pixelspercentimeter_fails # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_pixelspercentimeter_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -61,7 +61,7 @@ def self.gen_pixelspercentimeter_fails # rubocop:disable Metrics/AbcSize, Metric generate_tests "pixelspercentimeter_fails", test_proc end - def self.gen_bitonal_3spp_fails # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_bitonal_3spp_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -77,7 +77,7 @@ def self.gen_bitonal_3spp_fails # rubocop:disable Metrics/AbcSize, Metrics/Metho generate_tests "bitonal_3spp_fails", test_proc end - def self.gen_bitonal_resolution_fails # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_bitonal_resolution_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -93,7 +93,7 @@ def self.gen_bitonal_resolution_fails # rubocop:disable Metrics/AbcSize, Metrics generate_tests "bitonal_resolution_fails", test_proc end - def self.gen_contone_2spp_fails # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_contone_2spp_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) @@ -109,7 +109,7 @@ def self.gen_contone_2spp_fails # rubocop:disable Metrics/AbcSize, Metrics/Metho generate_tests "contone_2spp_fails", test_proc end - def self.gen_contone_resolution_fails # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_contone_resolution_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) @@ -125,7 +125,7 @@ def self.gen_contone_resolution_fails # rubocop:disable Metrics/AbcSize, Metrics generate_tests "contone_resolution_fails", test_proc end - def self.gen_garbage_tiff_fails # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_garbage_tiff_fails test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) diff --git a/test/jhove_test.rb b/test/jhove_test.rb index 3693f97..e283b3e 100755 --- a/test/jhove_test.rb +++ b/test/jhove_test.rb @@ -43,7 +43,7 @@ def self.gen_run generate_tests "run", test_proc end - def self.gen_error # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_error test_proc = proc { |shipment_class, test_shipment_class, dir, opts| setup_test(shipment_class, test_shipment_class, dir, opts) tiff = File.join(@shipment.objid_to_path(@shipment.objids[0]), @@ -64,7 +64,7 @@ def self.gen_error # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "error", test_proc end - def self.gen_error_fields # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_error_fields test_proc = proc { |shipment_class, test_shipment_class, dir, opts| setup_test(shipment_class, test_shipment_class, dir, opts) ENV["FAKE_FEED_VALIDATE_LONG"] = "1" diff --git a/test/jp2_test.rb b/test/jp2_test.rb index ad6305f..2f35e15 100755 --- a/test/jp2_test.rb +++ b/test/jp2_test.rb @@ -11,7 +11,7 @@ def test_new refute_nil jp2, "JP2 is not nil" end - def test_info # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def test_info shipment = TestShipment.new(test_name, "BC J contone 1") jp2 = JP2.new(shipment.image_files.first.path) info = jp2.info diff --git a/test/pagination_check_test.rb b/test/pagination_check_test.rb index eea2d40..d7ce891 100755 --- a/test/pagination_check_test.rb +++ b/test/pagination_check_test.rb @@ -27,7 +27,7 @@ def self.gen_run generate_tests "run", test_proc end - def self.gen_missing # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_missing test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1-2 T bitonal 4-5" test_shipment = test_shipment_class.new(dir, spec) @@ -43,7 +43,7 @@ def self.gen_missing # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "missing", test_proc end - def self.gen_missing_range # rubocop:disable Metrics/AbcSize + def self.gen_missing_range test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1 T bitonal 5" test_shipment = test_shipment_class.new(dir, spec) @@ -56,7 +56,7 @@ def self.gen_missing_range # rubocop:disable Metrics/AbcSize generate_tests "missing_range", test_proc end - def self.gen_duplicate # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_duplicate test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1 J contone 1" test_shipment = test_shipment_class.new(dir, spec) diff --git a/test/postflight_test.rb b/test/postflight_test.rb index 0b3b909..d3e40c7 100755 --- a/test/postflight_test.rb +++ b/test/postflight_test.rb @@ -4,7 +4,7 @@ require "minitest/autorun" require "postflight" -class PostflightTest < Minitest::Test # rubocop:disable Metrics/ClassLength +class PostflightTest < Minitest::Test def setup opts = {no_progress: true, feed_validate_script: "test/bin/fake_feed_validate.pl"} @@ -25,7 +25,7 @@ def self.gen_new generate_tests "new", test_proc end - def self.gen_run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_run test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1 T contone 2" test_shipment = test_shipment_class.new(dir, spec) @@ -39,7 +39,7 @@ def self.gen_run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "run", test_proc end - def self.gen_metadata_mismatch_removed # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_metadata_mismatch_removed test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1 BC T bitonal 1" test_shipment = test_shipment_class.new(dir, spec) @@ -57,7 +57,7 @@ def self.gen_metadata_mismatch_removed # rubocop:disable Metrics/AbcSize, Metric generate_tests "metadata_mismatch_removed", test_proc end - def self.gen_metadata_mismatch_added # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_metadata_mismatch_added test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1 BC T bitonal 1" test_shipment = test_shipment_class.new(dir, spec) @@ -75,7 +75,7 @@ def self.gen_metadata_mismatch_added # rubocop:disable Metrics/AbcSize, Metrics/ generate_tests "metadata_mismatch_added", test_proc end - def self.gen_feed_validate_error # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_feed_validate_error test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2") shipment = shipment_class.new(test_shipment.directory) @@ -100,7 +100,7 @@ def self.gen_feed_validate_error # rubocop:disable Metrics/AbcSize, Metrics/Meth generate_tests "feed_validate_error", test_proc end - def self.gen_feed_validate_crash # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_feed_validate_crash test_proc = proc { |shipment_class, test_shipment_class, dir, opts| ENV["FAKE_FEED_VALIDATE_CRASH"] = "1" test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2") @@ -116,7 +116,7 @@ def self.gen_feed_validate_crash # rubocop:disable Metrics/AbcSize, Metrics/Meth generate_tests "feed_validate_crash", test_proc end - def self.gen_checksum_mismatch # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_checksum_mismatch test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2") shipment = shipment_class.new(test_shipment.directory) @@ -134,7 +134,7 @@ def self.gen_checksum_mismatch # rubocop:disable Metrics/AbcSize, Metrics/Method generate_tests "checksum_mismatch", test_proc end - def self.gen_file_missing # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_file_missing test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2") shipment = shipment_class.new(test_shipment.directory) @@ -152,7 +152,7 @@ def self.gen_file_missing # rubocop:disable Metrics/AbcSize, Metrics/MethodLengt generate_tests "file_missing", test_proc end - def self.gen_file_added # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_file_added test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2") shipment = shipment_class.new(test_shipment.directory) diff --git a/test/preflight_test.rb b/test/preflight_test.rb index 6de3e9b..bd8f5b0 100755 --- a/test/preflight_test.rb +++ b/test/preflight_test.rb @@ -4,7 +4,7 @@ require "minitest/autorun" require "preflight" -class PreflightTest < Minitest::Test # rubocop:disable Metrics/ClassLength +class PreflightTest < Minitest::Test def setup @config = Config.new({no_progress: true}) end @@ -23,7 +23,7 @@ def self.gen_new generate_tests "new", test_proc end - def self.gen_run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_run test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1 BC T bitonal 1" test_shipment = test_shipment_class.new(dir, spec) @@ -52,7 +52,7 @@ def self.gen_validate_objid generate_tests "validate_objid", test_proc end - def self.gen_remove_ds_store # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_remove_ds_store test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -70,7 +70,7 @@ def self.gen_remove_ds_store # rubocop:disable Metrics/AbcSize, Metrics/MethodLe generate_tests "remove_ds_store", test_proc end - def self.gen_remove_thumbs_db # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_remove_thumbs_db test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -88,7 +88,7 @@ def self.gen_remove_thumbs_db # rubocop:disable Metrics/AbcSize, Metrics/MethodL generate_tests "remove_thumbs_db", test_proc end - def self.gen_remove_toplevel_ds_store # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_remove_toplevel_ds_store test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -104,7 +104,7 @@ def self.gen_remove_toplevel_ds_store # rubocop:disable Metrics/AbcSize, Metrics generate_tests "remove_toplevel_ds_store", test_proc end - def self.gen_remove_toplevel_thumbs_db # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_remove_toplevel_thumbs_db test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -122,7 +122,7 @@ def self.gen_remove_toplevel_thumbs_db # rubocop:disable Metrics/AbcSize, Metric generate_tests "remove_toplevel_thumbs_db", test_proc end - def self.gen_objid_directory_errors_and_warnings # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_objid_directory_errors_and_warnings test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC F spurious_file") shipment = shipment_class.new(test_shipment.directory) @@ -146,7 +146,7 @@ def self.gen_objid_directory_errors_and_warnings # rubocop:disable Metrics/AbcSi generate_tests "objid_directory_errors_and_warnings", test_proc end - def self.gen_shipment_directory_errors # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_shipment_directory_errors test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "F spurious_file") shipment = shipment_class.new(test_shipment.directory) diff --git a/test/processor_test.rb b/test/processor_test.rb index 7450232..fb11675 100755 --- a/test/processor_test.rb +++ b/test/processor_test.rb @@ -7,7 +7,7 @@ require "processor" require "fixtures" -class ProcessorTest < Minitest::Test # rubocop:disable Metrics/ClassLength +class ProcessorTest < Minitest::Test def setup @options = {config_dir: File.join(TEST_ROOT, "config")} end @@ -16,7 +16,7 @@ def teardown TestShipment.remove_test_shipments end - def self.gen_new # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_new test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir) processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -53,7 +53,7 @@ def self.gen_stages generate_tests "stages", test_proc end - def self.gen_run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_run test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC F .DS_Store") options = {no_progress: true} @@ -86,7 +86,7 @@ def self.gen_invalid_status_file # Don't pass TestShipment to anything we want to serialize -- # the initializer isn't JSON-aware - def self.gen_reload_status_file # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_reload_status_file test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bad_16bps 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -104,7 +104,7 @@ def self.gen_reload_status_file # rubocop:disable Metrics/AbcSize, Metrics/Metho # Don't pass TestShipment to anything we want to serialize -- # the initializer isn't JSON-aware - def self.gen_move_status_file # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_move_status_file test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -120,7 +120,7 @@ def self.gen_move_status_file # rubocop:disable Metrics/AbcSize, Metrics/MethodL generate_tests "move_status_file", test_proc end - def self.gen_restore_from_source_directory # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_restore_from_source_directory test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -141,7 +141,7 @@ def self.gen_restore_from_source_directory # rubocop:disable Metrics/AbcSize, Me generate_tests "restore_from_source_directory", test_proc end - def self.gen_finalize # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_finalize test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -159,7 +159,7 @@ def self.gen_finalize # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "finalize", test_proc end - def self.gen_finalize_does_nothing # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_finalize_does_nothing test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bad_16bps 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -177,7 +177,7 @@ def self.gen_finalize_does_nothing # rubocop:disable Metrics/AbcSize, Metrics/Me generate_tests "finalize_does_nothing", test_proc end - def self.gen_restart_finalized # rubocop:disable Metrics/MethodLength + def self.gen_restart_finalized test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -208,7 +208,7 @@ def teardown # Initial run detects bogus file, replacement allows second run to pass, # and fixity is updated with the new file. - def self.gen_error_correction # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_error_correction test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| spec = "BC T contone 1 BC T bad_16bps 1" test_shipment = test_shipment_class.new(dir, spec) diff --git a/test/query_tool_test.rb b/test/query_tool_test.rb index 8ef9ece..6ca6a44 100755 --- a/test/query_tool_test.rb +++ b/test/query_tool_test.rb @@ -5,7 +5,7 @@ require "stringio" require "query_tool" -class QueryToolTest < Minitest::Test # rubocop:disable Metrics/ClassLength +class QueryToolTest < Minitest::Test def setup @options = {config_dir: File.join(TEST_ROOT, "config"), no_progress: true} @@ -25,7 +25,7 @@ def self.gen_new generate_tests "new", test_proc end - def self.gen_agenda_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_agenda_cmd test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -42,7 +42,7 @@ def self.gen_agenda_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "agenda_cmd", test_proc end - def self.gen_agenda_cmd_no_agenda # rubocop:disable Metrics/MethodLength + def self.gen_agenda_cmd_no_agenda test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -71,7 +71,7 @@ def self.gen_objids_cmd generate_tests "objids_cmd", test_proc end - def self.gen_objids_cmd_with_errors # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_objids_cmd_with_errors test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bad_16bps 1") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -88,7 +88,7 @@ def self.gen_objids_cmd_with_errors # rubocop:disable Metrics/AbcSize, Metrics/M generate_tests "objids_cmd_with_errors", test_proc end - def self.gen_errors_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_errors_cmd test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| spec = "BC T contone 1 BC T contone 1" test_shipment = test_shipment_class.new(dir, spec) @@ -116,7 +116,7 @@ def self.gen_errors_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "errors_cmd", test_proc end - def self.gen_warnings_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_warnings_cmd test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| spec = "BC T contone 1 BC T contone 1" test_shipment = test_shipment_class.new(dir, spec) @@ -167,7 +167,7 @@ def self.gen_status_cmd_err generate_tests "status_cmd_err", test_proc end - def self.gen_fixity_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_fixity_cmd test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T contone 2-3") processor = Processor.new(test_shipment.directory, opts.merge(@options)) @@ -199,7 +199,7 @@ def self.gen_fixity_cmd # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "fixity_cmd", test_proc end - def self.gen_fixity_cmd_not_yet_populated # rubocop:disable Metrics/MethodLength + def self.gen_fixity_cmd_not_yet_populated test_proc = proc { |_shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC") processor = Processor.new(test_shipment.directory, opts.merge(@options)) diff --git a/test/shipment_test.rb b/test/shipment_test.rb index 8dd6b58..ef6245c 100755 --- a/test/shipment_test.rb +++ b/test/shipment_test.rb @@ -4,7 +4,7 @@ require "minitest/autorun" require "shipment" -class ShipmentTest < Minitest::Test # rubocop:disable Metrics/ClassLength +class ShipmentTest < Minitest::Test def teardown TestShipment.remove_test_shipments end @@ -31,7 +31,7 @@ def self.gen_directory generate_tests "directory", test_proc end - def self.gen_path_components_from_objid # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_path_components_from_objid test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| test_shipment = test_shipment_class.new(dir, "BC") shipment = shipment_class.new(test_shipment.directory) @@ -48,7 +48,7 @@ def self.gen_path_components_from_objid # rubocop:disable Metrics/AbcSize, Metri generate_tests "objid_to_path", test_proc end - def self.gen_source_directory # rubocop:disable Metrics/MethodLength + def self.gen_source_directory test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| test_shipment = test_shipment_class.new(dir) shipment = shipment_class.new(test_shipment.directory) @@ -62,7 +62,7 @@ def self.gen_source_directory # rubocop:disable Metrics/MethodLength generate_tests "source_directory", test_proc end - def self.gen_tmp_directory # rubocop:disable Metrics/MethodLength + def self.gen_tmp_directory test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| test_shipment = test_shipment_class.new(dir) shipment = shipment_class.new(test_shipment.directory) @@ -88,7 +88,7 @@ def self.gen_source_objids generate_tests "source_objids", test_proc end - def self.gen_image_files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_image_files test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| spec = "BC T contone 1 T contone 2 BC T contone 1 BC" test_shipment = test_shipment_class.new(dir, spec) @@ -102,7 +102,7 @@ def self.gen_image_files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "image_files", test_proc end - def self.gen_setup_source_directory # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_setup_source_directory test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) @@ -118,7 +118,7 @@ def self.gen_setup_source_directory # rubocop:disable Metrics/AbcSize, Metrics/M generate_tests "setup_source_directory", test_proc end - def self.gen_restore_from_source_directory # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_restore_from_source_directory test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) @@ -136,7 +136,7 @@ def self.gen_restore_from_source_directory # rubocop:disable Metrics/AbcSize, Me generate_tests "restore_from_source_directory", test_proc end - def self.gen_partial_restore_from_source_directory # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_partial_restore_from_source_directory test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| spec = "BC T contone 1 BC T contone 1" test_shipment = test_shipment_class.new(dir, spec) @@ -171,8 +171,8 @@ def self.gen_restore_from_nonexistent_source_directory generate_tests "restore_from_nonexistent_source_directory", test_proc end - def self.gen_fixity_check # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| # rubocop:disable Metrics/BlockLength + def self.gen_fixity_check + test_proc = proc { |shipment_class, test_shipment_class, dir, _opts| test_shipment = test_shipment_class.new(dir, "BC T contone 2-3") shipment = shipment_class.new(test_shipment.directory) shipment.setup_source_directory @@ -211,7 +211,7 @@ def self.gen_fixity_check # rubocop:disable Metrics/AbcSize, Metrics/MethodLengt generate_tests "fixity_check", test_proc end - def self.gen_finalize # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_finalize test_proc = proc { |shipment_class, test_shipment_class, dir| test_shipment = test_shipment_class.new(dir, "BC T contone 1") shipment = shipment_class.new(test_shipment.directory) diff --git a/test/stage_test.rb b/test/stage_test.rb index 56850a2..58e515f 100755 --- a/test/stage_test.rb +++ b/test/stage_test.rb @@ -46,7 +46,7 @@ def self.gen_cleanup_tempdirs generate_tests "cleanup_tempdirs", test_proc end - def self.gen_cleanup_delete_on_success # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_cleanup_delete_on_success test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC") shipment = shipment_class.new(test_shipment.directory) diff --git a/test/tagger_test.rb b/test/tagger_test.rb index e434ed8..8aba380 100755 --- a/test/tagger_test.rb +++ b/test/tagger_test.rb @@ -23,7 +23,7 @@ def self.gen_new generate_tests "new", test_proc end - def self.gen_default_tags # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_default_tags test_proc = proc { |shipment_class, test_shipment_class, dir, opts| spec = "BC T bitonal 1 BC T bitonal 1" test_shipment = test_shipment_class.new(dir, spec) @@ -43,7 +43,7 @@ def self.gen_default_tags # rubocop:disable Metrics/AbcSize, Metrics/MethodLengt generate_tests "default_tags", test_proc end - def self.gen_artist_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_artist_tag test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -59,7 +59,7 @@ def self.gen_artist_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "artist_tag", test_proc end - def self.gen_scanner_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_scanner_tag test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -78,7 +78,7 @@ def self.gen_scanner_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength generate_tests "scanner_tag", test_proc end - def self.gen_software_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_software_tag test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -106,7 +106,7 @@ def teardown TestShipment.remove_test_shipments end - def self.gen_custom_artist_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_custom_artist_tag test_proc = proc { |shipment_class, test_shipment_class, dir, opts| artist = "University of Michigan: Secret Vaults" test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") @@ -126,7 +126,7 @@ def self.gen_custom_artist_tag # rubocop:disable Metrics/AbcSize, Metrics/Method generate_tests "custom_artist_tag", test_proc end - def self.gen_custom_scanner_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_custom_scanner_tag test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -149,7 +149,7 @@ def self.gen_custom_scanner_tag # rubocop:disable Metrics/AbcSize, Metrics/Metho generate_tests "custom_scanner_tag", test_proc end - def self.gen_bogus_custom_scanner_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_bogus_custom_scanner_tag test_proc = proc { |shipment_class, test_shipment_class, dir, opts| test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") shipment = shipment_class.new(test_shipment.directory) @@ -163,7 +163,7 @@ def self.gen_bogus_custom_scanner_tag # rubocop:disable Metrics/AbcSize, Metrics generate_tests "bogus_custom_scanner_tag", test_proc end - def self.gen_custom_software_tag # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def self.gen_custom_software_tag test_proc = proc { |shipment_class, test_shipment_class, dir, opts| software = "WhizzySoft ScanR v33" test_shipment = test_shipment_class.new(dir, "BC T bitonal 1") diff --git a/test/test_helper.rb b/test/test_helper.rb index 53cfa58..610de31 100755 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require "dotenv" +require "byebug" Dotenv.load @@ -49,7 +50,7 @@ def self.add_test(name) # for the Minitest::Test class invoking it. # Each test case in the Minitest::Test creates test pairs using this # routine and then invokes them with this at the end of the file: - def self.generate_tests(name, block) # rubocop:disable Metrics/MethodLength + def self.generate_tests(name, block) add_test name ["", "DLXS"].each do |type| method_name = "test_#{name}" diff --git a/test/test_shipment.rb b/test/test_shipment.rb index 5d51919..9d6e1cb 100755 --- a/test/test_shipment.rb +++ b/test/test_shipment.rb @@ -5,14 +5,14 @@ require "fixtures" require_relative "../lib/shipment" -class TestShipment < Shipment # rubocop:disable Metrics/ClassLength +class TestShipment < Shipment attr_reader :ordered_objids PATH = File.join(__dir__, "shipments").freeze # Yes, we want this shared with subclasses def self.test_shipments - @@test_shipments ||= [] # rubocop:disable Style/ClassVars + @@test_shipments ||= [] end def self.remove_test_shipments @@ -56,7 +56,7 @@ def initialize(name, spec = "") process_spec spec end - def process_spec(spec) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength + def process_spec(spec) @current_dir = @dir elements = spec.split(/\s+/) while elements.any? @@ -95,7 +95,7 @@ def handle_bogus_objid_op @ordered_objids << objid end - def handle_tiff_op(name, dest) # rubocop:disable Metrics/MethodLength + def handle_tiff_op(name, dest) fixture = Fixtures.tiff_fixture(name) case dest when /^\d+$/ @@ -113,7 +113,7 @@ def handle_tiff_op(name, dest) # rubocop:disable Metrics/MethodLength end end - def handle_jp2_op(name, dest) # rubocop:disable Metrics/MethodLength + def handle_jp2_op(name, dest) fixture = Fixtures.jp2_fixture(name) case dest when /^\d+$/ diff --git a/test/test_shipment_test.rb b/test/test_shipment_test.rb index 364a0af..ae60608 100755 --- a/test/test_shipment_test.rb +++ b/test/test_shipment_test.rb @@ -23,7 +23,7 @@ def test_invalid_objids end end - def test_generate_test_shipment_objid # rubocop:disable Metrics/AbcSize + def test_generate_test_shipment_objid shipment = TestShipment.new(test_name, "BC") assert_equal 1, shipment.objids.count, "correct number of objids" assert File.directory?(shipment.directory), "#{test_name} is directory" @@ -34,7 +34,7 @@ def test_generate_test_shipment_objid # rubocop:disable Metrics/AbcSize "objid #{shipment.objids[0]} valid" end - def test_generate_test_shipment_bogus_objid # rubocop:disable Metrics/AbcSize + def test_generate_test_shipment_bogus_objid shipment = TestShipment.new(test_name, "BBC") assert_equal 1, shipment.objids.count, "correct number of objids" assert File.directory?(shipment.directory), "#{test_name} is directory" @@ -104,7 +104,7 @@ def test_unknown_jp2_format end class DLXSTestShipmentTest < Minitest::Test - def test_generate_test_shipment_dlxs_objid # rubocop:disable Metrics/AbcSize + def test_generate_test_shipment_dlxs_objid shipment = DLXSTestShipment.new(test_name, "BC") assert_equal 1, shipment.ordered_objids.count, "correct number of ordered objids" diff --git a/test/tiff_test.rb b/test/tiff_test.rb index 781d69c..d2a00bf 100755 --- a/test/tiff_test.rb +++ b/test/tiff_test.rb @@ -11,7 +11,7 @@ def test_new refute_nil tiff, "TIFF is not nil" end - def test_info # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def test_info shipment = TestShipment.new(test_name, "BC T contone 1") tiff = TIFF.new(shipment.image_files.first.path) info = tiff.info