Cache the DTD file.

The URLs in the DOCTYPE instruction are replaced with a path to a local file,
stored in the cache of the program.

The main reason is to avoid 429 (HTTP) errors from www.gnu.org when the DTD is
downloaded in the saxonb command.

Author: ayosec

lib/ffdocs/source_docs/collection.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "digest/sha2"
+require "net/http"
 require "nokogiri"
 require "tempfile"
 
@@ -11,6 +12,8 @@ module FFDocs::SourceDocs
 
   PATCHES_DIR = Pathname.new(File.expand_path("../patches", __FILE__))
 
+  DTD_MUTEX = Mutex.new
+
   class SourceNotFound < StandardError; end
   class XMLTransformFailed < StandardError; end
 
@@ -78,6 +81,8 @@ def initialize(options, storage, release)
         io.read
       end
 
+      xml = cache_dtd(xml, storage)
+
       ::FFDocs.log.info "Parsing source for #{@release.version} ..."
       parse_xml(release, Nokogiri::XML.parse(xml))
     end
@@ -200,6 +205,32 @@ def initialize(options, storage, release)
       html
     end
 
+    # Replace the URLs for the DTD to use a cached file.
+    private def cache_dtd(xml, storage)
+      xml.sub(%r[<!DOCTYPE .*?>]) do |m|
+        m.gsub(/https?:[^"]+/) do |url|
+          cached = storage.dtd_file(url)
+
+          DTD_MUTEX.synchronize do
+            if not cached.exist?
+              url = url.sub(/^http:/, "https:")
+              ::FFDocs.log.info "Downloading DTD for #{url} ..."
+
+              case Net::HTTP.get_response(URI(url))
+              in Net::HTTPSuccess => response
+                cached.write(response.body)
+              in _ => failure
+                ::FFDocs.log.error "Failed request: #{url}: #{failure}"
+              end
+            end
+          end
+
+          cached.to_s
+        end
+      end
+    end
+
+
   end
 
 end

lib/ffdocs/storage.rb

@@ -1,3 +1,4 @@
+require "digest/sha2"
 require "json"
 require "logger"
 require "pathname"
@@ -43,6 +44,13 @@ def get_file(version, path)
     end
   end
 
+  def dtd_file(url)
+    dtd_dir = CACHE_DIR.join("dtd")
+    dtd_dir.mkpath if not dtd_dir.directory?
+
+    dtd_dir.join(Digest::SHA2.hexdigest(url) + ".dtd")
+  end
+
   module SyncData
 
     GIT_URL = "https://github.com/FFmpeg/FFmpeg.git"