Fix HAN, OSCM, and WUR scrapers broken by site redesigns

lib/ingestors/taxila/han_ingestor.rb

@@ -4,6 +4,9 @@
 module Ingestors
   module Taxila
     class HanIngestor < Ingestor
+      DATE_REGEXP = /\b\d{1,2}\s+(?:januari|februari|maart|april|mei|juni|juli|augustus|september|oktober|november|december)\s+\d{4}\b/i
+      VENUE_REGEXP = /\bHAN in ([^.]+)/i
+
       def self.config
         {
           key: 'han_event',
@@ -32,21 +35,25 @@ def process_han(_url)
         event_page.each_with_index do |el, _idx|
           event = OpenStruct.new
           event.title = el.css('.card-skinny__content__title').first.text
-          event.url = "https://www.han.nl#{el.css('.card-skinny__content__buttons > .buttons > .buttons__button > a').first.get_attribute('href')}"
+          href = el.css('.card-skinny__content__buttons > .buttons > .buttons__button > a').first.get_attribute('href')
+          event.url = href.start_with?('http') ? href : "https://www.han.nl#{href}"
           event.description = el.css('.card-skinny__content__body').first.text
 
           sleep(1) unless Rails.env.test? and File.exist?('test/vcr_cassettes/ingestors/han.yml')
           event_page2 = Nokogiri::HTML5.parse(open_url(event.url.to_s, raise: true))
-          start_str = event_page2.css("div[class='course-superhero__payoff__content']")[0].css('p > span.course-superhero__payoff__subtitle')[0].text
-          start_str = convert_months(start_str)
-          event.start = Time.zone.parse(start_str.split('en')[0].strip)
+
+          details_text = extract_details_text(event_page2)
+
+          start_str = details_text[DATE_REGEXP]
+          raise "no date found for #{event.url}" unless start_str
+
+          event.start = Time.zone.parse(convert_months(start_str))
           event.end = event.start
           event.set_default_times
 
-          course_details = event_page2.css("div[class='course-details__sidebar__item']")[0]
-          venue_sub_css = course_details.css("span[class='nav-subswitch__title__label__sub']")[0]
-          venue_super_css = venue_sub_css.parent.css("strong.nav-subswitch__title__label__super")[0]
-          event.venue = "#{venue_super_css.text} #{venue_sub_css.text}"
+          venue_match = details_text.match(VENUE_REGEXP)
+          event.venue = venue_match[1].strip if venue_match
+
           event.source = "HAN"
           event.timezone = 'Amsterdam'
 
@@ -56,6 +63,15 @@ def process_han(_url)
         end
       end
 
+      def extract_details_text(page)
+        section = page.css('.collapsible').find do |collapsible|
+          header = collapsible.css('.collapsible__header, button, h2, h3').first
+          header && header.text =~ /tijden|startdat/i
+        end
+        scope = section ? section.css('.collapsible__content').first : page
+        (scope || page).text.gsub(/\s+/, ' ').strip
+      end
+
       def convert_months(my_str)
         {
           'januari': 'january',
@@ -77,4 +93,4 @@ def convert_months(my_str)
       end
     end
   end
-end
+end

lib/ingestors/taxila/oscm_ingestor.rb

@@ -32,7 +32,7 @@ def process_oscm(url)
         # Instead of using the sitemap we use the events page.
         # The sitemap shows also past events, but the ical link for those does not work, so we can't parse them with the below code.
         url = 'https://www.openscience-maastricht.nl/events/'
-        Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('.eventname > a').each do |event_link|
+        Nokogiri::HTML5.parse(open_url(url.to_s, raise: true)).css('.events-table a').each do |event_link|
           begin
             event_url = event_link.attributes['href']
             event_page = Nokogiri::HTML5.parse(open_url(event_url, raise: true))

lib/ingestors/taxila/wur_ingestor.rb

@@ -1,5 +1,6 @@
 require 'csv'
 require 'nokogiri'
+require 'json'
 
 module Ingestors
   module Taxila
@@ -26,59 +27,22 @@ def read(url)
       private
 
       def process_wur(url)
-        docs = Nokogiri::XML(open_url(url, raise: true)).xpath('//item')
-        docs.each do |event_item|
-          begin
-            event = OpenStruct.new
-            event.event_types = ['workshops_and_courses']
-            event_item.element_children.each do |element|
-              case element.name
-              when 'title'
-                event.title = element.text
-              when 'link'
-                event.url = element.text
-                # only include events which have this in their path
-                next unless event.url.include?('activity') || event.url.include?('Research-Results')
-              when 'creator'
-                # event.creator = element.text
-                # no creator field. Not sure needs one
-              when 'description'
-                event.description = convert_description element.text
-              when 'location'
-                event.venue = element.text
-                loc = element.text.split(',')
-                event.city = loc.first.strip
-                event.country = loc.last.strip
-              when 'provider'
-                event.organizer = element.text
-              when 'startdate', 'courseDate'
-                event.start = element.text.to_s
-              when 'enddate', 'courseEndDate'
-                event.end = element.text.to_s
-              when 'latitude'
-                event.latitude = element.text
-              when 'longitude'
-                event.longitude = element.text
-              when 'pubDate'
-                # Not really needed
-              else
-                # chuck away
-              end
-            end
-          end
-          # Now fetch the page to get the event date (until it is added to the RSS feed)
-          unless event.start and !event.url.starts_with('https://')
-            # should we do more against data exfiltration? URI.open is a known hazard
-            page = Nokogiri::XML(open_url(event.url, raise: true))
-            event.start, event.end = parse_dates(page.xpath('//th[.="Date"]').first&.parent&.xpath('td')&.last&.text&.strip, 'Amsterdam')
-            # in this case also grab the venue
-            event.venue = page.xpath('//th[.="Venue"]').first&.parent&.xpath('td')&.last&.text
-            unless Rails.env.test? and File.exist?('test/vcr_cassettes/ingestors/wur.yml')
-              sleep 1
-            end
-          end
+        html = open_url(url, raise: true).read
+
+        extract_activities(html).each do |activity|
+          event = OpenStruct.new
+          event.title = activity['title']
+          event.url = "https://www.wur.nl#{activity['path']}"
+          event.description = activity['summary']
+          event.event_types = ['workshops_and_courses']
 
+          location = activity['filterCriteria']&.find { |f| f['label'] == 'Location' }
+          event.venue = location && location['value']
+
+          event.start = Time.zone.parse(activity['date']) if activity['date']
+          event.end = activity['endDate'] ? Time.zone.parse(activity['endDate']) : event.start
           event.set_default_times
+
           event.source = 'WUR'
           event.timezone = 'Amsterdam'
 
@@ -87,6 +51,58 @@ def process_wur(url)
           @messages << "Extract event fields failed with: #{e.message}"
         end
       end
+
+      def extract_activities(html)
+        chunks = html.scan(/self\.__next_f\.push\(\[1,"(.*?)"\]\)/m).flatten
+        payload = chunks.map { |chunk| JSON.parse("\"#{chunk}\"") }.join
+
+        activities = []
+        payload.to_enum(:scan, /\{"id":\d+,"title":"[^"]*"/).each do
+          match = Regexp.last_match
+          object_str = extract_balanced_json(payload, match.begin(0))
+          next unless object_str
+
+          begin
+            object = JSON.parse(object_str)
+          rescue JSON::ParserError
+            next
+          end
+          activities << object if object['path']&.include?('/activity/')
+        end
+        activities
+      end
+
+      # Pulls out the JSON object starting at `start`, tracking brace depth
+      # so that nested objects (filterCriteria, gtm, headerMedia, ...) don't
+      # cause it to stop early.
+      def extract_balanced_json(str, start)
+        depth = 0
+        in_string = false
+        escaped = false
+        i = start
+        while i < str.length
+          char = str[i]
+          if in_string
+            if escaped
+              escaped = false
+            elsif char == '\\'
+              escaped = true
+            elsif char == '"'
+              in_string = false
+            end
+          else
+            case char
+            when '"' then in_string = true
+            when '{' then depth += 1
+            when '}'
+              depth -= 1
+              return str[start..i] if depth.zero?
+            end
+          end
+          i += 1
+        end
+        nil
+      end
     end
   end
 end

test/unit/ingestors/taxila/han_ingestor_test.rb

@@ -22,12 +22,12 @@ class HanIngestorTest < ActiveSupport::TestCase
     ingestor = Ingestors::Taxila::HanIngestor.new
 
     # check events don't exist
-    new_title = 'Synthetiseren en Karakteriseren van Moleculen'
-    new_url = 'https://www.han.nl/opleidingen/module/synthetiseren-karakteriseren-moleculen/'
+    new_title = 'Cervixcytologie'
+    new_url = 'https://www.han.nl/opleidingen/cursus/cervixcytologie/'
     refute Event.where(title: new_title, url: new_url).any?
 
     # run task
-    assert_difference('Event.count', 15) do
+    assert_difference('Event.count', 20) do
       freeze_time(2019) do
         VCR.use_cassette('ingestors/han') do
           ingestor.read(source.url)
@@ -44,8 +44,8 @@ class HanIngestorTest < ActiveSupport::TestCase
 
     assert_equal 'HAN', event.source
     assert_equal 'Amsterdam', event.timezone
-    assert_equal 'Nijmegen Laan van Scheut 2', event.venue
-    assert_equal Time.zone.parse('Mon, 1 Feb 2026 09:00:00.000000000 UTC +00:00'), event.start
-    assert_equal Time.zone.parse('Mon, 1 Feb 2026 17:00:00.000000000 UTC +00:00'), event.end
+    assert_equal 'Nijmegen, Laan van Scheut 2', event.venue
+    assert_equal Time.zone.parse('Tue, 1 Sep 2026 09:00:00.000000000 UTC +00:00'), event.start
+    assert_equal Time.zone.parse('Tue, 1 Sep 2026 17:00:00.000000000 UTC +00:00'), event.end
   end
 end

test/unit/ingestors/taxila/oscm_ingestor_test.rb

@@ -22,12 +22,12 @@ class OscmIngestorTest < ActiveSupport::TestCase
     ingestor = Ingestors::Taxila::OscmIngestor.new
 
     # check event doesn't
-    new_title = 'FAIR Coffee lecture - Mariëlle Prevoo (pre-announcement)'
-    new_url = 'https://www.openscience-maastricht.nl/events/fair-coffee-lecture-marielle-prevoo-pre-announcement/'
+    new_title = 'FAIR Coffee lecture - Mariëlle Prevoo'
+    new_url = 'https://www.openscience-maastricht.nl/events/fair-coffee-lecture-marielle-prevoo/'
     refute Event.where(title: new_title, url: new_url).any?
 
     # run task
-    assert_difference 'Event.count', 4 do
+    assert_difference 'Event.count', 10 do
       freeze_time(2019) do
         VCR.use_cassette("ingestors/oscm") do
           ingestor.read(source.url)
@@ -36,9 +36,9 @@ class OscmIngestorTest < ActiveSupport::TestCase
       end
     end
 
-    assert_equal 4, ingestor.events.count
+    assert_equal 10, ingestor.events.count
     assert ingestor.materials.empty?
-    assert_equal 4, ingestor.stats[:events][:added]
+    assert_equal 10, ingestor.stats[:events][:added]
     assert_equal 0, ingestor.stats[:events][:updated]
     assert_equal 0, ingestor.stats[:events][:rejected]
 
@@ -49,7 +49,7 @@ class OscmIngestorTest < ActiveSupport::TestCase
     assert_equal new_url, event.url
 
     # check other fields
-    assert_equal 'FAIR Coffee lecture - Mariëlle Prevoo (pre-announcement)', event.title
+    assert_equal 'FAIR Coffee lecture - Mariëlle Prevoo', event.title
     assert_equal 'Amsterdam', event.timezone
     assert_equal 'OSCM', event.source
     assert event.online?

test/unit/ingestors/taxila/wur_ingestor_test.rb

@@ -14,20 +14,20 @@ class WurIngestorTest < ActiveSupport::TestCase
 
   test 'can ingest events from wur' do
     source = @content_provider.sources.build(
-      url: 'https://www.wur.nl/en/Resources-1/RSS/Calendar.htm',
+      url: 'https://www.wur.nl/en/news-insights/activities-at-wur',
       method: 'wur',
       enabled: true
     )
 
     ingestor = Ingestors::Taxila::WurIngestor.new
 
     # check event doesn't
-    new_title = 'Genetic Diversity - key to transitions in agriculture and forestry'
-    new_url = 'https://www.wur.nl/en/activity/genetic-diversity-key-to-transitions-in-agriculture-and-forestry-1.htm'
+    new_title = 'Prying Genomes: Providing a molecular foundation for oyster restoration'
+    new_url = 'https://www.wur.nl/en/activity/prying-genomes-providing-molecular-foundation-oyster-restoration'
     refute Event.where(title: new_title, url: new_url).any?
 
     # run task
-    assert_difference 'Event.count', 24 do
+    assert_difference 'Event.count', 62 do
       freeze_time(2016) do
         VCR.use_cassette("ingestors/wur") do
           ingestor.read(source.url)
@@ -36,9 +36,9 @@ class WurIngestorTest < ActiveSupport::TestCase
       end
     end
 
-    assert_equal 24, ingestor.events.count
+    assert_equal 62, ingestor.events.count
     assert ingestor.materials.empty?
-    assert_equal 24, ingestor.stats[:events][:added]
+    assert_equal 62, ingestor.stats[:events][:added]
     assert_equal 0, ingestor.stats[:events][:updated]
     assert_equal 0, ingestor.stats[:events][:rejected]
 
@@ -51,7 +51,8 @@ class WurIngestorTest < ActiveSupport::TestCase
     # check other fields
     assert_equal 'WUR', event.source
     assert_equal 'Amsterdam', event.timezone
-    assert_equal Time.zone.parse('Wed, 15 Mar 2023 11:00:00.000000000 UTC +00:00'), event.start
-    assert_equal Time.zone.parse('Wed, 15 Mar 2023 17:00:00.000000000 UTC +00:00'), event.end
+    assert_equal 'Omnia - Building 105', event.venue
+    assert_equal Time.zone.parse('2026-08-24 09:00:00'), event.start
+    assert_equal Time.zone.parse('2026-08-24 17:00:00'), event.end
   end
 end