Custom downloader support ShadowDOM

Author: wangxianqiao

scraper/src/custom_downloader_middleware.py

@@ -28,7 +28,7 @@ def process_request(self, request, spider):
         self.driver.get(unquote_plus(
             request.url))  # Decode url otherwise firefox is not happy. Ex /#%21/ => /#!/%21
         time.sleep(spider.js_wait)
-        body = self.driver.page_source.encode('utf-8')
+        body = self.driver.execute_script("return document.documentElement.getInnerHTML();")
         url = self.driver.current_url
 
         return HtmlResponse(