From 622d608e44d41b3f86fda737fadd80e3e0231db4 Mon Sep 17 00:00:00 2001 From: Ali Zulfiqar Date: Mon, 3 Aug 2026 17:19:19 +0500 Subject: [PATCH 1/3] Fix PDF text extraction stopping after the first page Restore the extracted-text read to its position after the page loop in get_text_lines. The refactoring in #4606 removed one level of with-block nesting and dedented the whole function body except the final two lines, which silently moved them inside the for loop: - with the default max_pages=5, the function returned right after processing page 1, so text on pages 2-5 was never extracted and copyright/license/email/url detection silently missed it - when max_pages was reached via break, or the PDF had no pages, the function returned None, crashing the caller textcode.analysis.unicode_text_lines_from_pdf with a TypeError when it iterates the result Signed-off-by: Ali Zulfiqar --- src/textcode/pdf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/textcode/pdf.py b/src/textcode/pdf.py index 4e90c866f8..1c966a3a66 100644 --- a/src/textcode/pdf.py +++ b/src/textcode/pdf.py @@ -43,5 +43,5 @@ def get_text_lines(location, max_pages=5): interpreter.process_page(page) if max_pages and page_num == max_pages: break - extracted_text.seek(0) - return extracted_text.readlines() + extracted_text.seek(0) + return extracted_text.readlines() From 43db560cf8d686d8300fec06b4fb892674c25c6a Mon Sep 17 00:00:00 2001 From: Ali Zulfiqar Date: Mon, 3 Aug 2026 17:19:20 +0500 Subject: [PATCH 2/3] Add regression tests for multi-page PDF text extraction Add a 7-page test PDF with distinct text on each page and tests asserting that: - text is extracted from all pages up to the default max_pages=5 - reaching max_pages returns the extracted lines and not None - max_pages=0 extracts all pages The existing PDF test files only assert content from the first page, which is why the regression was not caught. Signed-off-by: Ali Zulfiqar --- tests/textcode/data/pdf/multi_page.pdf | Bin 0 -> 4255 bytes tests/textcode/test_pdf.py | 27 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tests/textcode/data/pdf/multi_page.pdf diff --git a/tests/textcode/data/pdf/multi_page.pdf b/tests/textcode/data/pdf/multi_page.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d369663e1879d8ff87532e03eb54ba62468022c5 GIT binary patch literal 4255 zcmdT|TX&*L8h!UyRJ0W@kvj-p<0W=%FHym$Q6mV2;sq3Nrh6Xd56ryH`~21^G&xP8 zYn_#K<|RG^YqR;N{aM+&Alj|f(-h8xqW}5#zy1sQaBn+aOI?8P(1MQYK^-6l&~5Fw zgO&&4_8wZUeRMRK2vL9p_F^5%=R-uD0yG--L&ahUTDlMWViSp4~*$#-SmTmpo*+(}4h^hls&$O+YZU>9UnF~jC zet|TY1|n{nNSq8CWzZUGDE*F>yN8--m{3RelTB}nrhK$K|H}z+)cqc1CQ(D@GkowrmT-7sNom)rHyH1p3JXke-^?BQ6r-RiM+o;eKodEb1* zbP z|KDG*KyE7dkoL5E^A_kTz76_YyDfAxh6H3_@9>g-hn^3QxEIxSjHOv*l4;|Tg$0F&2$UgEiA@Yw8;5m-rfC2u% zc$EGA2Ux!`^yKvsqeRrZ4?IRhZvO$}giBsVxa5hM3m!#fE_f8fU-CHqlE*Wbm>?o^ zdOtrT<3bxv(`XmV(5IX#p&Y9KVpFIa|! zIwX-^9SVX$(j3ci6lXwYq0*vS{By=E6b-x^SYCfOkWK^LL|eiSZwW#~!!m4e+6VSS m=(*TNkJbLx5)CBtAJAV!r|pO8c&BB>@GKdMM(-c%q5lE+8m7Yl literal 0 HcmV?d00001 diff --git a/tests/textcode/test_pdf.py b/tests/textcode/test_pdf.py index 7941953568..cc7b2d05da 100644 --- a/tests/textcode/test_pdf.py +++ b/tests/textcode/test_pdf.py @@ -51,6 +51,33 @@ def get_text(location): assert result == expected + def test_get_text_lines_extracts_all_pages_up_to_max_pages(self): + # regression test: text extraction must not stop after the first page + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file) + text = b''.join(result) + assert b'This is page 1 of a multi-page test document.' in text + assert b'This notice is on page 2.' in text + assert b'The last page 5 of this test document.' in text + # the default max_pages=5 must still be honored + assert b'Page 6' not in text + + def test_get_text_lines_returns_lines_when_max_pages_is_one(self): + # regression test: reaching max_pages must not return None + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file, max_pages=1) + assert result + text = b''.join(result) + assert b'This is page 1 of a multi-page test document.' in text + assert b'page 2' not in text + + def test_get_text_lines_extracts_all_pages_when_max_pages_is_zero(self): + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file, max_pages=0) + text = b''.join(result) + assert b'Page 6 is beyond the default max_pages limit.' in text + assert b'Page 7 is also beyond the default max_pages limit.' in text + def test_pdfminer_can_parse_faulty_broadcom_doc(self): # test for https://github.com/euske/pdfminer/issues/118 test_file = self.get_test_loc('pdf/pdfminer_bug_118/faulty.pdf') From a4673153d9d1f49f5558d095d6e33c596f15c319 Mon Sep 17 00:00:00 2001 From: Ali Zulfiqar Date: Mon, 3 Aug 2026 17:46:29 +0500 Subject: [PATCH 3/3] Add changelog entry for PDF text extraction fix Signed-off-by: Ali Zulfiqar --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d9a5a6b402..22b678782d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,10 @@ Next release ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 +- Fix a regression in PDF text extraction that silently stopped after the + first page, so copyright/license detection missed text on later pages. + https://github.com/aboutcode-org/scancode-toolkit/pull/5260 + v33.0.0rc1 - 2026-05-14 ------------------------