From 4dcbfd176b61460535b3acd6b2fcc5a4b592fb7f Mon Sep 17 00:00:00 2001 From: KevinBamwisho Date: Wed, 19 Aug 2026 22:50:35 -0600 Subject: [PATCH] Read Excel columns by name, and skip blank rows (#136) The Excel parser took column 0 as the question number, column 1 as the question and column 2 as the options, whatever the spreadsheet actually held. A sheet laid out as Questionnaire | Question # | Question text | Notes therefore ended up with the question number in question_text and the question itself in options, and blank rows used to separate one questionnaire from the next became questions with no text, which the schema rejects with a ValidationError. Look for a header row first and match the column names, falling back to the old positional reading when no header is recognised. A questionnaire column now splits a sheet into one instrument per questionnaire, and its name reaches both the instrument and its questions. A notes column is kept on the question, in place of the hardcoded "blah" intro. --- src/harmony/parsing/excel_parser.py | 234 +++++++++++++++-------- tests/test_convert_excel_fluid_format.py | 150 +++++++++++++++ 2 files changed, 306 insertions(+), 78 deletions(-) create mode 100644 tests/test_convert_excel_fluid_format.py diff --git a/src/harmony/parsing/excel_parser.py b/src/harmony/parsing/excel_parser.py index a775637..535326d 100644 --- a/src/harmony/parsing/excel_parser.py +++ b/src/harmony/parsing/excel_parser.py @@ -39,6 +39,23 @@ re_header_column = re.compile(r'(?i)(?:question|text|pergunta)') +# Headers we know how to read, and the role each one plays. These match the whole +# cell so that "Question #" is not mistaken for "Question text". +COLUMN_PATTERNS = { + "question": re.compile( + r"(?i)^\s*(?:question(?:\s*(?:text|wording))?|item(?:\s*text)?|text|wording|pergunta)\s*$"), + "question_no": re.compile( + r"(?i)^\s*(?:(?:question|item|q)\s*(?:no\.?|number|num|#)|no\.?|number|#)\s*$"), + "options": re.compile( + r"(?i)^\s*(?:(?:response|answer)s?\s*(?:options?|choices|scale|categories)?" + r"|options?|choices|categories)\s*$"), + "instrument": re.compile( + r"(?i)^\s*(?:questionnaire|instrument|scale|measure|survey)(?:\s*name)?\s*$"), + "notes": re.compile(r"(?i)^\s*(?:notes?|comments?|remarks?|description)\s*$"), +} + +NORMALISED_COLUMNS = ["question_no", "question", "options", "instrument", "notes"] + def clean_option_no(option_could_be_int): if option_could_be_int is None \ @@ -53,93 +70,154 @@ def clean_option_no(option_could_be_int): return str(option_could_be_int) +def find_header_row(df: pd.DataFrame, rows_to_scan: int = 5) -> tuple: + """Find the row that names the columns, and work out which column holds what. + + Returns (row index, {role: column}). The question text column is what we anchor + on: if no row names one, we return (None, {}) and the caller falls back to + reading the columns by position. + """ + for row_idx in range(min(rows_to_scan, len(df))): + roles = {} + for col in df.columns: + cell = df[col].iloc[row_idx] + if not isinstance(cell, str): + continue + for role, pattern in COLUMN_PATTERNS.items(): + if role not in roles and pattern.match(cell): + roles[role] = col + break + if "question" in roles: + return row_idx, roles + return None, {} + + +def columns_by_name(df: pd.DataFrame, header_row: int, roles: dict) -> pd.DataFrame: + """Pick out the named columns, dropping the header row and anything above it.""" + body = df.iloc[header_row + 1:] + result = pd.DataFrame(index=body.index) + for role in NORMALISED_COLUMNS: + col = roles.get(role) + result[role] = body[col] if col is not None else "" + return result + + +def columns_by_position(df: pd.DataFrame) -> pd.DataFrame: + """Read the columns by position: question number, question, options. + + This is the original behaviour, kept for sheets with no header we recognise. + """ + df_questions = df.copy() + + # check we have 3 columns. If more or less, adjust it by deleting or inserting. + if len(df_questions.columns) > 3: + if str(df_questions[df_questions.columns[3]].iloc[0]).lower() == "filename": + if len(df_questions.columns) > 4 and str( + df_questions[df_questions.columns[4]].iloc[0]).lower() == "language": + df_questions.drop(columns=df_questions.columns[5:], inplace=True) + else: + df_questions.drop(columns=df_questions.columns[4:], inplace=True) + else: + df_questions.drop(columns=df_questions.columns[3:], inplace=True) + elif len(df_questions.columns) < 3: + col_avg_lengths = [0] * len(df_questions.columns) + for col_idx, col_name in enumerate(df_questions.columns): + col_avg_lengths[col_idx] = df_questions[col_name].apply(lambda s: len(str(s))).mean() + biggest_col = int(np.argmax(col_avg_lengths)) + if biggest_col == 0: + df_questions.insert(0, "question_no", [str(n) for n in range(len(df_questions))]) + if len(df_questions.columns) < 3: + df_questions.insert(2, "options", [""] * len(df_questions)) + + # standardise the column names + if len(df_questions.columns) == 3: + df_questions.columns = ["question_no", "question", "options"] + elif len(df_questions.columns) == 4: + df_questions.columns = ["question_no", "question", "options", "filename"] + else: + df_questions.columns = ["question_no", "question", "options", "filename", "language"] + + # Check if header row present, in which case remove it + rows_to_delete = [] + for i in range(len(df_questions)): + if df_questions.question.iloc[i] is None or type(df_questions.question.iloc[i]) is not str or \ + re_header_column.match(df_questions.question.iloc[i]): + rows_to_delete.append(i) + break + + if len(rows_to_delete) > 0: + df_questions.drop(rows_to_delete, inplace=True) + + df_questions["instrument"] = "" + df_questions["notes"] = "" + + return df_questions[NORMALISED_COLUMNS] + + def convert_excel_to_instruments(file: RawFile) -> List[Instrument]: sheet_name_to_dataframe = parse_excel_to_pandas(file.content) instruments = [] - for sheet_idx, (sheet_name, df_questions) in enumerate(sheet_name_to_dataframe.items()): - - # check we have 3 columns. If more or less, adjust it by deleting or inserting. - if len(df_questions.columns) > 3: - if str(df_questions[df_questions.columns[3]].iloc[0]).lower() == "filename": - if len(df_questions.columns) > 4 and str( - df_questions[df_questions.columns[4]].iloc[0]).lower() == "language": - df_questions.drop(columns=df_questions.columns[5:], inplace=True) - else: - df_questions.drop(columns=df_questions.columns[4:], inplace=True) - else: - df_questions.drop(columns=df_questions.columns[3:], inplace=True) - elif len(df_questions.columns) < 3: - col_avg_lengths = [0] * len(df_questions.columns) - for col_idx, col_name in enumerate(df_questions.columns): - col_avg_lengths[col_idx] = df_questions[col_name].apply(lambda s: len(str(s))).mean() - biggest_col = int(np.argmax(col_avg_lengths)) - if biggest_col == 0: - df_questions.insert(0, "question_no", [str(n) for n in range(len(df_questions))]) - if len(df_questions.columns) < 3: - df_questions.insert(2, "options", [""] * len(df_questions)) - - # standardise the column names - if len(df_questions.columns) == 3: - df_questions.columns = ["question_no", "question", "options"] - elif len(df_questions.columns) == 4: - df_questions.columns = ["question_no", "question", "options", "filename"] - else: - df_questions.columns = ["question_no", "question", "options", "filename", "language"] - - # Check if header row present, in which case remove it - rows_to_delete = [] - for i in range(len(df_questions)): - if df_questions.question.iloc[i] is None or type(df_questions.question.iloc[i]) is not str or \ - re_header_column.match(df_questions.question.iloc[i]): - rows_to_delete.append(i) - break + for sheet_name, df_sheet in sheet_name_to_dataframe.items(): + # Blank rows are used to space questionnaires apart. Drop them up front so + # they can't be read as questions, and renumber so that the row positions + # used further down still line up with the row labels. + df_sheet = df_sheet.dropna(how="all").reset_index(drop=True) + if len(df_sheet) == 0: + continue - if len(rows_to_delete) > 0: - df_questions.drop(rows_to_delete, inplace=True) + header_row, roles = find_header_row(df_sheet) + if header_row is None: + df_questions = columns_by_position(df_sheet) + else: + df_questions = columns_by_name(df_sheet, header_row, roles) - # Make sure the whole DF is of type string. - df_questions["question_no"] = df_questions["question_no"].apply(clean_option_no) - df_questions["question"] = df_questions["question"].apply(clean_option_no) - df_questions["options"] = df_questions["options"].apply(clean_option_no) + for column in NORMALISED_COLUMNS: + df_questions[column] = df_questions[column].apply(clean_option_no) + # A row with no question text can't become a Question, since the schema asks + # for at least one character. Drop those rather than raising. + df_questions = df_questions[df_questions["question"].str.strip() != ""] if len(df_questions) == 0: continue - questions = [] - for idx in range(len(df_questions)): - o = df_questions.options.iloc[idx] - if type(o) is str: - options = o.split("/") - else: - options = [] - question = Question(question_no=str(df_questions.question_no.iloc[idx]), question_intro="blah", - question_text=str(df_questions.question.iloc[idx]), - options=options, source_page=0) - questions.append(question) - - language = "en" - try: - valid_questions = df_questions["question"].dropna() - valid_questions = [q for q in valid_questions if isinstance(q, str) and q.strip()] - if valid_questions: - language = detect(" ".join(valid_questions)) - except: - print("Error identifying language in Excel file") - traceback.print_exc() - traceback.print_stack() - - instrument = Instrument( - file_id=file.file_id, - instrument_id=file.file_id + "_" + str(sheet_idx), - file_name=file.file_name, - instrument_name=file.file_name + " / " + sheet_name, - file_type=file.file_type, - file_section=sheet_name, - language=language, - questions=questions - ) - - instruments.append(instrument) + # A questionnaire column lets one sheet hold several instruments. Carry the + # name downwards so that a name written once at the top of a block covers it. + instrument_names = df_questions["instrument"].replace("", np.nan).ffill().fillna("") + + for group_name, df_group in df_questions.groupby(instrument_names, sort=False): + instrument_name = str(group_name).strip() or f"{file.file_name} / {sheet_name}" + instrument_id = f"{file.file_id}_{len(instruments)}" + + questions = [] + for position, row in enumerate(df_group.itertuples(), start=1): + options = [o.strip() for o in row.options.split("/") if o.strip()] + questions.append(Question( + question_no=row.question_no or str(position), + question_intro=row.notes or None, + question_text=row.question, + options=options, + source_page=0, + instrument_id=instrument_id, + instrument_name=instrument_name, + )) + + language = "en" + try: + language = detect(" ".join(df_group["question"])) + except Exception: + print("Error identifying language in Excel file") + traceback.print_exc() + + instruments.append(Instrument( + file_id=file.file_id, + instrument_id=instrument_id, + file_name=file.file_name, + instrument_name=instrument_name, + file_type=file.file_type, + file_section=sheet_name, + language=language, + questions=questions + )) return instruments diff --git a/tests/test_convert_excel_fluid_format.py b/tests/test_convert_excel_fluid_format.py new file mode 100644 index 0000000..102573f --- /dev/null +++ b/tests/test_convert_excel_fluid_format.py @@ -0,0 +1,150 @@ +''' +MIT License + +Copyright (c) 2023 Ulster University (https://www.ulster.ac.uk). +Project: Harmony (https://harmonydata.ac.uk) +Maintainer: Thomas Wood (https://fastdatascience.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +''' + +import base64 +import io +import sys +import unittest + +import pandas as pd + +sys.path.append("../src") + +from harmony import convert_excel_to_instruments +from harmony.schemas.requests.text import RawFile + + +def make_excel(rows, file_name="questionnaires.xlsx"): + """Turn a list of rows into a RawFile holding a real xlsx, as the web app would.""" + buffer = io.BytesIO() + pd.DataFrame(rows).to_excel(buffer, index=False, header=False) + buffer.seek(0) + return RawFile.model_validate({ + "file_id": "0123456789abcdef0123456789abcdef", + "file_name": file_name, + "file_type": "xlsx", + "content": "data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64," + + base64.b64encode(buffer.read()).decode(), + }) + + +# The layout from issue #136: named columns in an order Harmony did not expect, and +# blank rows separating one questionnaire from the next. +WELLBEING_SCALES = [ + ["Questionnaire", "Question #", "Question text", "Notes"], + ["GAD 7", 1, "Feeling nervous, anxious, or on edge", "Multiple Choice (Not at all / Several days)"], + ["GAD 7", 2, "Not being able to stop or control worrying", "Yes / No / Sometimes"], + [None, None, None, None], + ["PHQ-9", 1, "Little interest or pleasure in doing things?", "Multiple Choice (Not at all / Several days)"], + ["PHQ-9", 2, "Feeling down, depressed, or hopeless?", "Yes / No / Sometimes"], + ["PHQ-9", 3, "Trouble falling or staying asleep?", "Yes / No / Sometimes"], +] + + +class TestConvertExcelFluidFormat(unittest.TestCase): + + def test_blank_rows_between_questionnaires_are_ignored(self): + # Blank rows used to reach the schema as questions with no text, which raised. + instruments = convert_excel_to_instruments(make_excel(WELLBEING_SCALES)) + self.assertEqual(5, sum(len(instrument.questions) for instrument in instruments)) + + def test_question_text_column_is_identified(self): + instruments = convert_excel_to_instruments(make_excel(WELLBEING_SCALES)) + self.assertEqual("Feeling nervous, anxious, or on edge", + instruments[0].questions[0].question_text) + + def test_question_number_column_is_identified(self): + instruments = convert_excel_to_instruments(make_excel(WELLBEING_SCALES)) + self.assertEqual(["1", "2"], [q.question_no for q in instruments[0].questions]) + + def test_one_sheet_can_hold_several_instruments(self): + instruments = convert_excel_to_instruments(make_excel(WELLBEING_SCALES)) + self.assertEqual(["GAD 7", "PHQ-9"], [i.instrument_name for i in instruments]) + + def test_instrument_ids_are_unique(self): + instruments = convert_excel_to_instruments(make_excel(WELLBEING_SCALES)) + self.assertEqual(len(instruments), len({i.instrument_id for i in instruments})) + + def test_instrument_name_reaches_the_questions(self): + instruments = convert_excel_to_instruments(make_excel(WELLBEING_SCALES)) + self.assertEqual("PHQ-9", instruments[1].questions[0].instrument_name) + + def test_notes_column_is_kept(self): + instruments = convert_excel_to_instruments(make_excel(WELLBEING_SCALES)) + self.assertEqual("Yes / No / Sometimes", instruments[0].questions[1].question_intro) + + def test_instrument_name_carries_down_the_block(self): + # Some spreadsheets name the questionnaire once, on its first row only. + instruments = convert_excel_to_instruments(make_excel([ + ["Questionnaire", "Question"], + ["GAD 7", "Feeling nervous, anxious, or on edge"], + [None, "Not being able to stop or control worrying"], + ["PHQ-9", "Feeling down, depressed, or hopeless?"], + ])) + self.assertEqual(["GAD 7", "PHQ-9"], [i.instrument_name for i in instruments]) + self.assertEqual(2, len(instruments[0].questions)) + + def test_alternative_column_names(self): + instruments = convert_excel_to_instruments(make_excel([ + ["Item", "Response options"], + ["Feeling nervous, anxious, or on edge", "Not at all / Several days"], + ])) + self.assertEqual("Feeling nervous, anxious, or on edge", + instruments[0].questions[0].question_text) + self.assertEqual(["Not at all", "Several days"], instruments[0].questions[0].options) + + def test_rows_above_the_header_are_ignored(self): + instruments = convert_excel_to_instruments(make_excel([ + ["Wellbeing scales, collected March 2026", None], + [None, None], + ["Question", "Options"], + ["Feeling nervous, anxious, or on edge", "Not at all / Several days"], + ])) + self.assertEqual(1, len(instruments[0].questions)) + self.assertEqual("Feeling nervous, anxious, or on edge", + instruments[0].questions[0].question_text) + + def test_sheet_without_a_header_is_read_by_position(self): + # No recognisable header, so fall back to question number, question, options. + instruments = convert_excel_to_instruments(make_excel([ + [1, "Feeling nervous, anxious, or on edge", "Not at all / Several days"], + [2, "Not being able to stop or control worrying", "Not at all / Several days"], + ])) + self.assertEqual(1, len(instruments)) + self.assertEqual("Feeling nervous, anxious, or on edge", + instruments[0].questions[0].question_text) + self.assertEqual(["Not at all", "Several days"], instruments[0].questions[0].options) + + def test_empty_sheet_yields_no_instruments(self): + self.assertEqual([], convert_excel_to_instruments(make_excel([[None, None], [None, None]]))) + + def test_header_with_no_questions_yields_no_instruments(self): + self.assertEqual([], convert_excel_to_instruments(make_excel([["Question", "Options"]]))) + + +if __name__ == '__main__': + unittest.main()