Skip to content

Commit 5856bf9

Browse files
committed
AUS-1011: Give the hf child process a UTF-8 environment in fetch.py
On Windows the hf CLI wrote the job log with the console code page and failed on the harness's box-drawing characters before fetch.py saw a byte.
1 parent 6e33700 commit 5856bf9

1 file changed

Lines changed: 4 additions & 1 deletion

File tree

‎benchmarks/hf/fetch.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Fetch a Hugging Face Job's UTF-8 log and framed sweep JSON files."""
22
import argparse
33
import json
4+
import os
45
import pathlib
56
import re
67
import subprocess
@@ -15,8 +16,10 @@ def main():
1516
parser.add_argument("out_dir", type=pathlib.Path)
1617
args = parser.parse_args()
1718

19+
# The hf CLI writes the log with the console's code page unless told otherwise; the harness prints box-drawing characters.
20+
env = dict(os.environ, PYTHONIOENCODING="utf-8", PYTHONUTF8="1")
1821
result = subprocess.run(["hf", "jobs", "logs", args.job_id], stdout=subprocess.PIPE,
19-
stderr=subprocess.PIPE, check=False)
22+
stderr=subprocess.PIPE, check=False, env=env)
2023
if result.returncode:
2124
raise SystemExit(result.stderr.decode("utf-8", errors="replace"))
2225
log = result.stdout.decode("utf-8", errors="replace")

0 commit comments

Comments
 (0)