gh-123018: Keep the libedit history file header when truncating (GH-157165)

libedit's history_truncate_file() keeps the last N lines of the file,
which drops the "_HiStOrY_V2_" header line that its own write_history()
emits and that its read_history() requires.  So on a libedit build,
readline.write_history_file() or readline.append_history_file() after
readline.set_history_length() produced a file that
readline.read_history_file() rejected with EINVAL.

Under the libedit emulation, truncate the file ourselves and keep the
header, resolving the default "~/.history" the same way libedit does.
Apple's libedit fork already preserves the header, so the workaround is
not compiled on macOS.
(cherry picked from commit 2638785)

Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com>

Author: gpshead

Lib/test/test_readline.py

@@ -168,6 +168,44 @@ def test_write_read_limited_history(self):
         # Readline seems to report an additional history element.
         self.assertIn(readline.get_current_history_length(), (2, 3))
 
+    def test_write_read_zero_length_history(self):
+        previous_length = readline.get_history_length()
+        self.addCleanup(readline.set_history_length, previous_length)
+
+        readline.clear_history()
+        readline.add_history("first line")
+        readline.set_history_length(0)
+        readline.write_history_file(TESTFN)
+        self.addCleanup(os.remove, TESTFN)
+
+        readline.clear_history()
+        # libedit cannot read an empty history file, only one that still
+        # has its header line.  How many items remain is not checked:
+        # libedit's own history_truncate_file() ignores a length of 0.
+        readline.read_history_file(TESTFN)
+
+    @unittest.skipUnless(hasattr(readline, "append_history_file"),
+                         "append_history not available")
+    def test_append_limited_history(self):
+        previous_length = readline.get_history_length()
+        self.addCleanup(readline.set_history_length, previous_length)
+
+        readline.clear_history()
+        readline.add_history("first line")
+        readline.add_history("second line")
+        readline.write_history_file(TESTFN)
+        self.addCleanup(os.remove, TESTFN)
+
+        readline.add_history("third line")
+        readline.set_history_length(2)
+        readline.append_history_file(1, TESTFN)
+
+        readline.clear_history()
+        readline.read_history_file(TESTFN)
+        self.assertEqual(readline.get_history_item(1), "second line")
+        self.assertEqual(readline.get_history_item(2), "third line")
+        self.assertEqual(readline.get_history_item(3), None)
+
 
 class TestReadline(unittest.TestCase):
 

Misc/NEWS.d/next/Library/2026-09-08-04-10-00.gh-issue-123018.hV2kQe.rst

@@ -0,0 +1,6 @@
+Fix :func:`readline.write_history_file` and
+:func:`readline.append_history_file` producing a history file that
+:func:`readline.read_history_file` could not load when a limit was set with
+:func:`readline.set_history_length` and Python was built against
+``libedit``.  This works around `NetBSD PR 60322
+<https://gnats.netbsd.org/60322>`_.

Modules/readline.c

@@ -10,16 +10,27 @@
 
 /* Standard definitions */
 #include "Python.h"
+#include "pycore_fileutils.h"     // _Py_open_noraise()
 #include "pycore_pyatomic_ft_wrappers.h"
 #include "pycore_pylifecycle.h"   // _Py_SetLocaleFromEnv()
 
 #include <errno.h>                // errno
+#include <pwd.h>                  // getpwuid_r()
 #include <signal.h>               // SIGWINCH
 #include <stdlib.h>               // free()
 #include <string.h>               // strdup()
+#ifdef HAVE_FCNTL_H
+#  include <fcntl.h>              // O_RDWR
+#endif
 #ifdef HAVE_SYS_SELECT_H
 #  include <sys/select.h>         // select()
 #endif
+#ifdef HAVE_SYS_STAT_H
+#  include <sys/stat.h>           // fstat()
+#endif
+#ifdef HAVE_UNISTD_H
+#  include <unistd.h>             // ftruncate()
+#endif
 
 #if defined(HAVE_SETLOCALE)
 /* GNU readline() mistakenly sets the LC_CTYPE locale.
@@ -319,6 +330,176 @@ readline_read_history_file_impl(PyObject *module, PyObject *filename_obj)
 
 static int _history_length = -1; /* do not truncate history by default */
 
+#ifndef __APPLE__
+/* macOS libedit carries a patch fixing the bug this works around. */
+/* Return libedit's default history file, "~/.history".  This must resolve
+ * the same file as libedit's private _default_history_file()
+ * (https://cvsweb.netbsd.org/bsdweb.cgi/src/lib/libedit/readline.c) so that we
+ * truncate the same file write_history() and append_history() just wrote: the
+ * home directory comes from the password database, $HOME is not consulted.
+ * The result must be freed with PyMem_RawFree(). */
+static char *
+_py_libedit_default_history_file(void)
+{
+    struct passwd *pw = NULL;
+    char *path = NULL;
+#ifdef HAVE_GETPWUID_R
+    struct passwd pwd;
+    char *buf = NULL;
+    Py_ssize_t bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
+    if (bufsize == -1) {
+        bufsize = 1024;
+    }
+    for (;;) {
+        char *newbuf = PyMem_RawRealloc(buf, bufsize);
+        if (newbuf == NULL) {
+            break;
+        }
+        buf = newbuf;
+        int status = getpwuid_r(getuid(), &pwd, buf, bufsize, &pw);
+        if (status == 0) {
+            break;
+        }
+        pw = NULL;
+        if (status != ERANGE || bufsize > (PY_SSIZE_T_MAX >> 1)) {
+            break;
+        }
+        bufsize <<= 1;
+    }
+#else
+    pw = getpwuid(getuid());
+#endif
+    if (pw != NULL && pw->pw_dir != NULL) {
+        size_t len = strlen(pw->pw_dir) + sizeof("/.history");
+        path = PyMem_RawMalloc(len);
+        if (path != NULL) {
+            PyOS_snprintf(path, len, "%s/.history", pw->pw_dir);
+        }
+    }
+#ifdef HAVE_GETPWUID_R
+    PyMem_RawFree(buf);
+#endif
+    return path;
+}
+
+/* libedit's history_truncate_file() keeps the last nlines lines of the
+ * file, which drops the "_HiStOrY_V2_" header line that its own
+ * write_history() emits and that its read_history() requires: once
+ * truncated, the file can no longer be loaded (gh-123018).  Truncate the
+ * file ourselves and keep the header.
+ *
+ * Upstream libedit (NetBSD, and the portable releases from thrysoee.dk)
+ * still has this bug, reported as https://gnats.netbsd.org/60322.  Apple's
+ * libedit fork patches its history_truncate_file() to copy the first line,
+ * so this workaround is not compiled on macOS.  We only keep the header
+ * when it is actually there, so a file without one (e.g. written by GNU
+ * readline) is not given a bogus header.
+ *
+ * History files are small, so the whole file is read into memory and
+ * rewritten in place, as libedit does.  GNU readline instead writes a
+ * temporary file and renames it over the original. */
+static int
+_py_libedit_history_truncate_file(const char *filename, int nlines)
+{
+    static const char cookie[] = "_HiStOrY_V2_\n";
+    const size_t cookie_len = sizeof(cookie) - 1;
+    char *default_file = NULL;
+    char *buf = NULL;
+    FILE *fp = NULL;
+    int ret = -1;
+
+    if (filename == NULL) {
+        default_file = _py_libedit_default_history_file();
+        if (default_file == NULL) {
+            return -1;
+        }
+        filename = default_file;
+    }
+    int fd = _Py_open_noraise(filename, O_RDWR);
+    if (fd < 0) {
+        goto done;
+    }
+    fp = fdopen(fd, "r+");
+    if (fp == NULL) {
+        close(fd);
+        goto done;
+    }
+    struct stat st;
+    if (fstat(fd, &st) != 0) {
+        goto done;
+    }
+    size_t size = (size_t)st.st_size;
+    buf = PyMem_RawMalloc(size);
+    if (buf == NULL || fread(buf, 1, size, fp) != size) {
+        goto done;
+    }
+
+    size_t header = 0;
+    if (size >= cookie_len && memcmp(buf, cookie, cookie_len) == 0) {
+        header = cookie_len;
+    }
+    const char *end = buf + size;
+
+    /* Count the lines following the header. */
+    size_t total = 0;
+    for (const char *p = buf + header;
+         (p = memchr(p, '\n', end - p)) != NULL;
+         p++) {
+        total++;
+    }
+    if (end > buf + header && end[-1] != '\n') {
+        total++;  /* an unterminated last line */
+    }
+    if (total <= (size_t)nlines) {
+        ret = 0;  /* nothing to drop */
+        goto done;
+    }
+
+    /* Skip the leading lines, keeping the header and the last nlines. */
+    const char *tail = buf + header;
+    for (size_t skip = total - (size_t)nlines; skip > 0; skip--) {
+        const char *nl = memchr(tail, '\n', end - tail);
+        if (nl == NULL) {
+            tail = end;  /* the unterminated last line is dropped too */
+            break;
+        }
+        tail = nl + 1;
+    }
+    size_t taillen = end - tail;
+    if (fseek(fp, 0, SEEK_SET) != 0
+        || fwrite(buf, 1, header, fp) != header
+        || fwrite(tail, 1, taillen, fp) != taillen
+        || fflush(fp) != 0
+        || ftruncate(fd, (off_t)(header + taillen)) != 0) {
+        goto done;
+    }
+    ret = 0;
+
+done:
+    if (fp != NULL) {
+        fclose(fp);
+    }
+    PyMem_RawFree(buf);
+    PyMem_RawFree(default_file);
+    return ret;
+}
+#endif  /* !__APPLE__ */
+
+/* Truncate the history file after write_history() or append_history().
+ * Like the history_truncate_file() calls this replaces, failures are
+ * ignored by the callers: the history was already saved successfully and
+ * a failed truncation only leaves the file longer than requested. */
+static int
+_py_history_truncate_file(const char *filename, int nlines)
+{
+#ifndef __APPLE__
+    if (using_libedit_emulation) {
+        return _py_libedit_history_truncate_file(filename, nlines);
+    }
+#endif
+    return history_truncate_file(filename, nlines);
+}
+
 /* Exported function to save a readline history file */
 
 /*[clinic input]
@@ -360,7 +541,7 @@ readline_write_history_file_impl(PyObject *module, PyObject *filename_obj)
     errno = err = write_history(filename);
     int history_length = FT_ATOMIC_LOAD_INT_RELAXED(_history_length);
     if (!err && history_length >= 0)
-        history_truncate_file(filename, history_length);
+        _py_history_truncate_file(filename, history_length);
     Py_XDECREF(filename_bytes);
     errno = err;
     if (errno)
@@ -419,7 +600,7 @@ readline_append_history_file_impl(PyObject *module, int nelements,
         nelements - libedit_append_replace_history_offset, filename);
     int history_length = FT_ATOMIC_LOAD_INT_RELAXED(_history_length);
     if (!err && history_length >= 0)
-        history_truncate_file(filename, history_length);
+        _py_history_truncate_file(filename, history_length);
     Py_XDECREF(filename_bytes);
     errno = err;
     if (errno)