fix(agent): keep new turns when the session transcript hits its byte cap

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to ShellClaw are documented here. Format follows [Keep a Cha
 
 ## [Unreleased]
 
+### Fixed
+- Session saves that exceed the 128 KiB transcript cap drop the oldest messages instead of skipping the write. Compaction only runs once the message count passes `max_context_messages`, so a few long turns used to fill the byte cap and every later exchange was forgotten while the old transcript stayed in place.
+
 ## [1.0.1] - 2026-10-18
 
 ### Fixed

src/core/agent.c

@@ -515,6 +515,35 @@ static void free_tool_calls_copy(const provider_tool_call_t *copy, size_t n)
 	free(owned);
 }
 
+/** Print @p arr, dropping oldest messages until the text fits in @p out_size.
+ *  Compaction is count-based, so a few long turns fill the byte cap while the
+ *  message count is still under max_context_messages. Refusing the save then
+ *  drops every later exchange and the session never shrinks. The newest
+ *  @p keep_tail messages (this turn) stay. Caller frees the returned cJSON string.
+ */
+static char *session_array_print_fitting(cJSON *arr, size_t out_size, int keep_tail,
+	const char *session_id)
+{
+	char *printed;
+	int dropped = 0;
+
+	if (!arr || out_size == 0 || keep_tail < 1)
+		return NULL;
+	printed = cJSON_PrintUnformatted(arr);
+	while (printed && strlen(printed) >= out_size && cJSON_GetArraySize(arr) > keep_tail) {
+		cJSON_DeleteItemFromArray(arr, 0);
+		cJSON_free(printed);
+		printed = cJSON_PrintUnformatted(arr);
+		dropped++;
+	}
+	if (dropped > 0) {
+		fprintf(stderr,
+			"agent: dropped %d oldest session message(s) to fit cap session_id=%s\n",
+			dropped, session_id ? session_id : "");
+	}
+	return printed;
+}
+
 /** Append user+assistant exchange to session JSON. Invalid or empty existing becomes []. */
 static int append_exchange_to_session_json(const char *existing_json, const char *user_message,
 	const char *assistant_content, char *out_buf, size_t out_size, const char *session_id)
@@ -527,21 +556,28 @@ static int append_exchange_to_session_json(const char *existing_json, const char
 			arr = NULL;
 		}
 	}
+	int added = 0;
 	if (!arr) arr = cJSON_CreateArray();
 	if (!arr) return -1;
 	cJSON *user_obj = cJSON_CreateObject();
 	if (user_obj) {
 		cJSON_AddItemToObject(user_obj, "role", cJSON_CreateString("user"));
 		cJSON_AddItemToObject(user_obj, "content", cJSON_CreateString(user_message ? user_message : ""));
 		cJSON_AddItemToArray(arr, user_obj);
+		added++;
 	}
 	cJSON *asst_obj = cJSON_CreateObject();
 	if (asst_obj) {
 		cJSON_AddItemToObject(asst_obj, "role", cJSON_CreateString("assistant"));
 		cJSON_AddItemToObject(asst_obj, "content", cJSON_CreateString(assistant_content ? assistant_content : ""));
 		cJSON_AddItemToArray(arr, asst_obj);
+		added++;
+	}
+	if (added == 0) {
+		cJSON_Delete(arr);
+		return -1;
 	}
-	char *printed = cJSON_PrintUnformatted(arr);
+	char *printed = session_array_print_fitting(arr, out_size, added, session_id);
 	cJSON_Delete(arr);
 	return copy_printed_session_json(out_buf, out_size, printed, session_id);
 }

tests/test_agent.c

@@ -2046,10 +2046,11 @@ static const provider_t keep_history_provider = {
 };
 
 /**
- * Concrete trigger: near-cap session JSON + fat assistant reply.
- * Before the fix, append truncated mid-JSON, session_save persisted corrupt
- * payload, and the next agent_run parse wiped history. After the fix, overflow
- * refuses to save and prior history remains parseable.
+ * Concrete trigger: near-cap session JSON (count still under max_context_messages)
+ * plus a fat assistant reply. A truncated save must not replace the row with
+ * invalid JSON. Dropping the new turn instead leaves the byte cap stuck, so
+ * every later turn is forgotten. The save stays valid JSON, keeps the newer
+ * stored turn, and persists this exchange by evicting the oldest message.
  */
 static int test_session_overflow_does_not_corrupt_history(void)
 {
@@ -2058,6 +2059,7 @@ static int test_session_overflow_does_not_corrupt_history(void)
 	const char *db_path = "build/test_agent_overflow.db";
 	const char *config_path = "build/test_agent_overflow.toml";
 	const char *session_id = "cli:overflow";
+	const char *oldest = "OLDEST_TURN_SHOULD_DROP";
 	const char *marker = "UNIQUE_HISTORY_MARKER_xyz";
 	/* Two large messages keep msg_count under max_context so compaction does not shrink first. */
 	enum { PAD_A = 62 * 1024, PAD_B = 62 * 1024, LOAD_CAP = 130 * 1024 };
@@ -2079,12 +2081,12 @@ static int test_session_overflow_does_not_corrupt_history(void)
 	pad_a[PAD_A] = '\0';
 	memset(pad_b, 'B', PAD_B);
 	pad_b[PAD_B] = '\0';
-	need = strlen(marker) + PAD_A + PAD_B + 128;
+	need = strlen(oldest) + strlen(marker) + PAD_A + PAD_B + 128;
 	session_json = malloc(need);
 	if (!session_json) goto cleanup;
 	off = (size_t)snprintf(session_json, need,
-		"[{\"role\":\"user\",\"content\":\"%s%s\"},{\"role\":\"assistant\",\"content\":\"%s\"}]",
-		marker, pad_a, pad_b);
+		"[{\"role\":\"user\",\"content\":\"%s%s\"},{\"role\":\"assistant\",\"content\":\"%s%s\"}]",
+		oldest, pad_a, marker, pad_b);
 	if (off == 0 || off >= need) goto cleanup;
 	if (session_save(session_id, session_json) != 0) goto cleanup;
 
@@ -2120,6 +2122,14 @@ static int test_session_overflow_does_not_corrupt_history(void)
 	}
 	cJSON_Delete(parsed);
 	if (strstr(loaded, marker) == NULL) goto cleanup;
+	if (strstr(loaded, "push over the limit") == NULL) {
+		fprintf(stderr, "FAIL: tests/test_agent.c: overflow dropped the new turn\n");
+		goto cleanup;
+	}
+	if (strstr(loaded, oldest) != NULL) {
+		fprintf(stderr, "FAIL: tests/test_agent.c: overflow kept the oldest turn and cannot fit\n");
+		goto cleanup;
+	}
 
 	/* Next turn must still see prior history (not wiped to empty []). */
 	spy_roles_clear();