diff --git a/.changeset/20534-import-date-cell-iso-real-day.md b/.changeset/20534-import-date-cell-iso-real-day.md new file mode 100644 index 00000000000..86693a4d079 --- /dev/null +++ b/.changeset/20534-import-date-cell-iso-real-day.md @@ -0,0 +1,89 @@ +--- +'@objectstack/rest': minor +--- + +fix(rest): `POST /api/v1/data/:object/import` reads a `date`, `datetime` or `time` cell only in ISO 8601, the export's own `YYYY-MM-DD HH:mm:ss` or a year-first date (`2026/7/15`), on a calendar day that exists, and keeps a `date`'s year at four digits (#20534) + +Clause-②: no (narrowing) + +**BREAKING for callers of the import door.** A text cell for a `date`, +`datetime` or `time` field is now read only in one of these spellings, after +trimming: + +- `YYYY-MM-DD`; +- `YYYY-MM-DDTHH:MM[:SS[.fraction]]`, then `Z`, a `+HH:MM` / `-HH:MM` / + `+HHMM` offset, or nothing (a wall clock, read in the importing user's + business timezone, as before); +- `YYYY-MM-DD HH:MM[:SS[.fraction]]` with no offset, which is what the export + writes for a `datetime` cell; +- a year-first date, `YYYY/M/D` or `YYYY-M-D` (a four-digit year, a one- or + two-digit month and day, the same separator twice), optionally followed by + one space and `H:MM` or `H:MM:SS` with no offset, read exactly as the + export shape is; +- for a `time` field, also a bare `HH:MM` / `HH:MM:SS`. + +The day must exist. Every other cell is that row's `invalid_date` error, with +the importer's existing sentence ("is not a valid date" / "datetime" / +"time"). The reader used to hand such a cell to the JavaScript date parser, +which read it in the SERVER PROCESS's timezone and month-first, and rolled an +impossible day into the next month, so the import reported success and stored +a different value. For each shape, change the cell FROM the refused spelling +TO an admitted one: + +- **An impossible day.** FROM `2026-02-30`, `2026-02-29`, `2026-04-31` in any + spelling (a `datetime` `2026-02-30` was stored as 2 March, and so was a + `date` written `2026-02-30T10:00:00Z`) TO the day you mean. Nothing is + rolled over. +- **A locale or prose date.** FROM `07/15/2026`, `07/15/2026 10:00`, + `07/08/2026`, `15 July 2026`, `Jul 15 2026 10:00` (stored hours apart on a + New York and a Shanghai server, a `date` a day apart, and `07/08/2026` read + as 8 July) TO `2026-07-15`, `2026-07-15 10:00`, `2026-07-08` or + `2026-08-07`. No timezone and no field order is guessed. Converting a + spreadsheet column to ISO (in Excel, the cell format `yyyy-mm-dd` or + `yyyy-mm-dd hh:mm:ss`) before export is the fix. +- **A year-first date outside its one form.** FROM a mixed separator + (`2026/7-15`) TO `2026/7/15` or `2026-07-15`. FROM a `T` or a zone on the + year-first form (`2026/7/15T9:00`, `2026/7/15 9:00Z`) TO `2026/7/15 9:00` + (a wall clock in the business timezone) or the ISO `2026-07-15T09:00:00Z`. + FROM a fraction of a second (`2026/07/15 10:00:00.123`) TO + `2026-07-15 10:00:00.123`. A two-digit year (`26/7/15`) is refused, as it + was. +- **A zone after a space, or lower-case `t` / `z`.** FROM + `2026-07-15 10:00Z`, `2026-07-15 10:00:00+08:00`, `2026-07-15t10:00:00z` TO + `2026-07-15T10:00Z`, `2026-07-15T10:00:00+08:00`, `2026-07-15T10:00:00Z`, + the spellings the create and update doors take. +- **A zone-naive `24:00`.** FROM `2026-07-15 24:00` or `2026/7/15 24:00` (read + in the server's zone) TO `2026-07-16 00:00` or `2026/7/16 0:00`. + `2026-07-15T24:00:00Z`, which names its instant, reads as before. +- **A number, reduced or expanded forms.** FROM a JSON number such as `2026` + or an Excel serial, `2026`, `2026-07`, `+002026-07-15` TO `2026-01-01`, + `2026-07-01`, `2026-07-15`. + +**Kept: year-first dates.** `2026/7/15`, `2026/07/15`, `2026-7-15`, +`2026/7/15 9:00` and `2026/08/01 06:00:00`, Excel's default short date in +zh-CN and ja-JP, stay admitted. They are now held to the same rules as every +other cell: the day must exist (`2026/2/30` is refused, never rolled into +March), the hour runs 0 to 23, and the day is stored in its padded ISO form +(`2026/7/15` is stored as `2026-07-15`). A year-first date with no clock +given to a `time` field reads as `00:00:00`, as an ISO day does; it used to +read the server's zone (`04:00:00` on a New York server). + +**The year keeps four digits.** A `date` cell for a year from 0001 to 0999 +(`0500-01-01`) used to leave the reader as `500-01-01`, which the write door +refuses, so the import refused a day the create door takes. It is stored as +written now. A bare day read into a `datetime` field in years 0001 to 0099 +(`0050-01-01`) used to be stored in the 1900s (`1950-01-01T00:00:00.000Z`); it +is stored in its own year now. + +**What is not affected.** Every ISO 8601 cell and every export-shape cell on +a real day reads exactly as before, including a zone-naive cell read in the +business timezone and an offset-bearing cell honoured as written. An xlsx +cell that Excel stores as a date is unaffected: it reaches the reader as the +export shape. The dry run answers the same verdicts as the real write. A +refused cell fails only its own row, and the rest of the batch imports as +before. The same narrowing applies to the exported `coerceRow` helper. + +**If you are refused.** The row's result carries `code: 'invalid_date'` and +quotes the cell, so the file can be corrected and re-imported. + + diff --git a/packages/rest/src/import-coerce.test.ts b/packages/rest/src/import-coerce.test.ts index 361f6b4b3f6..e1c409fb02b 100644 --- a/packages/rest/src/import-coerce.test.ts +++ b/packages/rest/src/import-coerce.test.ts @@ -6,7 +6,7 @@ * `coerceRow` driven by a fake reference resolver. */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, afterEach } from 'vitest'; import { parseBooleanCell, parseNumberCell, @@ -99,6 +99,119 @@ describe('parseDateCell', () => { }); }); +/** + * [#20534] A text cell is read only in ISO 8601, the export's own + * `YYYY-MM-DD HH:mm:ss` or a year-first date (`2026/7/15`, `2026/7/15 9:00`, + * by the maintainer ruling on the card), on a calendar day that exists; + * everything else is refused (`undefined`, so the row's `invalid_date`), never rolled over, never + * read in the host's zone and never read month-first. A `date`'s year keeps + * four digits. Every case runs under two host zones that disagree by twelve + * hours, and must answer the same under both — the host-zone reading this + * removes answered differently (`07/15/2026 10:00` was `…T14:00Z` in New York + * and `…T02:00Z` in Shanghai). + */ +describe('[#20534] parseDateCell — ISO 8601, the export shape or a year-first date, on a real day', () => { + const HOST_ZONES = ['America/New_York', 'Asia/Shanghai']; + const originalTz = process.env.TZ; + afterEach(() => { + if (originalTz === undefined) delete process.env.TZ; + else process.env.TZ = originalTz; + }); + + /** Run `fn` under each host zone; the answers must agree. */ + function onBothHosts(fn: () => string | undefined): string | undefined { + const answers = HOST_ZONES.map((tz) => { + process.env.TZ = tz; + expect(Intl.DateTimeFormat().resolvedOptions().timeZone).toBe(tz); + return fn(); + }); + expect(answers[1]).toBe(answers[0]); + return answers[0]; + } + + type Kind = 'date' | 'datetime' | 'time'; + + const REFUSED: ReadonlyArray = [ + // An impossible day, in every shape and on every branch — was rolled over. + ['2026-02-30', 'date'], ['2026-02-30', 'datetime'], ['2026-02-30', 'time'], + ['2026-02-29', 'datetime'], ['2026-04-31', 'datetime'], + ['2026-02-30 10:00', 'datetime'], ['2026-02-30 10:00', 'date'], ['2026-02-30 10:00', 'time'], + ['2026-02-30T10:00:00Z', 'datetime'], ['2026-02-30T10:00:00Z', 'date'], ['2026-02-30T10:00:00Z', 'time'], + // Locale and prose spellings — were read in the host zone, and month-first. + ['07/15/2026 10:00', 'datetime'], ['07/15/2026 10:00', 'date'], ['07/15/2026 10:00', 'time'], + ['07/08/2026', 'datetime'], ['07/08/2026', 'date'], + ['07/15/2026', 'date'], ['15 July 2026', 'date'], ['15 July 2026', 'datetime'], + ['Jul 15 2026 10:00', 'time'], ['Wed, 15 Jul 2026 10:00:00 GMT', 'datetime'], + // Year-first, outside its one form: an impossible day, a mixed separator, + // a clock out of range, a `T`, a zone, a fraction, a two-digit year. + ['2026/2/30', 'date'], ['2026/2/30', 'datetime'], ['2026/7-15', 'date'], + ['2026/7/15 24:00', 'datetime'], ['2026/7/15 9:60', 'datetime'], + ['2026/7/15T9:00', 'datetime'], ['2026/7/15 9:00Z', 'datetime'], + ['2026/7/15 9:00:00.5', 'datetime'], ['26/7/15', 'date'], ['07/15/2026', 'datetime'], + // Reduced / expanded forms, a zone after a space, lower-case `t` / `z`. + ['2026', 'date'], ['2026-07', 'datetime'], ['+002026-07-15', 'date'], + ['2026-07-15 10:00Z', 'datetime'], ['2026-07-15 10:00:00+08:00', 'datetime'], + ['2026-07-15t10:00:00z', 'datetime'], + // A zone-naive 24:00 — was handed to `new Date(s)`, in the host zone. + ['2026-07-15 24:00', 'datetime'], ['2026-07-15T24:00:00', 'date'], + // A number — `new Date(String(n))` read it as a year, in the host zone. + [2026, 'date'], [45000, 'datetime'], [45000, 'time'], + ]; + + it.each(REFUSED)('refuses %j as a %s cell on every host', (cell, kind) => { + expect(onBothHosts(() => parseDateCell(cell, kind))).toBeUndefined(); + }); + + const ADMITTED: ReadonlyArray = [ + // ISO 8601 and the export shape keep their readings. + ['2026-07-15', 'date', '2026-07-15'], + ['2026-07-15', 'datetime', '2026-07-15T00:00:00.000Z'], + ['2028-02-29', 'date', '2028-02-29'], + ['2028-02-29T10:00:00Z', 'datetime', '2028-02-29T10:00:00.000Z'], + ['2026-07-15 10:00:00', 'datetime', '2026-07-15T10:00:00.000Z'], + ['2026-07-15 10:00:00', 'date', '2026-07-15'], + ['2026-07-15 10:00:00', 'time', '10:00:00'], + ['2026-07-15T10:00', 'datetime', '2026-07-15T10:00:00.000Z'], + ['2026-07-15 10:00:00.123', 'datetime', '2026-07-15T10:00:00.123Z'], + ['2026-07-15T10:00:00Z', 'datetime', '2026-07-15T10:00:00.000Z'], + ['2026-07-15T10:00:00+08:00', 'datetime', '2026-07-15T02:00:00.000Z'], + ['2026-07-15T10:00:00+0800', 'datetime', '2026-07-15T02:00:00.000Z'], + ['2026-07-15T02:00:00+08:00', 'date', '2026-07-14'], + ['2026-07-15T10:00:00+08:00', 'time', '02:00:00'], + ['2026-07-15T24:00:00Z', 'datetime', '2026-07-16T00:00:00.000Z'], + [' 2026-07-15 ', 'date', '2026-07-15'], + ['10:00', 'time', '10:00:00'], + // A year-first date (Excel's zh-CN / ja-JP short date): the padded ISO day, + // and a clock read as a wall clock exactly as the export shape's is. + ['2026/6/3', 'date', '2026-06-03'], + ['2026/07/15', 'date', '2026-07-15'], + ['2026-7-15', 'date', '2026-07-15'], + ['2026/7/15', 'datetime', '2026-07-15T00:00:00.000Z'], + ['2026/7/15 9:00', 'datetime', '2026-07-15T09:00:00.000Z'], + ['2026/7/15 9:00', 'date', '2026-07-15'], + ['2026/7/15 9:00', 'time', '09:00:00'], + ['2026/08/01 06:00:00', 'datetime', '2026-08-01T06:00:00.000Z'], + ['2026-07-15 9:00', 'datetime', '2026-07-15T09:00:00.000Z'], + ['2028/2/29', 'date', '2028-02-29'], + ['0500/1/1', 'date', '0500-01-01'], + // The year keeps four digits on every `date` branch. + ['0500-01-01', 'date', '0500-01-01'], + ['0001-01-01', 'date', '0001-01-01'], + ['0999-12-31 10:00:00', 'date', '0999-12-31'], + ['0050-01-01T10:00:00Z', 'date', '0050-01-01'], + [new Date('0500-01-01T00:00:00Z'), 'date', '0500-01-01'], + // A bare day into a `datetime` is spelled from the day, never `Date.UTC(y, …)`, + // which read years 0..99 as 1900..1999 (`0001-01-01` was stored as 1901). + ['0001-01-01', 'datetime', '0001-01-01T00:00:00.000Z'], + ['0050-01-01', 'datetime', '0050-01-01T00:00:00.000Z'], + ['0500-01-01', 'datetime', '0500-01-01T00:00:00.000Z'], + ]; + + it.each(ADMITTED)('reads %j as a %s cell as %j on every host', (cell, kind, stored) => { + expect(onBothHosts(() => parseDateCell(cell, kind))).toBe(stored); + }); +}); + describe('matchOption', () => { const options = [{ label: '高', value: 'high' }, { label: '低', value: 'low' }]; it('matches by option value (code)', () => { diff --git a/packages/rest/src/import-coerce.ts b/packages/rest/src/import-coerce.ts index afaf602da19..4d5fb196925 100644 --- a/packages/rest/src/import-coerce.ts +++ b/packages/rest/src/import-coerce.ts @@ -31,7 +31,7 @@ * untouched, so an import stays byte-identical to the pre-coercion behaviour. */ -import { zonedWallClockToUtcMs, type WallClockParts } from '@objectstack/core'; +import { temporalStorageForm, zonedWallClockToUtcMs, type WallClockParts } from '@objectstack/core'; import type { ExportFieldMeta } from './export-format.js'; import { SINGLE_OPTION_TYPES as OPTION_TYPES, @@ -273,36 +273,143 @@ function pad2(n: number): string { const TIME_OF_DAY = /^([01]\d|2[0-3]):[0-5]\d(:[0-5]\d)?$/; /** - * A date-time cell carrying **no offset**: `YYYY-MM-DD HH:mm[:ss[.sss]]`, `T` - * or space separated, `/` accepted for `-` like the date fast path. Anchored at - * both ends, so a trailing `Z` or `+08:00` does NOT match — that cell already - * names an instant and is left to `Date.parse` (#8485 ruling: an explicit offset - * keeps being honoured exactly as written). + * [#20534] The text shapes a `date` / `datetime` / `time` cell is read in, + * after trimming — ISO 8601's extended calendar date and date-time, and the + * platform's own export shape: + * + * - `YYYY-MM-DD` — a calendar day; + * - `YYYY-MM-DDTHH:MM[:SS[.f…]]`, then `Z`, a `±HH:MM` / `±HHMM` offset, or + * nothing (a zone-naive wall clock); + * - `YYYY-MM-DD HH:MM[:SS[.f…]]`, zone-naive only — the `YYYY-MM-DD HH:mm:ss` + * the export writes for a `datetime` cell and an xlsx date cell is read as. + * + * A four-digit year, two-digit month, day, hour, minute and second, an + * upper-case `T` and `Z`: the spellings the write door admits for a `datetime` + * string (`@objectstack/objectql`'s `record-validator.ts`, #20525), each of + * which names one day and one clock whatever host reads it. Every other + * spelling is refused, never guessed: `07/15/2026` and `15 July 2026` went to + * `new Date(s)`, which reads them in the SERVER PROCESS's zone (the same cell + * stored as `2026-07-15T14:00Z` on a New York host and `…T02:00Z` on a + * Shanghai one, a `date` a day apart) and reads `07/08/2026` month-first. A + * space before a zone (`2026-07-15 10:00Z`) is refused as the write door + * refuses it. The one other text shape read at all is the year-first date, + * {@link YEAR_FIRST_CELL}. + */ +const ISO_TEMPORAL_CELL = + /^(\d{4})-(\d{2})-(\d{2})(?:(T| )(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?(Z|[+-]\d{2}:?\d{2})?)?$/; + +/** + * [#20534] Does `year-month-day` name a calendar day that exists — month + * 01..12, day 01 to that month's length, February 29 only in a leap year? + * Arithmetic, never a `Date` round trip: `Date.UTC` and `Date.parse` ROLL an + * impossible day over (`2026-02-30` is March 2), which is the defect this + * refuses, and `Date.UTC` reads a year 0..99 as 1900..1999. The same rule as + * the write door's `namesRealCalendarDay` (#20525), which is private there. + */ +function namesRealCalendarDay(year: number, month: number, day: number): boolean { + if (month < 1 || month > 12 || day < 1) return false; + const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const length = month === 2 ? (leap ? 29 : 28) : month === 4 || month === 6 || month === 9 || month === 11 ? 30 : 31; + return day <= length; +} + +/** + * A cell in one of the {@link ISO_TEMPORAL_CELL} shapes, on a day that exists. + * `day` is the cell's own `YYYY-MM-DD` text — the year keeps the four digits it + * was written with, so `0500-01-01` is never re-spelled `500-01-01`. Exactly + * one reading follows it: none (a bare day), `wall` (a zone-naive clock) or + * `instantMs` (the instant a zone-bearing cell names). + */ +interface IsoTemporalCell { + day: string; + wall?: WallClockParts & { hour: number; minute: number; second: number; millisecond: number }; + instantMs?: number; +} + +/** + * Read a trimmed cell as an {@link ISO_TEMPORAL_CELL} shape, or `undefined`. + * An impossible day, an out-of-range clock (`24:00` zone-naive, `10:60`) and a + * zone after a space are `undefined` too — refused, never rolled over and + * never handed to `new Date(s)`. A zone-bearing cell is read by `Date.parse`, + * which reads each of these ISO spellings as the same instant on every host + * (`T24:00Z` is the next day's midnight, as ISO 8601 has it). + */ +function readIsoTemporalCell(s: string): IsoTemporalCell | undefined { + const m = ISO_TEMPORAL_CELL.exec(s); + if (!m) return undefined; + const [, y, mo, d, sep, hh, mi, ss, frac, zone] = m; + if (!namesRealCalendarDay(Number(y), Number(mo), Number(d))) return undefined; + const day = `${y}-${mo}-${d}`; + if (sep === undefined) return { day }; + if (zone !== undefined) { + if (sep !== 'T') return undefined; + const instantMs = Date.parse(s); + return Number.isNaN(instantMs) ? undefined : { day, instantMs }; + } + const wall = { + year: Number(y), + month: Number(mo), + day: Number(d), + hour: Number(hh), + minute: Number(mi), + second: ss ? Number(ss) : 0, + millisecond: frac ? Number(frac.slice(0, 3).padEnd(3, '0')) : 0, + }; + if (wall.hour > 23 || wall.minute > 59 || wall.second > 59) return undefined; + return { day, wall }; +} + +/** + * [#20534, maintainer ruling] The year-first date a spreadsheet writes, after + * trimming: `YYYY/M/D` or `YYYY-M-D` — a four-digit year, a one- or two-digit + * month and day, the SAME separator in both places — optionally followed by + * one space and a zone-naive `H:MM` or `H:MM:SS` with a one- or two-digit hour. + * `2026/7/15`, `2026/07/15`, `2026-7-15`, `2026/7/15 9:00`, + * `2026/08/01 06:00:00`, `2026-07-15 9:00`. + * + * It is Excel's default short date in zh-CN and ja-JP, and a CSV saved from + * Excel writes the text it displays. The year comes first, so there is no + * field order to guess and no zone to read: it names one day and one wall + * clock on every host. No zone, no fraction and no `T` separator are read in + * this form, and a mixed separator (`2026/7-15`) is not this form. A + * month-first or day-first date (`07/15/2026`, `15/07/2026`) and a two-digit + * year (`26/7/15`) stay refused. */ -const NAIVE_DATE_TIME = - /^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T ](\d{1,2}):([0-5]\d)(?::([0-5]\d))?(?:\.(\d{1,3})\d*)?$/; +const YEAR_FIRST_CELL = /^(\d{4})([/-])(\d{1,2})\2(\d{1,2})(?: (\d{1,2}):(\d{2})(?::(\d{2}))?)?$/; /** - * Read an offset-free cell as the wall clock it is, or `undefined` when the - * shape does not match (caller falls through to `Date.parse`). Out-of-range - * components are rejected here rather than silently rolled over by `Date.UTC`. + * Read a trimmed cell as a {@link YEAR_FIRST_CELL} shape, or `undefined`. The + * rules every other admitted cell keeps: the day must exist + * ({@link namesRealCalendarDay}, never rolled over), the hour runs 0..23 and + * the minute and second 00..59 (`24:00` is refused), and the day is the padded + * ISO `YYYY-MM-DD` (`2026/7/15` → `2026-07-15`). A clock is a wall clock, read + * exactly as the export shape's is. */ -function parseNaiveWallClock(s: string): WallClockParts | undefined { - const m = NAIVE_DATE_TIME.exec(s); +function readYearFirstCell(s: string): IsoTemporalCell | undefined { + const m = YEAR_FIRST_CELL.exec(s); if (!m) return undefined; - const parts: WallClockParts = { - year: Number(m[1]), - month: Number(m[2]), - day: Number(m[3]), - hour: Number(m[4]), - minute: Number(m[5]), - second: m[6] ? Number(m[6]) : 0, - millisecond: m[7] ? Number(m[7].padEnd(3, '0')) : 0, + const [, y, , mo, d, hh, mi, ss] = m; + const month = Number(mo); + const date = Number(d); + if (!namesRealCalendarDay(Number(y), month, date)) return undefined; + const day = `${y}-${pad2(month)}-${pad2(date)}`; + if (hh === undefined) return { day }; + const wall = { + year: Number(y), + month, + day: date, + hour: Number(hh), + minute: Number(mi), + second: ss ? Number(ss) : 0, + millisecond: 0, }; - if (parts.month < 1 || parts.month > 12) return undefined; - if (parts.day < 1 || parts.day > 31) return undefined; - if ((parts.hour ?? 0) > 23) return undefined; - return parts; + if (wall.hour > 23 || wall.minute > 59 || wall.second > 59) return undefined; + return { day, wall }; +} + +/** The `HH:MM:SS` UTC clock of an instant. */ +function utcClock(t: Date): string { + return `${pad2(t.getUTCHours())}:${pad2(t.getUTCMinutes())}:${pad2(t.getUTCSeconds())}`; } /** @@ -310,16 +417,46 @@ function parseNaiveWallClock(s: string): WallClockParts | undefined { * - `date` → `YYYY-MM-DD` * - `datetime` → full ISO-8601 (`toISOString`) * - `time` → `HH:MM` / `HH:MM:SS` - * Returns `undefined` when the cell is not a recognisable date/time. + * Returns `undefined` when the cell is not a recognisable date/time, and the + * caller fails the row with `invalid_date` (`import_invalid_date` / + * `import_invalid_datetime` / `import_invalid_time`). + * + * ## Which text is read at all (#20534) * - * Unambiguous `YYYY-MM-DD` / `YYYY/MM/DD` inputs are normalised directly to - * avoid timezone drift; everything else falls back to `Date.parse` (which - * covers ISO datetimes and locale-default `MM/DD/YYYY`). + * A text cell is read only in an {@link ISO_TEMPORAL_CELL} shape — ISO 8601, + * or the export's own `YYYY-MM-DD HH:mm:ss` — or as a year-first date + * ({@link YEAR_FIRST_CELL}: `2026/7/15`, `2026/7/15 9:00`, stored padded, the + * clock read exactly as the export shape's), and only on a calendar day that + * exists. A `time` cell may also be a bare `HH:MM` / `HH:MM:SS`. Every other + * cell is refused on every branch; nothing reaches `new Date(s)`: + * + * - **an impossible day** (`2026-02-30`, `2026-02-29`, `2026-04-31`, + * `2026/2/30`, in any of the shapes) — `Date.UTC` and `Date.parse` rolled it + * into the next month, so a `datetime` was stored as March 2 and a `date` + * given in the `T…Z` spelling likewise; never rolled over now; + * - **a locale or prose spelling** (`07/15/2026 10:00`, `07/08/2026`, + * `15 July 2026`) — `new Date(s)` read it in the server process's zone and + * month-first, so the stored instant, and a `date`'s day, were properties + * of the deployment host; no zone and no field order is guessed now; + * - **a number** (`2026`, an Excel serial) — `new Date(String(n))` read it + * as a year, in the process zone. + * + * An xlsx date cell is unaffected: `import-prepare.ts` renders it as the + * export shape before it gets here. A `Date` (a programmatic caller's) names + * an instant and is read as before, save that a `date`'s year is padded. + * + * The year is padded to four digits on every `date` branch: a text cell keeps + * the four digits it was written with, and an instant takes core's + * `temporalStorageForm` `date` rule, which pads 0001..0999. `0500-01-01` used + * to leave here as `500-01-01`, which the write door refuses, so the import + * refused a day the write door takes. A bare day read into a `datetime` is + * midnight UTC spelled from the day itself, never `Date.UTC(y, …)`, which read + * `0050-01-01` as 1950. * * ## Which clock an offset-free cell is read in (#8485) * * A spreadsheet cell like `2026-08-01 06:00:00` carries no offset, so it is a - * **wall clock**, not an instant — and `new Date(s)` resolves it against the + * **wall clock**, not an instant — and `new Date(s)` resolved it against the * **process** `TZ`. That made the stored instant a property of the deployment * host: the same file, same tenant, same cell landed eight hours apart on two * hosts, decided by a setting nobody authoring the spreadsheet can see. Since @@ -327,17 +464,17 @@ function parseNaiveWallClock(s: string): WallClockParts | undefined { * advertised export → edit → re-import round trip was lossless only where the * host `TZ` happened to equal that zone. * - * So a naive **datetime** cell is now read in `timezone` — the caller's + * So a naive **datetime** cell is read in `timezone` — the caller's * `ExecutionContext.timezone`, the same value the export renders in — through * `@objectstack/core`'s `zonedWallClockToUtcMs` (DST-safe via the platform tz * database, and the primitive the date-bucket drill path already used in its * date-only form). Three things deliberately do NOT change: * * - **an offset-bearing cell** (`…Z`, `…+08:00`) already names one instant and - * is honoured exactly as written — `NAIVE_DATE_TIME` cannot match it; - * - **the date-only fast path** stays UTC (ECMAScript reads a date-only form as - * UTC, and a `date` is a timezone-naive calendar day under ADR-0053 — moving - * it would re-time every date-only import to fix nothing); + * is honoured exactly as written; + * - **a bare day** stays UTC midnight for a `datetime` (a `date` is a + * timezone-naive calendar day under ADR-0053 — moving it would re-time every + * date-only import to fix nothing); * - **no resolved timezone ⇒ UTC**, never the process clock. That is the * fallback the export cell path takes when no zone resolves, so the round * trip stays exact for deployments that configure none — and a process-`TZ` @@ -346,8 +483,8 @@ function parseNaiveWallClock(s: string): WallClockParts | undefined { * * For a naive cell landing in a `date` or `time` field the typed components are * taken verbatim (`2026-08-01 06:00:00` → `2026-08-01` / `06:00:00`), which is - * both zone-free and host-`TZ`-free; previously those two branches also read the - * cell through the process clock and could report the wrong calendar day. + * both zone-free and host-`TZ`-free. An offset-bearing cell landing in either + * takes the UTC calendar day or UTC clock of the instant it names. */ export function parseDateCell( raw: unknown, @@ -359,47 +496,30 @@ export function parseDateCell( // to re-interpret, so no zone question to answer. if (Number.isNaN(raw.getTime())) return undefined; if (kind === 'datetime') return raw.toISOString(); - if (kind === 'date') return `${raw.getUTCFullYear()}-${pad2(raw.getUTCMonth() + 1)}-${pad2(raw.getUTCDate())}`; - return `${pad2(raw.getUTCHours())}:${pad2(raw.getUTCMinutes())}:${pad2(raw.getUTCSeconds())}`; + // [#20534] Core's `date` storage rule: the UTC calendar day, the year padded. + if (kind === 'date') return String(temporalStorageForm(raw, 'date')); + return utcClock(raw); } const s = String(raw).trim(); if (s === '') return undefined; - const wall = parseNaiveWallClock(s); - - if (kind === 'time') { - if (TIME_OF_DAY.test(s)) return s.length === 5 ? `${s}:00` : s; - // A full datetime for a time field: take its clock component. Offset-free → - // the clock as typed; offset-bearing → the instant's UTC clock, as before. - if (wall) return `${pad2(wall.hour ?? 0)}:${pad2(wall.minute ?? 0)}:${pad2(wall.second ?? 0)}`; - const t = new Date(s); - if (!Number.isNaN(t.getTime())) return `${pad2(t.getUTCHours())}:${pad2(t.getUTCMinutes())}:${pad2(t.getUTCSeconds())}`; - return undefined; - } + if (kind === 'time' && TIME_OF_DAY.test(s)) return s.length === 5 ? `${s}:00` : s; - // Fast path: bare calendar date, no timezone games. - const ymd = s.match(/^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$/); - if (ymd) { - const y = Number(ymd[1]); - const mo = Number(ymd[2]); - const d = Number(ymd[3]); - if (mo < 1 || mo > 12 || d < 1 || d > 31) return undefined; - if (kind === 'date') return `${y}-${pad2(mo)}-${pad2(d)}`; - return new Date(Date.UTC(y, mo - 1, d)).toISOString(); - } + const cell = readIsoTemporalCell(s) ?? readYearFirstCell(s); + if (!cell) return undefined; - if (wall) { - if (kind === 'date') return `${wall.year}-${pad2(wall.month)}-${pad2(wall.day)}`; - const ms = zonedWallClockToUtcMs(wall, timezone); + if (cell.wall) { + if (kind === 'date') return cell.day; + if (kind === 'time') return `${pad2(cell.wall.hour)}:${pad2(cell.wall.minute)}:${pad2(cell.wall.second)}`; + const ms = zonedWallClockToUtcMs(cell.wall, timezone); return Number.isNaN(ms) ? undefined : new Date(ms).toISOString(); } - const parsed = new Date(s); - if (Number.isNaN(parsed.getTime())) return undefined; - if (kind === 'date') { - return `${parsed.getUTCFullYear()}-${pad2(parsed.getUTCMonth() + 1)}-${pad2(parsed.getUTCDate())}`; - } - return parsed.toISOString(); + // A bare day is midnight UTC; a zone-bearing cell is the instant it names. + const instant = new Date(cell.instantMs ?? Date.parse(`${cell.day}T00:00:00.000Z`)); + if (kind === 'datetime') return instant.toISOString(); + if (kind === 'date') return cell.instantMs === undefined ? cell.day : String(temporalStorageForm(instant, 'date')); + return utcClock(instant); } // ── options (select / multiselect) ───────────────────────────────── diff --git a/packages/rest/src/import-date-cell-iso-real-day.test.ts b/packages/rest/src/import-date-cell-iso-real-day.test.ts new file mode 100644 index 00000000000..0f264bde917 --- /dev/null +++ b/packages/rest/src/import-date-cell-iso-real-day.test.ts @@ -0,0 +1,287 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20534] `POST /api/v1/data/:object/import` reads a `date`, `datetime` or + * `time` text cell only in ISO 8601, the platform's own export shape + * (`YYYY-MM-DD HH:mm:ss`) or a year-first date (`2026/7/15`, `2026/7/15 9:00`, + * by the maintainer ruling on the card), on a calendar day that exists, and + * pads a `date`'s year to four digits — through the real route over a real `SqlDriver` + * (better-sqlite3 `:memory:`), under two host zones twelve hours apart. + * + * Measured through this route, JSON rows, `writeMode: 'insert'`, no business + * timezone, on `InMemoryDriver` and on `SqlDriver` (better-sqlite3) alike, + * under `TZ=America/New_York` and `TZ=Asia/Shanghai`, at the base + * (`f11b5f20a2`): + * + * | cell | kind | base: stored (New York · Shanghai) | head | + * |:--|:--|:--|:--| + * | `2026-02-30` | datetime | `2026-03-02T00:00:00.000Z` on both | row refused, `invalid_date` | + * | `2026-02-30 10:00` / `2026-02-30T10:00:00Z` | datetime | `2026-03-02T10:00:00.000Z` on both | row refused | + * | `2026-02-30T10:00:00Z` | date | `2026-03-02` on both | row refused | + * | `07/15/2026 10:00` | datetime | `…T14:00:00.000Z` · `…T02:00:00.000Z` | row refused | + * | `07/08/2026` | datetime | `2026-07-08T04:00Z` · `2026-07-07T16:00Z` (month-first) | row refused | + * | `07/15/2026` / `15 July 2026` | date | `2026-07-15` · `2026-07-14` | row refused | + * | `07/15/2026 10:00` | time | `14:00:00` · `02:00:00` | row refused | + * | `2026-07-15 24:00` | datetime | `2026-07-16T04:00Z` · `2026-07-15T16:00Z` | row refused | + * | `0500-01-01` / `0001-01-01` | date | row refused (`500-01-01` reached the write door) | `0500-01-01` / `0001-01-01` | + * | `0001-01-01` | datetime | `1901-01-01T00:00:00.000Z` | `0001-01-01T00:00:00.000Z` | + * | `2026/7/15` | date | `2026-07-15` | unchanged | + * | `2026/7/15 9:00` | datetime | `2026-07-15T09:00:00.000Z` | unchanged | + * | `2026/2/30` | date | refused (`2026-02-30` reached the write door) | row refused by the reader | + * + * The `InMemoryDriver` column is this file's by construction, not by a second + * arm: every cell is judged by the import's own reader (`parseDateCell`) before + * a driver is reached, so one verdict holds on every driver. A test import of + * `@objectstack/driver-memory` is also not this file's to add — its test + * consumers are a ruled, ledgered set (`pnpm check:driver-memory-census`). + * + * The reader's own case table is `import-coerce.test.ts`'s `[#20534]` block; + * this file pins the door. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; +import { loadExcelJs } from './xlsx-module.js'; + +const OBJECT = 'import_date_cell_20534'; + +const LEDGER = { + name: OBJECT, label: 'Ledger 20534', systemFields: false, + fields: { + id: { name: 'id', type: 'text' as const, primaryKey: true, label: 'ID' }, + d: { name: 'd', type: 'date' as const, label: 'Day' }, + dt: { name: 'dt', type: 'datetime' as const, label: 'At' }, + t: { name: 't', type: 'time' as const, label: 'Clock' }, + }, +}; + +type Field = 'd' | 'dt' | 't'; + +/** The card's rows: each used to be stored as some other value, or not at all. */ +const REFUSED: ReadonlyArray = [ + ['dt', '2026-02-30'], + ['dt', '2026-02-30 10:00'], + ['dt', '2026-02-30T10:00:00Z'], + ['d', '2026-02-30T10:00:00Z'], + ['d', '2026-02-30'], + ['dt', '07/15/2026 10:00'], + ['dt', '07/08/2026'], + ['d', '07/15/2026'], + ['d', '15 July 2026'], + ['t', '07/15/2026 10:00'], + ['dt', '2026-07-15 24:00'], + ['d', '2026/2/30'], +]; + +/** Admitted cells and what they store — the padding, the ISO control and the export shape. */ +const ADMITTED: ReadonlyArray = [ + ['d', '0500-01-01', '0500-01-01'], + ['d', '0001-01-01', '0001-01-01'], + ['dt', '0001-01-01', '0001-01-01T00:00:00.000Z'], + ['d', '2026-07-15', '2026-07-15'], + ['dt', '2026-07-15T10:00:00Z', '2026-07-15T10:00:00.000Z'], + ['dt', '2026-07-15T10:00:00+08:00', '2026-07-15T02:00:00.000Z'], + ['dt', '2026-07-15 10:00:00', '2026-07-15T10:00:00.000Z'], + ['t', '10:00', '10:00:00'], + // A year-first date (Excel's zh-CN / ja-JP short date) stays admitted, padded. + ['d', '2026/7/15', '2026-07-15'], + ['dt', '2026/7/15 9:00', '2026-07-15T09:00:00.000Z'], +]; + +function makeSqliteDriver() { + return new SqlDriver({ + client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true, + }); +} + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function makeRes() { + const chunks: string[] = []; + const res: any = { + write: (s: unknown) => { chunks.push(String(s)); return true; }, + end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + res._text = () => chunks.join(''); + return res; +} + +const liveEngines: ObjectQL[] = []; +afterEach(async () => { + while (liveEngines.length) { + try { await liveEngines.pop()?.destroy(); } catch { /* noop */ } + } +}); + +async function boot() { + const engine = new ObjectQL(); + liveEngines.push(engine); + engine.registerDriver(makeSqliteDriver(), true); + await engine.init(); + engine.registry.registerObject(LEDGER as any); + await engine.syncSchemas(); + const protocol = new ObjectStackProtocolImplementation(engine as any); + const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const find = (method: string, path: string) => + rest.getRoutes().find((r: any) => r.method === method && r.path === path) as any; + const importRoute = find('POST', '/api/v1/data/:object/import'); + const createRoute = find('POST', '/api/v1/data/:object'); + const exportRoute = find('GET', '/api/v1/data/:object/export'); + expect(importRoute).toBeDefined(); + expect(createRoute).toBeDefined(); + expect(exportRoute).toBeDefined(); + const send = async (route: any, req: Record) => { + const res = makeRes(); + await route.handler({ params: { object: OBJECT }, ...req } as any, res); + return res; + }; + return { + engine, + importRows: (body: Record) => send(importRoute, { body }), + create: (body: Record) => send(createRoute, { body }), + exportCsv: () => send(exportRoute, { query: { format: 'csv' } }), + }; +} + +const stored = async (engine: ObjectQL, id: string, field: Field) => + (await engine.findOne(OBJECT, { where: { id } }))?.[field]; + +// Two host zones twelve hours apart: a host-zone reading answers differently +// under each, so a cell that stores one value under both was not read in the +// host's zone. +const HOST_ZONES = [ + { tz: 'America/New_York', julyOffset: 240 }, + { tz: 'Asia/Shanghai', julyOffset: -480 }, +]; +const originalTz = process.env.TZ; + +describe.each(HOST_ZONES)('[#20534] /import date cells, host TZ=$tz', ({ tz, julyOffset }) => { + let ctx: Awaited>; + beforeEach(async () => { + process.env.TZ = tz; + // The host really did change — otherwise both legs assert one host. + expect(Intl.DateTimeFormat().resolvedOptions().timeZone).toBe(tz); + expect(new Date(2026, 6, 15).getTimezoneOffset()).toBe(julyOffset); + ctx = await boot(); + }); + afterEach(() => { + if (originalTz === undefined) delete process.env.TZ; + else process.env.TZ = originalTz; + }); + + it.each(REFUSED)('refuses the %s cell %j as that row\'s invalid_date and writes its sibling row', async (field, cell) => { + const res = await ctx.importRows({ + format: 'json', writeMode: 'insert', + rows: [{ id: 'bad', [field]: cell }, { id: 'good', d: '2026-07-15' }], + }); + + expect(res._status ?? 200).toBe(200); + expect(res._json).toMatchObject({ total: 2, ok: 1, errors: 1, created: 1 }); + expect(res._json.results[0]).toMatchObject({ + row: 1, ok: false, action: 'failed', field, code: 'invalid_date', + }); + // Refused, not stored as some other day or instant. + expect(await ctx.engine.findOne(OBJECT, { where: { id: 'bad' } })).toBeNull(); + expect(await stored(ctx.engine, 'good', 'd')).toBe('2026-07-15'); + }); + + it.each(ADMITTED)('stores the %s cell %j as %j', async (field, cell, value) => { + const res = await ctx.importRows({ + format: 'json', writeMode: 'insert', + rows: [{ id: 'r', [field]: cell }], + }); + + expect(res._json).toMatchObject({ total: 1, ok: 1, errors: 0, created: 1 }); + expect(await stored(ctx.engine, 'r', field)).toBe(value); + }); + + it('takes a padded year the write door takes, and stores what the write door stores', async () => { + for (const [i, day] of ['0500-01-01', '0001-01-01'].entries()) { + const direct = await ctx.create({ id: `w${i}`, d: day }); + expect(direct._status ?? 201).toBeLessThan(300); + const imported = await ctx.importRows({ format: 'json', writeMode: 'insert', rows: [{ id: `i${i}`, d: day }] }); + expect(imported._json).toMatchObject({ ok: 1, errors: 0 }); + expect(await stored(ctx.engine, `i${i}`, 'd')).toBe(await stored(ctx.engine, `w${i}`, 'd')); + expect(await stored(ctx.engine, `i${i}`, 'd')).toBe(day); + } + }); + + it('stores a year-first date-time as the same instant the export shape stores', async () => { + const res = await ctx.importRows({ + format: 'json', writeMode: 'insert', + rows: [{ id: 'yf', dt: '2026/7/15 9:00' }, { id: 'ex', dt: '2026-07-15 09:00:00' }], + }); + expect(res._json).toMatchObject({ total: 2, ok: 2, errors: 0, created: 2 }); + const yearFirst = await stored(ctx.engine, 'yf', 'dt'); + expect(yearFirst).toBe(await stored(ctx.engine, 'ex', 'dt')); + expect(yearFirst).toBe('2026-07-15T09:00:00.000Z'); + }); + + it('round-trips the export shape: an exported row re-imports as the same day and instant', async () => { + await ctx.engine.insert(OBJECT, { id: 'src', d: '2026-07-15', dt: '2026-07-15T10:00:00.000Z', t: '10:00:00' }); + const exported = await ctx.exportCsv(); + const lines = exported._text().split('\r\n').filter((l: string) => l.length > 0); + expect(lines[0]).toBe('ID,Day,At,Clock'); + expect(lines[1]).toBe('src,2026-07-15,2026-07-15 10:00:00,10:00:00'); + + const csv = [lines[0], lines[1].replace(/^src,/, 'back,')].join('\n'); + const res = await ctx.importRows({ format: 'csv', csv, writeMode: 'insert', mapping: { ID: 'id', Day: 'd', At: 'dt', Clock: 't' } }); + expect(res._json).toMatchObject({ total: 1, ok: 1, errors: 0, created: 1 }); + const back = await ctx.engine.findOne(OBJECT, { where: { id: 'back' } }); + const src = await ctx.engine.findOne(OBJECT, { where: { id: 'src' } }); + expect({ d: back?.d, dt: back?.dt, t: back?.t }).toEqual({ d: src?.d, dt: src?.dt, t: src?.t }); + expect(back?.dt).toBe('2026-07-15T10:00:00.000Z'); + }); + + it('reads an xlsx date cell as before', async () => { + const ExcelJS = await loadExcelJs(); + const wb = new ExcelJS.Workbook(); + const ws = wb.addWorksheet('Sheet1'); + ws.addRow(['ID', 'Day', 'At']); + ws.addRow(['x1', new Date('2026-06-30T00:00:00Z'), new Date('2026-06-30T10:00:00Z')]); + const xlsxBase64 = Buffer.from(await wb.xlsx.writeBuffer()).toString('base64'); + + const res = await ctx.importRows({ format: 'xlsx', xlsxBase64, writeMode: 'insert', mapping: { ID: 'id', Day: 'd', At: 'dt' } }); + expect(res._json).toMatchObject({ total: 1, ok: 1, errors: 0, created: 1 }); + expect(await stored(ctx.engine, 'x1', 'd')).toBe('2026-06-30'); + expect(await stored(ctx.engine, 'x1', 'dt')).toBe('2026-06-30T10:00:00.000Z'); + }); + + it('gives quoted CSV cells the same verdicts', async () => { + const csv = [ + 'ID,Day,At,Clock', + ...REFUSED.map(([field, cell], i) => `r${i},${field === 'd' ? `"${cell}"` : ''},${field === 'dt' ? `"${cell}"` : ''},${field === 't' ? `"${cell}"` : ''}`), + ].join('\n'); + const res = await ctx.importRows({ format: 'csv', csv, writeMode: 'insert', mapping: { ID: 'id', Day: 'd', At: 'dt', Clock: 't' } }); + + expect(res._json).toMatchObject({ total: REFUSED.length, ok: 0, errors: REFUSED.length }); + const failedFields = res._json.results.map((r: any) => [r.field, r.code]); + expect(failedFields).toEqual(REFUSED.map(([field]) => [field, 'invalid_date'])); + }); + + it('dry run predicts the same refusals and persists nothing', async () => { + const res = await ctx.importRows({ + format: 'json', writeMode: 'insert', dryRun: true, + rows: REFUSED.map(([field, cell], i) => ({ id: `d${i}`, [field]: cell })), + }); + + expect(res._json).toMatchObject({ dryRun: true, total: REFUSED.length, ok: 0, errors: REFUSED.length }); + for (const [i, r] of res._json.results.entries()) { + expect(r).toMatchObject({ ok: false, field: REFUSED[i][0], code: 'invalid_date' }); + } + for (const [i] of REFUSED.entries()) { + expect(await ctx.engine.findOne(OBJECT, { where: { id: `d${i}` } })).toBeNull(); + } + }); +}); diff --git a/packages/rest/src/import-integration.test.ts b/packages/rest/src/import-integration.test.ts index bbee2f0450f..43afdae950b 100644 --- a/packages/rest/src/import-integration.test.ts +++ b/packages/rest/src/import-integration.test.ts @@ -361,6 +361,8 @@ describe('import route — real engine + protocol integration', () => { expect(String(one.due)).toContain('2026-06-30'); const two = await engine.findOne('task', { where: { id: '2' } }); expect(two).toMatchObject({ title: '测试', done: false, priority: 'low', score: 3, owner: 'u2' }); + // [#20534] A year-first text cell (Excel's zh-CN short date) is read as its ISO day. + expect(two.due).toBe('2026-07-01'); }); it('reads xlsxBase64 without an explicit format and honors the sheet selector', async () => {