Add TSU format: Unicode version of TS style files (#2772)

TeXmacs/plugins/tsu/progs/data/tsu.scm

@@ -0,0 +1,40 @@
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; MODULE      : data/tsu.scm
+;; DESCRIPTION : tsu data format
+;; COPYRIGHT   : (C) 2024   Darcy Shen
+;;               (C) 2026   Eshaan Gupta
+;;
+;; This software falls under the GNU general public license version 3 or later.
+;; It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE
+;; in the root directory or <http://www.gnu.org/licenses/gpl-3.0.html>.
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(texmacs-module (data tsu))
+
+(define-format tsu
+  (:name "TSU")
+  (:suffix "tsu"))
+
+(tm-define (texmacs->tsu t)
+  (serialize-tsu (herk-tree->utf8-tree t)))
+
+(tm-define (tsu->texmacs t)
+  (utf8-tree->herk-tree (parse-tsu t)))
+
+(define (tsu-snippet->texmacs t)
+  (utf8-tree->herk-tree (parse-tsu-snippet t)))
+
+(converter tsu-document texmacs-tree
+  (:function tsu->texmacs))
+
+(converter texmacs-tree tsu-document
+  (:function texmacs->tsu))
+
+(converter tsu-snippet texmacs-tree
+  (:function tsu-snippet->texmacs))
+
+(converter texmacs-tree tsu-snippet
+  (:function texmacs->tsu))

TeXmacs/progs/init-research.scm

@@ -343,6 +343,7 @@
 (lazy-format (data mgs) mgs)
 (lazy-format (data stm) stm)
 (lazy-format (data tmu) tmu)
+(lazy-format (data tsu) tsu)
 (lazy-format (data docx) docx)
 (lazy-format (data html) html)
 (lazy-define (convert images tmimage)

TeXmacs/progs/texmacs/texmacs/tm-files.scm

@@ -235,7 +235,7 @@
   (learn-interactive 'recent-buffer (list (cons "0" (url->system name)))))
 
 (define (has-faithful-format? name)
-  (in? (url-suffix name) '("tm" "ts" "tp" "stm" "scm" "tmu")))
+  (in? (url-suffix name) '("tm" "ts" "tp" "stm" "scm" "tmu" "tsu")))
 
 (define (save-buffer-post name opts)
   ;;(display* "save-buffer-post " name "\n")
@@ -252,10 +252,10 @@
           (buffer-pretend-modified name)
           (set-message `(concat "Could not save " ,vname) "Save file"))
         (begin
-          (if (== (url-suffix name) "ts") (style-clear-cache))
+          (if (in? (url-suffix name) '("ts" "tsu")) (style-clear-cache))
           (autosave-remove name)
           (buffer-notify-recent name)
-          ;; Remember directory for file dialog 
+          ;; Remember directory for file dialog
           (remember-file-dialog-directory name)
           (set-message `(concat "Saved " ,vname) "Save file")
           (save-buffer-post name opts)))))
@@ -478,7 +478,7 @@
            (suffix (if (rescue-mode?) "#" "~"))
            (aname (if (url-scratch? name) name (url-autosave name suffix)))
            (fm (url-format name)))
-      (cond ((nin? fm (list "texmacs" "stm" "mgs" "tmu"))
+      (cond ((nin? fm (list "texmacs" "stm" "mgs" "tmu" "tsu"))
              (when (not (rescue-mode?))
                (set-message `(concat "Warning: " ,vname " not auto-saved")
                             "Auto-save file")))

TeXmacs/progs/texmacs/texmacs/tm-server.scm

@@ -357,7 +357,7 @@
             (set-message `(concat "Could not save " ,vname) "Save file")
             (callback #f))
           (begin
-            (if (== (url-suffix name) "ts") (style-clear-cache))
+            (if (in? (url-suffix name) '("ts" "tsu")) (style-clear-cache))
             (set-message `(concat "Saved " ,vname) "Save file")
             (save-buffer-post name opts)
             (callback #t))))))

src/Data/Convert/Mogan/from_tsu.cpp

@@ -0,0 +1,52 @@
+
+/******************************************************************************
+ * MODULE     : from_tsu.cpp
+ * DESCRIPTION: Conversion from the TSU format
+ * COPYRIGHT  : (C) 2024  Darcy Shen
+ *              (C) 2026  Eshaan Gupta
+ *******************************************************************************
+ * This software falls under the GNU general public license version 3 or later.
+ * It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE
+ * in the root directory or <http://www.gnu.org/licenses/gpl-3.0.html>.
+ ******************************************************************************/
+
+#include "convert.hpp"
+#include "tree_helper.hpp"
+
+#include <moebius/vars.hpp>
+
+using namespace moebius;
+
+/******************************************************************************
+ * Conversion of TSU strings to TeXmacs trees
+ ******************************************************************************/
+
+tree
+tsu_document_to_tree (string s) {
+  tree error (ERROR, "bad format or data");
+
+  if (starts (s, "<TSU|<tuple|")) {
+    int           i            = index_of (s, '>');
+    string        version_tuple= s (N ("<TSU|<tuple|"), i);
+    array<string> version_arr  = tokenize (version_tuple, "|");
+    string        tsu_version  = version_arr[0];
+    string        xmacs_version= version_arr[1];
+    tree          doc          = tmu_to_tree (s, xmacs_version);
+
+    if (is_compound (doc, "TeXmacs", 1) || is_expand (doc, "TeXmacs", 1) ||
+        is_apply (doc, "TeXmacs", 1))
+      doc= tree (DOCUMENT, doc);
+
+    if (!is_document (doc)) return error;
+
+    if (N (doc) == 0 || !is_compound (doc[0], "TeXmacs", 1)) {
+      tree d (DOCUMENT);
+      d << compound ("TeXmacs", string (TEXMACS_VERSION));
+      d << A (doc);
+      doc= d;
+    }
+
+    return doc;
+  }
+  return error;
+}

src/Data/Convert/Mogan/to_tmu.cpp

@@ -274,6 +274,15 @@ tmu_writer::write (tree t) {
  * Conversion of TeXmacs trees to TeXmacs strings
  ******************************************************************************/
 
+string
+mogan_tree_serialize (tree t) {
+  tmu_writer tmw;
+  tmw.write (t);
+  tmw.flush ();
+  tmw.buf << "\n";
+  return tmw.buf;
+}
+
 string
 tree_to_tmu (tree t) {
   if (!is_snippet (t)) {
@@ -294,9 +303,5 @@ tree_to_tmu (tree t) {
     t= r;
   }
 
-  tmu_writer tmw;
-  tmw.write (t);
-  tmw.flush ();
-  tmw.buf << "\n"; // append an extra newline at the end of TMU file
-  return tmw.buf;
+  return mogan_tree_serialize (t);
 }

src/Data/Convert/Mogan/to_tsu.cpp

@@ -0,0 +1,43 @@
+
+/******************************************************************************
+ * MODULE     : to_tsu.cpp
+ * DESCRIPTION: conversion of TeXmacs trees to the TSU file format
+ * COPYRIGHT  : (C) 2024  Darcy Shen
+ *              (C) 2026  Eshaan Gupta
+ *******************************************************************************
+ * This software falls under the GNU general public license version 3 or later.
+ * It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE
+ * in the root directory or <http://www.gnu.org/licenses/gpl-3.0.html>.
+ ******************************************************************************/
+
+#include "convert.hpp"
+#include "tree_helper.hpp"
+
+static const string TSU_VERSION= "1.1.0";
+
+/******************************************************************************
+ * Conversion of TeXmacs trees to TSU strings
+ ******************************************************************************/
+
+string
+tree_to_tsu (tree t) {
+  if (!is_snippet (t)) {
+    int  t_N= N (t);
+    tree r (t, t_N);
+    for (int i= 0; i < t_N; i++) {
+      if (is_compound (t[i], "style", 1)) {
+        tree style= t[i][0];
+        if (is_func (style, TUPLE, 1)) style= style[0];
+        r[i]   = copy (t[i]);
+        r[i][0]= style;
+      }
+      else if (is_compound (t[i], "TeXmacs")) {
+        r[i]= compound ("TSU", tuple (TSU_VERSION, string (XMACS_VERSION)));
+      }
+      else r[i]= t[i];
+    }
+    t= r;
+  }
+
+  return mogan_tree_serialize (t);
+}

src/Data/Convert/convert.hpp

@@ -53,11 +53,19 @@ tree                 nonumber_to_eqnumber (tree t);
 tree                 eqnumber_to_nonumber (tree t);
 string               search_metadata (tree doc, string kind);
 
+/*** Mogan serialization ***/
+string mogan_tree_serialize (tree t);
+
 /*** TMU ***/
 tree   tmu_to_tree (string s);
+tree   tmu_to_tree (string s, string version);
 tree   tmu_document_to_tree (string s);
 string tree_to_tmu (tree t);
 
+/*** TSU ***/
+tree   tsu_document_to_tree (string s);
+string tree_to_tsu (tree t);
+
 /*** Verbatim ***/
 string tree_to_verbatim (tree t, bool wrap= false, string enc= "default");
 tree   verbatim_to_tree (string s, bool wrap= false, string enc= "default");

src/Data/Document/new_style.cpp

@@ -63,6 +63,10 @@ static url
 resolve_local_style (string style_name) {
   string pack= style_name * ".ts";
   url    ret = resolve (url ("$TEXMACS_STYLE_PATH") * pack);
+  if (is_none (ret)) {
+    pack= style_name * ".tsu";
+    ret = resolve (url ("$TEXMACS_STYLE_PATH") * pack);
+  }
   if (DEBUG_IO) {
     debug_io << "Resolved local style: " << style_name << " -> " << ret << LF;
   }
@@ -73,6 +77,10 @@ static url
 resolve_relative_style (string style_name, url master) {
   string pack= style_name * ".ts";
   url    ret = resolve (expand (head (master) * url_ancestor () * pack));
+  if (is_none (ret)) {
+    pack= style_name * ".tsu";
+    ret = resolve (expand (head (master) * url_ancestor () * pack));
+  }
   if (DEBUG_IO) {
     debug_io << "Resolved relative style: (" << style_name << ", " << master
              << ")" << LF << "-> " << ret << LF;
@@ -135,7 +143,7 @@ static string
 cache_file_name_sub (tree t) {
   if (is_atomic (t)) {
     string s= t->label;
-    if (ends (s, ".ts")) {
+    if (ends (s, ".ts") || ends (s, ".tsu")) {
       url style= url_system (s);
       if (is_rooted_web (style)) {
         url local_style= get_from_web (s);
@@ -372,6 +380,7 @@ hidden_package (url u, string name, bool hidden) {
   if (hidden && is_atomic (u)) {
     string l= as_string (u);
     if (ends (l, ".ts")) l= l (0, N (l) - 3);
+    else if (ends (l, ".tsu")) l= l (0, N (l) - 4);
     else if (ends (l, ".hook")) l= l (0, N (l) - 5);
     else return false;
     return name == l;
@@ -415,6 +424,7 @@ compute_style_menu (url u, int kind) {
   if (is_atomic (u)) {
     string l= as_string (u);
     if (ends (l, ".ts")) l= l (0, N (l) - 3);
+    else if (ends (l, ".tsu")) l= l (0, N (l) - 4);
     else if (ends (l, ".hook")) l= l (0, N (l) - 5);
     else return "";
     string cmd ("set-main-style");

src/Scheme/L4/glue_convert.lua

@@ -71,6 +71,30 @@ function main()
                     "string"
                 }
             },
+            {
+                scm_name = "parse-tsu",
+                cpp_name = "tsu_document_to_tree",
+                ret_type = "tree",
+                arg_list = {
+                    "string"
+                }
+            },
+            {
+                scm_name = "serialize-tsu",
+                cpp_name = "tree_to_tsu",
+                ret_type = "string",
+                arg_list = {
+                    "tree"
+                }
+            },
+            {
+                scm_name = "parse-tsu-snippet",
+                cpp_name = "tmu_to_tree",
+                ret_type = "tree",
+                arg_list = {
+                    "string"
+                }
+            },
             {
                 scm_name = "texmacs->stm",
                 cpp_name = "tree_to_scheme",

src/System/Misc/data_cache.cpp

@@ -135,8 +135,9 @@ do_cache_stat (string name) {
 
 bool
 do_cache_stat_fail (string name) {
-  return !ends (name, ".ts") && (starts (name, texmacs_path_string) ||
-                                 starts (name, texmacs_doc_path_string));
+  return !ends (name, ".ts") && !ends (name, ".tsu") &&
+         (starts (name, texmacs_path_string) ||
+          starts (name, texmacs_doc_path_string));
 }
 
 /******************************************************************************

src/Texmacs/Data/new_buffer.cpp

@@ -508,17 +508,33 @@ buffer_load (url name) {
 
 hashmap<string, tree> style_tree_cache ("");
 
+static tree
+style_document_to_tree (string doc_s, url name) {
+  if (ends (as_string (name), ".tsu") || starts (doc_s, "<TSU|"))
+    return tsu_document_to_tree (doc_s);
+  return texmacs_document_to_tree (doc_s);
+}
+
 tree
 load_style_tree (string package) {
   if (style_tree_cache->contains (package)) return style_tree_cache[package];
   url name= url_none ();
   url styp= "$TEXMACS_STYLE_PATH";
   if (ends (package, ".ts")) name= package;
-  else name= styp * (package * ".ts");
-  name= resolve (name);
+  else if (ends (package, ".tsu")) name= package;
+  else {
+    name= styp * (package * ".ts");
+    name= resolve (name);
+    if (is_none (name)) {
+      name= styp * (package * ".tsu");
+      name= resolve (name);
+    }
+  }
+  if (ends (package, ".ts") || ends (package, ".tsu"))
+    name= resolve (name);
   string doc_s;
   if (!load_string (name, doc_s, false)) {
-    tree doc= texmacs_document_to_tree (doc_s);
+    tree doc= style_document_to_tree (doc_s, name);
     if (is_compound (doc)) doc= extract (doc, "body");
     style_tree_cache (package)= doc;
     return doc;

src/Typeset/Env/env_exec.cpp

@@ -1216,12 +1216,22 @@ edit_env_rep::exec_use_package (tree t) {
       styp= styp | ::expand (head (base_file_name) * url_ancestor ());
     else styp= styp | head (base_file_name);
     if (ends (as_string (t[i]), ".ts")) name= url_system (as_string (t[i]));
-    else name= styp * (as_string (t[i]) * string (".ts"));
+    else if (ends (as_string (t[i]), ".tsu")) name= url_system (as_string (t[i]));
+    else {
+      name= styp * (as_string (t[i]) * string (".ts"));
+      name= resolve (name);
+      if (is_none (name))
+        name= styp * (as_string (t[i]) * string (".tsu"));
+    }
     name= resolve (name);
     // cout << as_string (t[i]) << " -> " << name << "\n";
     string doc_s;
     if (!load_string (name, doc_s, false)) {
-      tree doc= texmacs_document_to_tree (doc_s);
+      tree doc;
+      if (ends (as_string (name), ".tsu") || starts (doc_s, "<TSU|"))
+        doc= tsu_document_to_tree (doc_s);
+      else
+        doc= texmacs_document_to_tree (doc_s);
       if (is_compound (doc)) exec (filter_style (extract (doc, "body")));
     }
   }