backup: stream UBI volumes, back up all of Novatek's partitions (#234)

.github/workflows/pr-build-check.yml

@@ -140,10 +140,11 @@ jobs:
           cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Release
           cmake --build build --target cYAML_test reginfo_test longse_test \
                                       sensors_test anyka_test hal_hisi_test \
-                                      report_test http_test
+                                      report_test http_test mtd_alias_test
           ./build/cYAML_test
           ./build/report_test
           ./build/http_test
+          ./build/mtd_alias_test
           ./build/reginfo_test
           ./build/longse_test
           ./build/anyka_test

CMakeLists.txt

@@ -351,6 +351,8 @@ set(IPCTOOL_SRC
     src/membw.h
     src/mtd.c
     src/mtd.h
+    src/mtd_alias.c
+    src/mtd_alias.h
     src/network.c
     src/network.h
     src/ptrace.c
@@ -506,4 +508,8 @@ if(NOT ONLY_LIBRARY)
   add_executable(http_test src/http_test.c src/http.c src/dns.c src/report.c
                            src/cjson/cJSON.c)
   target_include_directories(http_test PRIVATE include src)
+
+  # Which partitions are another name for the chip ("all" on Novatek).
+  add_executable(mtd_alias_test src/mtd_alias_test.c src/mtd_alias.c)
+  target_include_directories(mtd_alias_test PRIVATE include src)
 endif()

src/backup.c

@@ -58,36 +58,56 @@ static bool cb_mtd_backup(int i, const char *name, struct mtd_info_user *mtd,
         int nvols = enum_ubi_volumes(ubi_num, vols, MAX_UBI_VOLS);
         for (int v = 0; v < nvols; v++) {
             if (c->count == c->cap) {
+                fprintf(stderr, "  %s: more partitions than ipctool backs up\n",
+                        name);
                 c->missed++;
                 continue;
             }
-            size_t out_len = 0;
-            char *buf = read_ubi_volume(ubi_num, vols[v].vol_id,
-                                        vols[v].data_bytes, &out_len);
-            if (!buf) {
+            /* Streamed from the volume as it is sent, not read into RAM: a
+             * NAND camera's rootfs volume is tens of MB, more than such a
+             * camera has free, and malloc() failing on it was the whole
+             * backup refused as "1 partition could not be read". */
+            char devpath[64];
+            snprintf(devpath, sizeof(devpath), "/dev/ubi%d_%d", ubi_num,
+                     vols[v].vol_id);
+            int fd = open(devpath, O_RDONLY);
+            if (fd < 0) {
+                fprintf(stderr, "  %s (UBI volume %s, %s): %s\n", name,
+                        vols[v].name, devpath, strerror(errno));
                 c->missed++;
                 continue;
             }
-            c->blocks[c->count].data = buf;
-            c->blocks[c->count].len = out_len;
+            close(fd);
+            char *path = strdup(devpath);
+            if (!path) {
+                c->missed++;
+                continue;
+            }
+            c->blocks[c->count].data = NULL;
+            c->blocks[c->count].len = vols[v].data_bytes;
+            c->blocks[c->count].path = path;
             c->count++;
         }
         return true;
     }
 
     if (c->count == c->cap) {
+        fprintf(stderr, "  %s: more partitions than ipctool backs up\n", name);
         c->missed++;
         return true;
     }
     int fd;
     char *addr = open_mtdblock(i, &fd, mtd->size, 0);
     if (!addr) {
+        fprintf(stderr, "  %s (/dev/mtdblock%d): %s\n", name, i,
+                strerror(errno));
         c->missed++;
         return true;
     }
 
     c->blocks[c->count].data = addr;
     c->blocks[c->count].len = mtd->size;
+    c->blocks[c->count].path = NULL;
     c->count++;
     return true;
 }
@@ -127,6 +147,35 @@ int save_file(const char *filename, span_t blocks[MAX_MTDBLOCKS + 1],
             fwrite(&len_header, 1, sizeof(len_header), fp);
         }
 
+        if (blocks[i].path) {
+            // A UBI volume, streamed as the upload streams it.
+            static char piece[65536];
+            FILE *src = fopen(blocks[i].path, "rb");
+            size_t left = blocks[i].len;
+            const char *failed = src ? NULL : blocks[i].path;
+            while (src && left) {
+                size_t n = fread(
+                    piece, 1, left < sizeof(piece) ? left : sizeof(piece), src);
+                if (!n) {
+                    failed = blocks[i].path;
+                    break;
+                }
+                if (fwrite(piece, 1, n, fp) != n) {
+                    failed = filename;
+                    break;
+                }
+                left -= n;
+            }
+            if (src)
+                fclose(src);
+            if (failed) {
+                fprintf(stderr, "Error %s '%s', aborting\n",
+                        failed == filename ? "writing" : "reading", failed);
+                fclose(fp);
+                return 1;
+            }
+            continue;
+        }
         int nbytes = fwrite(blocks[i].data, 1, blocks[i].len, fp);
         if (nbytes == -1)
             break;
@@ -144,6 +193,7 @@ size_t backup_blocks(const char *yaml, size_t yaml_len, span_t *blocks,
                      size_t *missed) {
     blocks[0].data = yaml;
     blocks[0].len = yaml_len + 1; // end string data with \0
+    blocks[0].path = NULL;
     return map_mtdblocks(blocks + 1, MAX_MTDBLOCKS, missed) + 1;
 }
 

src/http.c

@@ -251,13 +251,46 @@ int http_post(const char *hostname, int port, const char *path, nservers_t *ns,
 
     size_t sent = 0;
     int shown = -1;
+    static char piece[65536];
     for (size_t i = 0; i < nspans; i++) {
+        int src = -1;
+        if (spans[i].path && (src = open(spans[i].path, O_RDONLY)) < 0) {
+            fprintf(stderr, "\nCould not open %s: %s\n", spans[i].path,
+                    strerror(errno));
+            close(s);
+            return ERR_SOURCE;
+        }
         /* In 64 KB pieces, so a 16 MB partition shows its progress. */
         for (size_t off = 0; off < spans[i].len;) {
             size_t chunk = spans[i].len - off;
-            if (chunk > 65536)
-                chunk = 65536;
-            if (write_all(s, spans[i].data + off, chunk)) {
+            if (chunk > sizeof(piece))
+                chunk = sizeof(piece);
+            const char *from = piece;
+            if (src < 0) {
+                from = spans[i].data + off;
+            } else {
+                size_t got = 0;
+                while (got < chunk) {
+                    ssize_t n = read(src, piece + got, chunk - got);
+                    if (n < 0 && errno == EINTR)
+                        continue;
+                    if (n <= 0) {
+                        // The length was promised in Content-Length: a
+                        // short body is a broken one, so stop here.
+                        fprintf(stderr, "\n%s ended at %zu of %zu bytes%s%s\n",
+                                spans[i].path, (size_t)(off + got),
+                                spans[i].len, n < 0 ? ": " : "",
+                                n < 0 ? strerror(errno) : "");
+                        close(src);
+                        close(s);
+                        return ERR_SOURCE;
+                    }
+                    got += (size_t)n;
+                }
+            }
+            if (write_all(s, from, chunk)) {
+                if (src >= 0)
+                    close(src);
                 // The server may have answered and closed: read what it
                 // said, so a refusal reaches the user as a reason. Or the
                 // body stopped moving, and nothing is coming back either.
@@ -278,6 +311,8 @@ int http_post(const char *hostname, int port, const char *path, nservers_t *ns,
                 }
             }
         }
+        if (src >= 0)
+            close(src);
     }
     if (shown >= 0)
         fprintf(stderr, "\n");

src/http.h

@@ -9,9 +9,13 @@
 
 #define MAX_MTDBLOCKS 20
 
+/* len bytes: at data, or -- when path is set -- read from that file while
+ * they are sent, for what is too big to hold in RAM (a UBI volume of tens of
+ * MB on a camera with less than that free). */
 typedef struct {
     const char *data;
     size_t len;
+    const char *path;
 } span_t;
 
 #define HTTP_ERR(err) ((int)err < 0 && (int)err > -100 ? (-(int)err) : 0)
@@ -24,6 +28,7 @@ typedef struct {
 #define ERR_HTTP 6
 #define ERR_MALLOC 7
 #define ERR_STALLED 8
+#define ERR_SOURCE 9
 #define ERR_BUTT 10
 
 /* The upload's timing, variables so http_test can shorten them: how long a
@@ -36,7 +41,8 @@ extern int http_late_answer_s;
 
 /* POST spans as one body of total bytes; resp gets the answer's body, status
  * its HTTP status, location its Location header ("" without one). 0, or one
- * of the ERR_ codes above: ERR_STALLED when the body stopped moving and no
+ * of the ERR_ codes above: ERR_SOURCE when a span's file could not be read
+ * whole (http_post says which), ERR_STALLED when the body stopped moving and no
  * answer came. An answer that arrives before the body is sent -- a redirect
  * or a refusal on the headers alone -- is taken without sending the body. */
 int http_post(const char *hostname, int port, const char *path, nservers_t *ns,

src/http_test.c

@@ -173,6 +173,34 @@ int main(void) {
           status);
     CHECK(reaped(cont) == 16, "after a 100 the body did not arrive whole");
 
+    // A span read from a file as it is sent -- a UBI volume too big for RAM
+    // -- arrives whole, between spans held in memory.
+    char tmpl[] = "/tmp/http_test.XXXXXX";
+    int tf = mkstemp(tmpl);
+    CHECK(tf >= 0 && write(tf, big, total) == (ssize_t)total, "temp file");
+    close(tf);
+    span_t mixed[3] = {
+        {big, 1u << 20, NULL}, {NULL, total, tmpl}, {big, 1u << 20, NULL}};
+    int file_port;
+    pid_t fil = spawn(ACCEPT, &file_port, 0);
+    err = http_post("127.0.0.1", file_port, "/", &ns,
+                    "application/octet-stream", mixed, 3, total + (2u << 20),
+                    resp, sizeof(resp), &status, loc, sizeof(loc));
+    CHECK(!err && status == 201, "file span: err %d, status %d", err, status);
+    CHECK(reaped(fil) == 18, "the file span did not arrive whole");
+
+    // A file shorter than its span promised stops the upload: the
+    // Content-Length is already sent, and a short body is a broken one.
+    span_t longer = {NULL, total + 1, tmpl};
+    int short_port;
+    pid_t sho = spawn(ACCEPT, &short_port, 0);
+    err = http_post("127.0.0.1", short_port, "/", &ns,
+                    "application/octet-stream", &longer, 1, total + 1, resp,
+                    sizeof(resp), &status, loc, sizeof(loc));
+    CHECK(err == ERR_SOURCE, "a short file: err %d", err);
+    reaped(sho);
+    unlink(tmpl);
+
     // A server that never reads: a stall, in the stall timeout plus the late
     // answer's, not the minutes it used to take.
     int freeze_port;

src/mtd.c

@@ -18,7 +18,9 @@
 #include "chipid.h"
 #include "hal/common.h"
 #include "hal/hisi/hal_hisi.h"
+#include "http.h" // MAX_MTDBLOCKS
 #include "mtd.h"
+#include "mtd_alias.h"
 #include "sha1.h"
 #include "tools.h"
 #include "uboot.h"
@@ -29,6 +31,9 @@
 
 #define MTD_NORFLASH 3
 #define MTD_NANDFLASH 4
+// A UBI volume shown as an MTD device by gluebi: not flash, a second view of
+// what its UBI partition holds.
+#define MTD_UBIVOLUME 7
 
 #define MAX_MPOINTS 10
 #define MPOINT_LEN 90
@@ -203,34 +208,6 @@ int enum_ubi_volumes(int ubi_num, ubi_vol_info_t *vols, int max_vols) {
     return count;
 }
 
-char *read_ubi_volume(int ubi_num, int vol_id, size_t data_bytes,
-                      size_t *out_len) {
-    char devpath[64];
-    snprintf(devpath, sizeof(devpath), "/dev/ubi%d_%d", ubi_num, vol_id);
-
-    int fd = open(devpath, O_RDONLY);
-    if (fd == -1)
-        return NULL;
-
-    char *buf = malloc(data_bytes);
-    if (!buf) {
-        close(fd);
-        return NULL;
-    }
-
-    size_t total = 0;
-    while (total < data_bytes) {
-        ssize_t n = read(fd, buf + total, data_bytes - total);
-        if (n <= 0)
-            break;
-        total += n;
-    }
-    close(fd);
-
-    *out_len = total;
-    return buf;
-}
-
 bool sha1_ubi_volume(int ubi_num, int vol_id, size_t data_bytes,
                      unsigned char digest[20], size_t *out_len) {
     char devpath[64];
@@ -415,7 +392,9 @@ static bool cb_mtd_info(int i, const char *name, struct mtd_info_user *mtd,
     return true;
 }
 
-#define MAX_MTD 10
+/* Novatek has twelve partitions (ipctool#234); 10 dropped the last two
+ * without a word. No more than MAX_MTDBLOCKS: restore indexes by number. */
+#define MAX_MTD MAX_MTDBLOCKS
 
 struct mtd_entry {
     int i;
@@ -439,7 +418,12 @@ void enum_mtd_info(void *ctx, cb_mtd cb) {
                 if (devfd < 0)
                     goto skip;
 
-                if (ioctl(devfd, MEMGETINFO, &mtds[n].mtd) >= 0)
+                /* gluebi's devices are skipped: listing them showed a
+                 * volume twice in the report and put it twice in a backup
+                 * -- only the volumes not mounted, since a mounted one
+                 * refuses O_RDWR here (gk7205v510 NAND, 2026-10). */
+                if (ioctl(devfd, MEMGETINFO, &mtds[n].mtd) >= 0 &&
+                    mtds[n].mtd.type != MTD_UBIVOLUME)
                     mtds[n].valid = true;
 
                 close(devfd);
@@ -475,6 +459,29 @@ void enum_mtd_info(void *ctx, cb_mtd cb) {
         mtds[1] = tmp;
     }
 
+    // A partition that is another name for others -- "all" -- is left out.
+    uint64_t offs[MAX_MTD], sizes[MAX_MTD];
+    bool has_off[MAX_MTD], alias[MAX_MTD];
+    int idx[MAX_MTD], m = 0;
+    for (int i = 0; i < n; i++) {
+        if (!mtds[i].valid)
+            continue;
+        char path[64];
+        snprintf(path, sizeof(path), "/sys/class/mtd/mtd%d/offset", mtds[i].i);
+        FILE *f = fopen(path, "r");
+        unsigned long long o = 0;
+        has_off[m] = f && fscanf(f, "%llu", &o) == 1;
+        if (f)
+            fclose(f);
+        offs[m] = o;
+        sizes[m] = mtds[i].mtd.size;
+        idx[m++] = i;
+    }
+    mtd_mark_aliases(m, offs, has_off, sizes, alias);
+    for (int k = 0; k < m; k++)
+        if (alias[k])
+            mtds[idx[k]].valid = false;
+
     for (int i = 0; i < n; i++) {
         if (mtds[i].valid && !cb(mtds[i].i, mtds[i].name, &mtds[i].mtd, ctx))
             break;

src/mtd.h

@@ -65,8 +65,6 @@ typedef struct {
 
 int find_ubi_for_mtd(int mtd_num);
 int enum_ubi_volumes(int ubi_num, ubi_vol_info_t *vols, int max_vols);
-char *read_ubi_volume(int ubi_num, int vol_id, size_t data_bytes,
-                      size_t *out_len);
 bool sha1_ubi_volume(int ubi_num, int vol_id, size_t data_bytes,
                      unsigned char digest[20], size_t *out_len);