diff --git a/.github/workflows/pr-build-check.yml b/.github/workflows/pr-build-check.yml index c833b9c3..24a0053d 100644 --- a/.github/workflows/pr-build-check.yml +++ b/.github/workflows/pr-build-check.yml @@ -140,9 +140,10 @@ jobs: cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Release cmake --build build --target cYAML_test reginfo_test longse_test \ sensors_test anyka_test hal_hisi_test \ - report_test + report_test http_test ./build/cYAML_test ./build/report_test + ./build/http_test ./build/reginfo_test ./build/longse_test ./build/anyka_test diff --git a/CMakeLists.txt b/CMakeLists.txt index 77f25c3f..901145c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -499,4 +499,11 @@ if(NOT ONLY_LIBRARY) # like the others. add_executable(report_test src/report_test.c src/report.c src/cjson/cJSON.c) target_include_directories(report_test PRIVATE include src) + + # How the upload meets a server: a redirect on the headers, an interim + # 100, a body taken whole, and a connection that stops draining -- against + # servers forked on the loopback. + add_executable(http_test src/http_test.c src/http.c src/dns.c src/report.c + src/cjson/cJSON.c) + target_include_directories(http_test PRIVATE include src) endif() diff --git a/src/http.c b/src/http.c index d77050d5..1cd02c4a 100644 --- a/src/http.c +++ b/src/http.c @@ -3,8 +3,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -122,12 +124,29 @@ static int common_connect(const char *hostname, int port, nservers_t *ns, return ret; } +/* A send that cannot place a byte for this long means the body has stopped + * moving; nothing on a working connection waits that long for buffer space. */ +int http_stall_timeout_s = 30; + // send(), not write(): a server that answers early (413, 429) and closes -// must not kill ipctool with SIGPIPE before it can print the reason. +// must not kill ipctool with SIGPIPE before it can print the reason. Each +// send waits for buffer space through poll(), so a connection that stops +// draining fails after http_stall_timeout_s with EAGAIN -- SO_SNDTIMEO would +// restart its clock whenever a few bytes squeeze in, and a frozen upload took +// minutes to give up. static int write_all(int s, const char *data, size_t len) { while (len) { - ssize_t n = send(s, data, len, MSG_NOSIGNAL); - if (n < 0 && errno == EINTR) + struct pollfd pfd = {.fd = s, .events = POLLOUT}; + int ready = poll(&pfd, 1, http_stall_timeout_s * 1000); + if (ready < 0 && errno == EINTR) + continue; + if (ready == 0) + errno = EAGAIN; + if (ready <= 0) + return -1; + ssize_t n = send(s, data, len, MSG_NOSIGNAL | MSG_DONTWAIT); + if (n < 0 && + (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)) continue; if (n <= 0) return -1; @@ -137,9 +156,47 @@ static int write_all(int s, const char *data, size_t len) { return 0; } +/* The value of header name in the head of an answer (up to its blank line), + * into out; "" when there is none. */ +static void find_header(const char *head, const char *name, char *out, + size_t cap) { + size_t nl = strlen(name); + *out = '\0'; + for (const char *line = strstr(head, "\r\n"); line && line[2] != '\r'; + line = strstr(line + 2, "\r\n")) { + const char *h = line + 2; + if (strncasecmp(h, name, nl) || h[nl] != ':') + continue; + h += nl + 1; + while (*h == ' ' || *h == '\t') + h++; + size_t n = strcspn(h, "\r\n"); + if (n >= cap) + n = cap - 1; + memcpy(out, h, n); + out[n] = '\0'; + while (n && isspace((unsigned char)out[n - 1])) + out[--n] = '\0'; + return; + } +} + +/* How long the server has to answer the headers alone, before the body + * goes. A server that redirects or refuses says so at once, and a body it + * will not read is never sent: through a filter that freezes a connection + * after its first few kilobytes, the answer to the headers still gets back, + * and the body would not. A server that wants the body says nothing, and + * this is what it costs. */ +int http_early_answer_ms = 2000; +/* After a stall or a refusal mid-body, how long an answer may take. */ +int http_late_answer_s = 10; + int http_post(const char *hostname, int port, const char *path, nservers_t *ns, const char *content_type, const span_t *spans, size_t nspans, - size_t total, char *resp, size_t cap, int *status) { + size_t total, char *resp, size_t cap, int *status, char *location, + size_t loccap) { + if (loccap) + *location = '\0'; int s; int rc = common_connect(hostname, port, ns, &s); /* common_connect() answers ERR_GENERAL when it is connected. */ @@ -148,7 +205,9 @@ int http_post(const char *hostname, int port, const char *path, nservers_t *ns, struct timeval tv = {.tv_sec = 120}; setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + // The whole body is in: the server stores it, then answers. setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + bool stalled = false; char host[300]; if (port == 80) @@ -171,6 +230,25 @@ int http_post(const char *hostname, int port, const char *path, nservers_t *ns, return ERR_SEND; } + struct pollfd early = {.fd = s, .events = POLLIN}; + int ready; + do + ready = poll(&early, 1, http_early_answer_ms); + while (ready < 0 && errno == EINTR); + if (ready > 0) { + // An interim 1xx asks for the body (no server should send one to + // HTTP/1.0, but it costs nothing to allow): send it, and the 1xx is + // skipped with the final answer below. Anything else is the answer. + char peek[64]; + ssize_t n = recv(s, peek, sizeof(peek) - 1, MSG_PEEK); + if (n <= 0) + goto answer; + peek[n] = '\0'; + int code = get_http_respcode(peek); + if (code < 100 || code > 199) + goto answer; + } + size_t sent = 0; int shown = -1; for (size_t i = 0; i < nspans; i++) { @@ -181,9 +259,13 @@ int http_post(const char *hostname, int port, const char *path, nservers_t *ns, chunk = 65536; if (write_all(s, spans[i].data + off, chunk)) { // The server may have answered and closed: read what it - // said, so a refusal reaches the user as a reason. + // said, so a refusal reaches the user as a reason. Or the + // body stopped moving, and nothing is coming back either. + stalled = errno == EAGAIN; if (shown >= 0) fprintf(stderr, "\n"); + tv.tv_sec = http_late_answer_s; + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); goto answer; } off += chunk; @@ -216,8 +298,15 @@ answer:; resp[got] = '\0'; *status = get_http_respcode(resp); char *body = strstr(resp, "\r\n\r\n"); + while (*status >= 100 && *status <= 199 && body) { + memmove(resp, body + 4, strlen(body + 4) + 1); + *status = get_http_respcode(resp); + body = strstr(resp, "\r\n\r\n"); + } if (*status < 0 || !body) - return got ? ERR_HTTP : ERR_SEND; + return got ? ERR_HTTP : stalled ? ERR_STALLED : ERR_SEND; + if (loccap) + find_header(resp, "Location", location, loccap); memmove(resp, body + 4, strlen(body + 4) + 1); return 0; } diff --git a/src/http.h b/src/http.h index 830dbfaf..9f2db2c8 100644 --- a/src/http.h +++ b/src/http.h @@ -23,12 +23,25 @@ typedef struct { #define ERR_SEND 5 #define ERR_HTTP 6 #define ERR_MALLOC 7 +#define ERR_STALLED 8 #define ERR_BUTT 10 +/* The upload's timing, variables so http_test can shorten them: how long a + * server has to answer the headers alone, how long a send may wait for + * buffer space before the body counts as stalled, and how long an answer may + * take after a stall or a refusal mid-body. */ +extern int http_early_answer_ms; +extern int http_stall_timeout_s; +extern int http_late_answer_s; + /* POST spans as one body of total bytes; resp gets the answer's body, status - * its HTTP status. 0, or one of the ERR_ codes above. */ + * its HTTP status, location its Location header ("" without one). 0, or one + * of the ERR_ codes above: ERR_STALLED when the body stopped moving and no + * answer came. An answer that arrives before the body is sent -- a redirect + * or a refusal on the headers alone -- is taken without sending the body. */ int http_post(const char *hostname, int port, const char *path, nservers_t *ns, const char *content_type, const span_t *spans, size_t nspans, - size_t total, char *resp, size_t cap, int *status); + size_t total, char *resp, size_t cap, int *status, char *location, + size_t loccap); #endif /* HTTP_H */ diff --git a/src/http_test.c b/src/http_test.c new file mode 100644 index 00000000..20f8df3e --- /dev/null +++ b/src/http_test.c @@ -0,0 +1,195 @@ +/* http_test: http_post() against servers forked on the loopback, the way + * `ipctool upload` meets them -- one that redirects on the headers, one that + * takes the body, one that sends an interim 100 first, and one that never + * reads, as a filter that freezes the connection does (#232). The timeouts + * are shortened through the variables http.h exposes for this. */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "http.h" +#include "report.h" + +static int failed; +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + failed = 1; \ + } \ + } while (0) + +enum mode { REDIRECT, ACCEPT, CONTINUE, FREEZE }; + +/* The server's half, in a child: one connection, then exit. Its exit code + * is how many body bytes it read, in MB rounded down, or 99 when it read + * any body at all where it should not have. */ +static void serve(int ls, enum mode mode, int peer_port) { + int c = accept(ls, NULL, NULL); + if (c < 0) + _exit(98); + char buf[65536]; + size_t got = 0; + char *end = NULL; + while (!end) { + ssize_t n = recv(c, buf + got, sizeof(buf) - 1 - got, 0); + if (n <= 0) + _exit(97); + got += (size_t)n; + buf[got] = '\0'; + end = strstr(buf, "\r\n\r\n"); + } + size_t body = got - (size_t)(end + 4 - buf); + if (mode == FREEZE) { + pause(); // killed by the parent + _exit(0); + } + if (mode == REDIRECT) { + char a[256]; + int l = snprintf(a, sizeof(a), + "HTTP/1.1 307 Temporary Redirect\r\n" + "Location: http://127.0.0.1:%d/next\r\n" + "Content-Length: 0\r\nConnection: close\r\n\r\n", + peer_port); + send(c, a, l, 0); + usleep(200000); + // Whatever arrived after the headers is body this server never + // asked for. + ssize_t n = recv(c, buf, sizeof(buf), MSG_DONTWAIT); + _exit(body || n > 0 ? 99 : 0); + } + if (mode == CONTINUE) { + const char *cont = "HTTP/1.1 100 Continue\r\n\r\n"; + send(c, cont, strlen(cont), 0); + } + // http_post() spells it so; strcasestr() is a GNU extension. + const char *cl = strstr(buf, "Content-Length:"); + size_t want = cl ? strtoul(cl + 15, NULL, 10) : 0; + while (body < want) { + ssize_t n = recv(c, buf, sizeof(buf), 0); + if (n <= 0) + break; + body += (size_t)n; + } + const char *ok = + "HTTP/1.1 201 Created\r\nContent-Type: application/json\r\n" + "Connection: close\r\n\r\n{\"id\":\"r-test1234\"}"; + send(c, ok, strlen(ok), 0); + close(c); + _exit((int)(body >> 20)); +} + +static pid_t spawn(enum mode mode, int *port, int peer_port) { + int ls = socket(AF_INET, SOCK_STREAM, 0); + if (mode == FREEZE) { + // Small, so the client's sends stop finding room soon. + int small = 4096; + setsockopt(ls, SOL_SOCKET, SO_RCVBUF, &small, sizeof(small)); + } + struct sockaddr_in a = {.sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK)}; + socklen_t al = sizeof(a); + if (bind(ls, (struct sockaddr *)&a, sizeof(a)) || listen(ls, 1) || + getsockname(ls, (struct sockaddr *)&a, &al)) { + perror("listen"); + exit(2); + } + *port = ntohs(a.sin_port); + pid_t pid = fork(); + if (pid == 0) + serve(ls, mode, peer_port); + close(ls); + return pid; +} + +static int reaped(pid_t pid) { + int st; + waitpid(pid, &st, 0); + return WIFEXITED(st) ? WEXITSTATUS(st) : -1; +} + +static double since(const struct timespec *t0) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (t.tv_sec - t0->tv_sec) + (t.tv_nsec - t0->tv_nsec) / 1e9; +} + +int main(void) { + signal(SIGPIPE, SIG_IGN); + http_early_answer_ms = 300; + http_stall_timeout_s = 1; + http_late_answer_s = 1; + + size_t total = 16u << 20; // a 16 MB flash + char *big = malloc(total); + memset(big, 0x5a, total); + span_t span = {big, total}; + nservers_t ns = {.len = 0}; + static char resp[4096], loc[256]; + int status, err; + + // A redirect on the headers: no body goes to that server, and the + // Location names where it should. + int accept_port, redirect_port; + pid_t acc = spawn(ACCEPT, &accept_port, 0); + pid_t red = spawn(REDIRECT, &redirect_port, accept_port); + err = http_post("127.0.0.1", redirect_port, "/api/v1/reports", &ns, + "application/octet-stream", &span, 1, total, resp, + sizeof(resp), &status, loc, sizeof(loc)); + CHECK(!err && status == 307, "redirect: err %d, status %d", err, status); + CHECK(reaped(red) == 0, "the redirecting server was sent the body"); + char host[64], path[64]; + int port; + CHECK(report_redirect(loc, host, sizeof(host), &port, path, sizeof(path)) && + port == accept_port && !strcmp(path, "/next"), + "Location %s", loc); + + // Followed: the whole body arrives, and the answer is read. + err = http_post(host, port, path, &ns, "application/octet-stream", &span, 1, + total, resp, sizeof(resp), &status, loc, sizeof(loc)); + CHECK(!err && status == 201 && strstr(resp, "r-test1234"), + "accept: err %d, status %d, %s", err, status, resp); + CHECK(reaped(acc) == 16, "the body did not arrive whole"); + + // An interim 100 is not the answer: the body still goes, and the final + // 201 is what comes back. + int cont_port; + pid_t cont = spawn(CONTINUE, &cont_port, 0); + err = http_post("127.0.0.1", cont_port, "/", &ns, + "application/octet-stream", &span, 1, total, resp, + sizeof(resp), &status, loc, sizeof(loc)); + CHECK(!err && status == 201, "100 Continue: err %d, status %d", err, + status); + CHECK(reaped(cont) == 16, "after a 100 the body did not arrive whole"); + + // A server that never reads: a stall, in the stall timeout plus the late + // answer's, not the minutes it used to take. + int freeze_port; + pid_t frz = spawn(FREEZE, &freeze_port, 0); + struct timespec t0; + clock_gettime(CLOCK_MONOTONIC, &t0); + err = http_post("127.0.0.1", freeze_port, "/", &ns, + "application/octet-stream", &span, 1, total, resp, + sizeof(resp), &status, loc, sizeof(loc)); + double took = since(&t0); + kill(frz, SIGKILL); + reaped(frz); + CHECK(err == ERR_STALLED, "freeze: err %d", err); + CHECK(took < 10, "freeze took %.1f s", took); + + free(big); + if (!failed) + printf("http_test: ok\n"); + return failed; +} diff --git a/src/report.c b/src/report.c index d62d5c09..b04c0930 100644 --- a/src/report.c +++ b/src/report.c @@ -1,5 +1,7 @@ #include +#include #include +#include #include "cjson/cJSON.h" #include "report.h" @@ -94,8 +96,7 @@ void report_parse_answer(int status, const char *body, report_answer_t *a) { cJSON *root = body ? cJSON_Parse(body) : NULL; if (!root) { if (status / 100 != 2) - snprintf(a->error, sizeof(a->error), - "openipc.org answered %d without an explanation", status); + snprintf(a->error, sizeof(a->error), "no explanation came with it"); return; } copy_string(root, "id", a->id, sizeof(a->id)); @@ -116,3 +117,39 @@ void report_parse_answer(int status, const char *body, report_answer_t *a) { } cJSON_Delete(root); } + +bool report_redirect(const char *location, char *host, size_t hostcap, + int *port, char *path, size_t pathcap) { + static const char scheme[] = "http://"; + if (strncasecmp(location, scheme, sizeof(scheme) - 1)) + return false; + const char *h = location + sizeof(scheme) - 1; + const char *p = strpbrk(h, "/?#"); + if (!p) + p = h + strlen(h); + size_t hl = (size_t)(p - h); + const char *colon = memchr(h, ':', hl); + size_t namelen = colon ? (size_t)(colon - h) : hl; + if (!namelen || namelen >= hostcap || memchr(h, '@', hl)) + return false; + int prt = 80; + if (colon) { + char *end; + long v = strtol(colon + 1, &end, 10); + if (end != p || v <= 0 || v > 65535) + return false; + prt = (int)v; + } + // The fragment is the client's alone; a query with no path before it + // still asks for "/". + size_t pl = strcspn(p, "#"); + const char *lead = *p == '/' ? "" : "/"; + if (strlen(lead) + pl >= pathcap || memchr(p, ' ', pl) || + memchr(p, '\r', pl) || memchr(p, '\n', pl)) + return false; + memcpy(host, h, namelen); + host[namelen] = '\0'; + *port = prt; + snprintf(path, pathcap, "%s%.*s", lead, (int)pl, p); + return true; +} diff --git a/src/report.h b/src/report.h index a3d0e64e..2607b88e 100644 --- a/src/report.h +++ b/src/report.h @@ -53,6 +53,13 @@ typedef struct { void report_parse_answer(int status, const char *body, report_answer_t *a); +/* Where a redirect sends the report: location must be plain http://, which + * is all stock firmware speaks, with a host, an optional port and a path. + * false for anything else -- https://, a relative address, credentials -- + * and the report is not followed there. */ +bool report_redirect(const char *location, char *host, size_t hostcap, + int *port, char *path, size_t pathcap); + /* ipctool's report as the no-argument run prints it; malloc'ed (main.c). */ char *build_report_yaml(void); diff --git a/src/report_cmd.c b/src/report_cmd.c index 3dac8fdb..a05f88d4 100644 --- a/src/report_cmd.c +++ b/src/report_cmd.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -269,21 +270,65 @@ int report_cmd(int argc, char **argv) { snprintf(ctype, sizeof(ctype), "multipart/form-data; boundary=%s", boundary); static char resp[16384]; - int status = 0; - int err = http_post(hostname, port, REPORTS_PATH, &ns, ctype, body->spans, - body->nspans, body->total, resp, sizeof(resp), &status); + char path[256] = REPORTS_PATH, location[512]; + int status = 0, err; + /* A server may send the report elsewhere with a 307 or 308 -- openipc.org + * does, for a network that cannot reach it -- and says so before the body + * is sent (http_post), so following costs nothing but the hop. + * + * To another host only from openipc.org itself: the owner agreed to + * send this report, maybe with the whole flash in it, to the host named + * above, and where openipc.org forwards its own reports is part of that. + * Any other server may move it only within its own name. */ + for (int hops = 0;; hops++) { + err = http_post(hostname, port, path, &ns, ctype, body->spans, + body->nspans, body->total, resp, sizeof(resp), &status, + location, sizeof(location)); + if (err || (status != 307 && status != 308) || hops == 3) + break; + char next[256], nextpath[256]; + int nport; + if (!report_redirect(location, next, sizeof(next), &nport, nextpath, + sizeof(nextpath))) + break; + if (strcasecmp(next, hostname) && strcasecmp(hostname, DEFAULT_HOST)) { + fprintf(stderr, + "%s sends the report to %s, another host. It goes only " + "where it was asked to: --host %s sends it there.\n", + hostname, location, next); + *location = '\0'; + break; + } + snprintf(path, sizeof(path), "%s", nextpath); + fprintf(stderr, "%s sends the report to %s.\n", hostname, location); + snprintf(hostname, sizeof(hostname), "%s", next); + port = nport; + } free(body); free(yaml); + if (err == ERR_STALLED) { + fprintf(stderr, + "The upload to %s stopped moving and no answer came. Nothing " + "was stored. Something between this camera and %s holds the " + "connection; --host can send the report another way.\n", + hostname, hostname); + return EXIT_FAILURE; + } if (err) { fprintf(stderr, "Could not reach %s (error %d). Nothing was stored.\n", - host, err); + hostname, err); return EXIT_FAILURE; } report_answer_t a; report_parse_answer(status, resp, &a); if (status != 201 || !*a.id) { - fprintf(stderr, "%s did not take the report (%d): %s\n", host, status, - *a.error ? a.error : "no reason given"); + if ((status == 307 || status == 308) && *location) + fprintf(stderr, + "%s sends the report to %s, which ipctool cannot " + "follow.\n", + hostname, location); + fprintf(stderr, "%s did not take the report (%d): %s\n", hostname, + status, *a.error ? a.error : "no reason given"); return EXIT_FAILURE; } printf("\nReceived as %s. Its state: %s\n", a.id, a.receipt_url); diff --git a/src/report_test.c b/src/report_test.c index 0fde5078..ef0d8713 100644 --- a/src/report_test.c +++ b/src/report_test.c @@ -116,6 +116,52 @@ int main(int argc, char **argv) { CHECK(!strcmp(a.error, "10 reports a day") && !*a.id, "refusal"); report_parse_answer(502, "bad gateway", &a); CHECK(*a.error, "a non-JSON failure has no explanation"); + // The answer may be from any --host, or a proxy in front of one: it must + // not be put in openipc.org's mouth (#232, "openipc.ru did not take the + // report (502): openipc.org answered 502"). + CHECK(!strstr(a.error, "openipc.org"), "names a host: %s", a.error); + + char host[64], path[64]; + int port = 0; + CHECK(report_redirect("http://mirror.example/api/v1/reports", host, + sizeof(host), &port, path, sizeof(path)) && + !strcmp(host, "mirror.example") && port == 80 && + !strcmp(path, "/api/v1/reports"), + "plain redirect: %s %d %s", host, port, path); + CHECK(report_redirect("HTTP://bench:8080/r", host, sizeof(host), &port, + path, sizeof(path)) && + !strcmp(host, "bench") && port == 8080 && !strcmp(path, "/r"), + "with a port: %s %d %s", host, port, path); + CHECK(report_redirect("http://bare", host, sizeof(host), &port, path, + sizeof(path)) && + !strcmp(path, "/"), + "no path"); + CHECK(report_redirect("http://m.example?t=1", host, sizeof(host), &port, + path, sizeof(path)) && + !strcmp(host, "m.example") && !strcmp(path, "/?t=1"), + "a query with no path: %s %s", host, path); + CHECK(report_redirect("http://m.example:81/r?a=b#frag", host, sizeof(host), + &port, path, sizeof(path)) && + port == 81 && !strcmp(path, "/r?a=b"), + "the fragment stays behind: %d %s", port, path); + CHECK(!report_redirect("https://openipc.org/api/v1/reports", host, + sizeof(host), &port, path, sizeof(path)), + "https cannot be followed by stock firmware"); + CHECK(!report_redirect("/api/v1/reports", host, sizeof(host), &port, path, + sizeof(path)), + "relative"); + CHECK(!report_redirect("http://u:p@evil/x", host, sizeof(host), &port, path, + sizeof(path)), + "credentials"); + CHECK(!report_redirect("http://h:0/x", host, sizeof(host), &port, path, + sizeof(path)) && + !report_redirect("http://h:99999/x", host, sizeof(host), &port, + path, sizeof(path)) && + !report_redirect("http://h:8a/x", host, sizeof(host), &port, path, + sizeof(path)) && + !report_redirect("http:///x", host, sizeof(host), &port, path, + sizeof(path)), + "bad port or host"); if (!failed) printf("report_test: ok\n");