menlo: handle large numbers in API responses

efd6 · efd6 · commit c350d286dc45 · 2025-12-01T10:29:10.000+10:30
Some numeric values from the API are outside the range of values that
are naturally marshaled as integer numbers. So use string number
unmarshaling and special-case the numbers that are needed to send the
POST requests. No change is needed in the ingest pipeline since the
numeric fields were already being converted to long.

The new test case in the pipeline tests was derived from the case above
it with the number stringified. Since no large number was present in the
system tests, one of the NAs was made to be a large number (obtained
frome the issue that this fixes).
diff --git a/packages/menlo/_dev/deploy/docker/config.yml b/packages/menlo/_dev/deploy/docker/config.yml
@@ -332,7 +332,7 @@ rules:
                       "is_iframe": "false",
                       "origin_ip": "192.18.1.4",
                       "has_password": "false",
-                      "file_size": "NA",
+                      "file_size": 3402270721,
                       "browser_and_version": "Chrome_119",
                       "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
                       "egress_ip": "192.18.1.2",
diff --git a/packages/menlo/changelog.yml b/packages/menlo/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.1"
+  changes:
+    - description: Fix handling of large numbers.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/16168
 - version: "1.7.0"
   changes:
     - description: Add parsing for proxy events returned by the Menlo Web API.
diff --git a/packages/menlo/data_stream/dlp/agent/stream/input.yml.hbs b/packages/menlo/data_stream/dlp/agent/stream/input.yml.hbs
@@ -30,36 +30,60 @@ program: |
         (string(int(now() - duration(state.initial_interval))))
     ),
     "application/json",
-    {"token": state.token, "log_type": "dlp", "pagingIdentifiers": (
-      has(state.cursor) && has(state.cursor.paging) && state.cursor.paging != null ?
-        state.cursor.paging
-      :
-        {}
-    )
+    {
+      "token": state.token,
+      "log_type": "dlp",
+      "pagingIdentifiers": (
+        has(state.?cursor.paging) && state.cursor.paging != null ?
+          state.cursor.paging.with({
+            "hashes": state.cursor.paging.hashes.transformMap(_, v, try(int(v), "fail").as(t, has(t.fail) ? v : t))
+          })
+        :
+          {}
+      )
     }.encode_json()
-  ).do_request().as(resp, bytes(resp.Body).decode_json().as(body, {
-    "events": body[0].result.events.map(e, {
-        "message": e.encode_json(),
-      }),
+  ).do_request().as(resp, resp.StatusCode == 200 ?
+    bytes(resp.Body).decode_json_string_numbers().as(body, {
+      "events": body[0].result.events.map(e, {
+          "message": e.encode_json(),
+        }),
+        "url": state.url,
+        "batch_size": state.batch_size,
+        "token": state.token,
+        "cursor": {
+          "last_timestamp": (
+            has(body[0].timestamp) && body[0].timestamp.size() > 0
+            ?
+              body[0].timestamp.parse_time(time_layout.RFC3339)
+            :
+              null
+            ),
+          "paging": (
+            has(body[0].result.pagingIdentifiers) && body[0].result.pagingIdentifiers.size() > 0 ?
+              body[0].result.pagingIdentifiers
+            : 
+              null
+          )
+        }
+      })
+  : 
+    {
+      "events": {
+        "error": {
+          "code": string(resp.StatusCode),
+          "id": string(resp.Status),
+          "message": "POST " + state.url.trim_right("/") + (
+            (size(resp.Body) != 0) ?
+              string(resp.Body)
+            :
+              string(resp.Status) + " (" + string(resp.StatusCode) + ")"
+          ),
+        },
+      },
       "url": state.url,
       "batch_size": state.batch_size,
       "token": state.token,
-      "cursor": {
-        "last_timestamp": (
-          has(body[0].timestamp) && body[0].timestamp.size() > 0
-          ?
-            body[0].timestamp.parse_time(time_layout.RFC3339)
-          :
-            null
-          ),
-        "paging": (
-          has(body[0].result.pagingIdentifiers) && body[0].result.pagingIdentifiers.size() > 0 ?
-            body[0].result.pagingIdentifiers
-          : 
-            null
-        )
-      }
-    })
+    }
   )
 tags:
 {{#if preserve_original_event}}
diff --git a/packages/menlo/data_stream/dlp/sample_event.json b/packages/menlo/data_stream/dlp/sample_event.json
@@ -1,24 +1,24 @@
 {
-    "@timestamp": "2025-06-03T13:29:06.251Z",
+    "@timestamp": "2025-11-30T23:37:30.347Z",
     "agent": {
-        "ephemeral_id": "b46aee26-1420-4f46-8e52-3bec2e7e48f6",
-        "id": "7af6091c-f1cb-4ddd-accf-48118fcd2a5a",
-        "name": "elastic-agent-94876",
+        "ephemeral_id": "9998f9dc-1878-4fdc-a74f-3906648b186c",
+        "id": "02dc436f-6c46-4084-9ba1-bf15f0b48d5e",
+        "name": "elastic-agent-90978",
         "type": "filebeat",
-        "version": "8.18.1"
+        "version": "8.19.0"
     },
     "data_stream": {
         "dataset": "menlo.dlp",
-        "namespace": "71436",
+        "namespace": "57455",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "7af6091c-f1cb-4ddd-accf-48118fcd2a5a",
+        "id": "02dc436f-6c46-4084-9ba1-bf15f0b48d5e",
         "snapshot": false,
-        "version": "8.18.1"
+        "version": "8.19.0"
     },
     "event": {
         "action": "block",
@@ -30,7 +30,7 @@
         "created": "2020-03-09T17:17:22.227Z",
         "dataset": "menlo.dlp",
         "id": "a4c2161b3f81a287ec46d3c993a33f3b97ded5fd854fa184e7f50679303111ce",
-        "ingested": "2025-06-03T13:29:09Z",
+        "ingested": "2025-11-30T23:37:33Z",
         "kind": "alert",
         "original": "{\"event\":{\"action\":\"block\",\"alerted\":\"false\",\"categories\":\"Download Sites\",\"ccl_ids\":\"CreditordebitcardnumbersGlobal\",\"ccl_match_counts\":\"1\",\"ccl_scores\":\"1\",\"domain\":\"tinynewupload.com\",\"dst_url\":\"http://tinynewupload.com/\",\"event_id\":\"a4c2161b3f81a287ec46d3c993a33f3b97ded5fd854fa184e7f50679303111ce\",\"event_time\":\"2020-03-09T17:17:22.227000\",\"file_type\":\"CSV\",\"filename\":\"more_credit_cards.csv\",\"name\":\"file_upload\",\"product\":\"MSIP\",\"protocol\":\"http\",\"request_type\":\"GET\",\"rule_id\":\"1f3ef32c-ec62-42fb-8cad-e1fee3375099\",\"rule_name\":\"Credit card block rule\",\"severity\":\"5\",\"sha256\":\"fd1aee671d92aba0f9f0a8a6d5c6b843e09c8295ced9bb85e16d97360b4d7b3a\",\"src_url\":\"http://tinynewupload.com/\",\"status\":\"dirty\",\"stream_name\":\"/safefile-input/working_file\",\"user_input\":\"false\",\"userid\":\"admin@menlosecurity.com\",\"vendor\":\"Menlo Security\",\"version\":\"2.0\"}}",
         "outcome": "success",
diff --git a/packages/menlo/data_stream/web/_dev/test/pipeline/test-web.log b/packages/menlo/data_stream/web/_dev/test/pipeline/test-web.log
@@ -1,3 +1,4 @@
 {"event":{"top_url":"http://elastic.co/","egress_country":"US","domain":"elastic.co","protocol":"http","risk_tally":"-1","is_iframe":"false","origin_ip":"192.168.1.1","has_password":"false","file_size":"NA","browser_and_version":"Chrome_119","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36","egress_ip":"192.168.4.55","sandboxActivity":"NA","event_time":"2023-11-21T13:12:37.102000","full_session_id":"nLxXe_iU-1","dst":"192.168.1.1","filename":"NA","risk_score":"low","version":"2.0","pe_rulename":"Business and Economy Category","soph_dlp_ref":"NA","numSubfiles":"0","xff_ip":"NA","product":"MSIP","origin_country":"US","vendor":"Menlo Security","rendering_mode":"ACR1","inconsistent_domain":"false","x-client-country":"US","sandboxResult":"NA","request_type":"GET","fullScanResult":"NA","tab_id":"1","pe_reason":"a77757d5-d3be-47ab-9394-cfff5887ade4","categories":"Business and Economy","severity":"5","x-client-ip":"192.168.4.3","name":"page_request","url":"http://elastic.co/","region":"us-east-1c","userid":"example_user","magicName":"NA","pe_action":"isolate","ua_type":"supported_browser","content-type":"text/html; charset=UTF-8","response_code":"308"}}
 {"event":{"top_url":"https://elastic.co/","egress_country":"US","domain":"elastic.co","protocol":"https","risk_tally":"-1","is_iframe":"false","origin_ip":"192.168.1.1","has_password":"false","file_size":"NA","browser_and_version":"Chrome_119","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36","egress_ip":"192.168.4.55","sandboxActivity":"NA","event_time":"2023-11-21T13:12:37.132000","full_session_id":"nLxXe_iU-1","dst":"192.168.1.1","filename":"NA","risk_score":"low","version":"2.0","pe_rulename":"Business and Economy Category","soph_dlp_ref":"NA","numSubfiles":"0","xff_ip":"NA","product":"MSIP","origin_country":"US","vendor":"Menlo Security","rendering_mode":"ACR1","inconsistent_domain":"false","x-client-country":"US","sandboxResult":"NA","request_type":"GET","referer":"http://elastic.co/","fullScanResult":"NA","tab_id":"1","pe_reason":"a77757d5-d3be-47ab-9394-cfff5887ade4","categories":"Business and Economy","severity":"5","x-client-ip":"192.168.4.3","name":"page_request","url":"https://elastic.co/","region":"us-east-1c","userid":"example_user","magicName":"NA","pe_action":"isolate","ua_type":"supported_browser","response_code":"301"}}
 {"event":{"top_url":"https://www.elastic.co/","egress_country":"US","domain":"www.elastic.co","protocol":"https","risk_tally":"-1","is_iframe":"false","origin_ip":"192.168.1.2","has_password":"false","file_size":"NA","browser_and_version":"Chrome_119","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36","egress_ip":"192.168.4.55","sandboxActivity":"NA","event_time":"2023-11-21T13:12:37.207000","full_session_id":"nLxXe_iU-1","dst":"192.168.1.2","filename":"NA","risk_score":"low","version":"2.0","pe_rulename":"Business and Economy Category","soph_dlp_ref":"NA","numSubfiles":"0","xff_ip":"NA","product":"MSIP","origin_country":"US","vendor":"Menlo Security","rendering_mode":"ACR1","inconsistent_domain":"false","x-client-country":"US","sandboxResult":"NA","request_type":"GET","referer":"http://elastic.co/","fullScanResult":"NA","tab_id":"1","pe_reason":"a77757d5-d3be-47ab-9394-cfff5887ade4","categories":"Business and Economy","severity":"5","x-client-ip":"192.168.4.3","name":"page_request","url":"https://www.elastic.co/","region":"us-east-1c","userid":"example_user","magicName":"NA","pe_action":"isolate","ua_type":"supported_browser","content-type":"text/html; charset=utf-8","response_code":"200"}}
+{"event":{"top_url":"https://www.elastic.co/","egress_country":"US","domain":"www.elastic.co","protocol":"https","risk_tally":"-1","is_iframe":"false","origin_ip":"192.168.1.2","has_password":"false","file_size":"3402270721","browser_and_version":"Chrome_119","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36","egress_ip":"192.168.4.55","sandboxActivity":"NA","event_time":"2023-11-21T13:12:37.207000","full_session_id":"nLxXe_iU-1","dst":"192.168.1.2","filename":"NA","risk_score":"low","version":"2.0","pe_rulename":"Business and Economy Category","soph_dlp_ref":"NA","numSubfiles":"0","xff_ip":"NA","product":"MSIP","origin_country":"US","vendor":"Menlo Security","rendering_mode":"ACR1","inconsistent_domain":"false","x-client-country":"US","sandboxResult":"NA","request_type":"GET","referer":"http://elastic.co/","fullScanResult":"NA","tab_id":"1","pe_reason":"a77757d5-d3be-47ab-9394-cfff5887ade4","categories":"Business and Economy","severity":"5","x-client-ip":"192.168.4.3","name":"page_request","url":"https://www.elastic.co/","region":"us-east-1c","userid":"example_user","magicName":"NA","pe_action":"isolate","ua_type":"supported_browser","content-type":"text/html; charset=utf-8","response_code":"200"}}
diff --git a/packages/menlo/data_stream/web/_dev/test/pipeline/test-web.log-expected.json b/packages/menlo/data_stream/web/_dev/test/pipeline/test-web.log-expected.json
@@ -376,6 +376,136 @@
                 },
                 "version": "119.0.0.0"
             }
+        },
+        {
+            "@timestamp": "2023-11-21T13:12:37.207Z",
+            "client": {
+                "geo": {
+                    "country_iso_code": "US"
+                },
+                "ip": "192.168.4.3"
+            },
+            "cloud": {
+                "region": "us-east-1c"
+            },
+            "destination": {
+                "geo": {
+                    "country_iso_code": "US"
+                },
+                "ip": "192.168.1.2"
+            },
+            "dns": {
+                "answers": {
+                    "data": [
+                        "192.168.1.2"
+                    ]
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "isolate",
+                "category": [
+                    "web",
+                    "network",
+                    "threat"
+                ],
+                "id": "nLxXe_iU-1",
+                "kind": "alert",
+                "original": "{\"event\":{\"top_url\":\"https://www.elastic.co/\",\"egress_country\":\"US\",\"domain\":\"www.elastic.co\",\"protocol\":\"https\",\"risk_tally\":\"-1\",\"is_iframe\":\"false\",\"origin_ip\":\"192.168.1.2\",\"has_password\":\"false\",\"file_size\":\"3402270721\",\"browser_and_version\":\"Chrome_119\",\"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36\",\"egress_ip\":\"192.168.4.55\",\"sandboxActivity\":\"NA\",\"event_time\":\"2023-11-21T13:12:37.207000\",\"full_session_id\":\"nLxXe_iU-1\",\"dst\":\"192.168.1.2\",\"filename\":\"NA\",\"risk_score\":\"low\",\"version\":\"2.0\",\"pe_rulename\":\"Business and Economy Category\",\"soph_dlp_ref\":\"NA\",\"numSubfiles\":\"0\",\"xff_ip\":\"NA\",\"product\":\"MSIP\",\"origin_country\":\"US\",\"vendor\":\"Menlo Security\",\"rendering_mode\":\"ACR1\",\"inconsistent_domain\":\"false\",\"x-client-country\":\"US\",\"sandboxResult\":\"NA\",\"request_type\":\"GET\",\"referer\":\"http://elastic.co/\",\"fullScanResult\":\"NA\",\"tab_id\":\"1\",\"pe_reason\":\"a77757d5-d3be-47ab-9394-cfff5887ade4\",\"categories\":\"Business and Economy\",\"severity\":\"5\",\"x-client-ip\":\"192.168.4.3\",\"name\":\"page_request\",\"url\":\"https://www.elastic.co/\",\"region\":\"us-east-1c\",\"userid\":\"example_user\",\"magicName\":\"NA\",\"pe_action\":\"isolate\",\"ua_type\":\"supported_browser\",\"content-type\":\"text/html; charset=utf-8\",\"response_code\":\"200\"}}",
+                "outcome": "failure",
+                "reason": "a77757d5-d3be-47ab-9394-cfff5887ade4",
+                "severity": 5
+            },
+            "file": {
+                "size": 3402270721
+            },
+            "http": {
+                "request": {
+                    "method": "GET",
+                    "referrer": "http://elastic.co/"
+                },
+                "response": {
+                    "status_code": 200
+                }
+            },
+            "menlo": {
+                "web": {
+                    "categories": "Business and Economy",
+                    "content_type": "text/html; charset=utf-8",
+                    "has_password": false,
+                    "is_iframe": "false",
+                    "request_type": "page_request",
+                    "risk_score": "low",
+                    "tab_id": "1",
+                    "tally": -1,
+                    "ua_type": "supported_browser"
+                }
+            },
+            "network": {
+                "protocol": "https"
+            },
+            "observer": {
+                "geo": {
+                    "country_iso_code": "US"
+                },
+                "ip": [
+                    "192.168.4.55"
+                ],
+                "product": "MSIP",
+                "vendor": "Menlo Security",
+                "version": "2.0"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.3",
+                    "192.168.1.2"
+                ],
+                "user": [
+                    "example_user"
+                ]
+            },
+            "server": {
+                "geo": {
+                    "country_iso_code": "US"
+                },
+                "ip": "192.168.1.2"
+            },
+            "source": {
+                "geo": {
+                    "country_iso_code": "US"
+                },
+                "ip": "192.168.4.3"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "url": {
+                "domain": "www.elastic.co",
+                "original": "https://www.elastic.co/",
+                "path": "/",
+                "registered_domain": "elastic.co",
+                "scheme": "https",
+                "subdomain": "www",
+                "top_level_domain": "co"
+            },
+            "user": {
+                "name": "example_user"
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Mac"
+                },
+                "name": "Chrome",
+                "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+                "os": {
+                    "full": "Mac OS X 10.15.7",
+                    "name": "Mac OS X",
+                    "version": "10.15.7"
+                },
+                "version": "119.0.0.0"
+            }
         }
     ]
 }
diff --git a/packages/menlo/data_stream/web/agent/stream/input.yml.hbs b/packages/menlo/data_stream/web/agent/stream/input.yml.hbs
@@ -30,36 +30,60 @@ program: |
         (string(int(now() - duration(state.initial_interval))))
     ),
     "application/json",
-    {"token": state.token, "log_type": "web", "pagingIdentifiers": (
-      has(state.cursor) && has(state.cursor.paging) && state.cursor.paging != null ?
-        state.cursor.paging
-      :
-        {}
-    )
+    {
+      "token": state.token,
+      "log_type": "web",
+      "pagingIdentifiers": (
+        has(state.?cursor.paging) && state.cursor.paging != null ?
+          state.cursor.paging.with({
+            "hashes": state.cursor.paging.hashes.transformMap(_, v, try(int(v), "fail").as(t, has(t.fail) ? v : t))
+          })
+        :
+          {}
+      )
     }.encode_json()
-  ).do_request().as(resp, bytes(resp.Body).decode_json().as(body, {
-    "events": body[0].result.events.map(e, {
-        "message": e.encode_json(),
-      }),
+  ).do_request().as(resp, resp.StatusCode == 200 ?
+    bytes(resp.Body).decode_json_string_numbers().as(body, {
+      "events": body[0].result.events.map(e, {
+          "message": e.encode_json(),
+        }),
+        "url": state.url,
+        "batch_size": state.batch_size,
+        "token": state.token,
+        "cursor": {
+          "last_timestamp": (
+            has(body[0].timestamp) && body[0].timestamp.size() > 0
+            ?
+              body[0].timestamp.parse_time(time_layout.RFC3339)
+            :
+              null
+            ),
+          "paging": (
+            has(body[0].result.pagingIdentifiers) && body[0].result.pagingIdentifiers.size() > 0 ?
+              body[0].result.pagingIdentifiers
+            : 
+              null
+          )
+        }
+      })
+  : 
+    {
+      "events": {
+        "error": {
+          "code": string(resp.StatusCode),
+          "id": string(resp.Status),
+          "message": "POST " + state.url.trim_right("/") + (
+            (size(resp.Body) != 0) ?
+              string(resp.Body)
+            :
+              string(resp.Status) + " (" + string(resp.StatusCode) + ")"
+          ),
+        },
+      },
       "url": state.url,
       "batch_size": state.batch_size,
       "token": state.token,
-      "cursor": {
-        "last_timestamp": (
-          has(body[0].timestamp) && body[0].timestamp.size() > 0
-          ?
-            body[0].timestamp.parse_time(time_layout.RFC3339)
-          :
-            null
-          ),
-        "paging": (
-          has(body[0].result.pagingIdentifiers) && body[0].result.pagingIdentifiers.size() > 0 ?
-            body[0].result.pagingIdentifiers
-          : 
-            null
-        )
-      }
-    })
+    }
   )
 tags:
 {{#if preserve_original_event}}
diff --git a/packages/menlo/data_stream/web/sample_event.json b/packages/menlo/data_stream/web/sample_event.json
diff --git a/packages/menlo/docs/README.md b/packages/menlo/docs/README.md
diff --git a/packages/menlo/manifest.yml b/packages/menlo/manifest.yml
