Segmentation fault in __socket_sendmsg at ../lib/uring.c:52

[ccppprogrammer]

rtpengine version the issue has been seen with

26.0.1.9+0~mr26.0.1.9

Used distribution and its version

Debian GNU/Linux 13 (trixie)

Linux kernel version used

6.12.88+deb13-cloud-amd64

CPU architecture issue was seen on (see uname -m)

x86_64

Expected behaviour you didn't see

No response

Unexpected behaviour you saw

No response

Steps to reproduce the problem

No response

Additional program output to the terminal or logs illustrating the issue

Anything else?

journalctl:

May 26 15:46:15 ip-10-0-0-27 kernel: show_signal_msg: 99 callbacks suppressed
May 26 15:46:15 ip-10-0-0-27 kernel: ICE[900]: segfault at c8 ip 000055fe380581e1 sp 00007f375af9b750 error 4 in rtpengine[1481e1,55fe37f41000+131000] likely on CPU 4 (core 0, socket 0)
May 26 15:46:15 ip-10-0-0-27 kernel: Code: ff 25 f3 f2 07 00 0f 1f 00 f3 0f 1e fa 64 ff 24 25 e0 fb ff ff 0f 1f 40 00 f3 0f 1e fa 55 53 4c 89 c3 48 83 ec 08 48 8b 47 08 <ff> 90 c8 00 00 00 48 89 df 31 d2 31 f6 48 89 c5 ff 13 48 83 c4 08
May 26 15:46:16 ip-10-0-0-27 systemd-coredump[21696]: Process 831 (rtpengine) of user 106 terminated abnormally with signal 11/SEGV, processing...

coredumpctl debug 831

Core was generated by `/usr/bin/rtpengine -f -E --no-log-timestamps --pidfile /run/rtpengine/rtpengine'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x000055fe380581e1 in __socket_sendmsg (s=0x7f3664f01cf8, m=0x7f375af9b7b0, e=0x7f372656ef80, ss=0x7f375af9ba48, r=0x7f375af9b790) at ../lib/uring.c:52

warning: 52     ../lib/uring.c: No such file or directory
[Current thread is 1 (Thread 0x7f375af9d6c0 (LWP 900))]
(gdb) bt
#0  0x000055fe380581e1 in __socket_sendmsg (s=0x7f3664f01cf8, m=0x7f375af9b7b0, e=0x7f372656ef80, ss=0x7f375af9ba48, r=0x7f375af9b790) at ../lib/uring.c:52
#1  0x000055fe37f95386 in stun_binding_request (dst=0x7f372656ef80, transaction=<optimized out>, pwd=0x7f366ea26e68, ufrags=0x7f366ea26e38, controlling=1, tiebreaker=<optimized out>, priority=1862270975, sock=0x7f3664f01cf8, to_use=1)
    at ./daemon/stun.c:713
#2  0x000055fe37fd0883 in __do_ice_check (pair=<optimized out>) at ./daemon/ice.c:797
#3  0x000055fe37fd0f25 in __do_ice_checks (ag=0x7f366ea26cf0) at ./daemon/ice.c:958
#4  0x000055fe37fd27f5 in ice_agents_timer_run (ptr=0x7f366ea26cf0) at ./daemon/ice.c:1519
#5  0x000055fe38013098 in timerthread_run (tt=0x55fe6432c670) at ./daemon/timerthread.c:111
#6  0x000055fe37f5651e in thread_detach_func (dt=<optimized out>) at ./daemon/helpers.c:257
#7  0x00007f3765588b7b in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:448
#8  0x00007f3765606630 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:100

[rfuchs]

Post the gdb output of

p *s
p *m
p *e

please

[ccppprogrammer]

(gdb) p *s
$1 = {fd = 0, family = 0x0, local = {address = {family = 0x35, {ipv4 = {s_addr = 3406898065}, ipv6 = {__in6_u = {__u6_addr8 = "\221#\021\3130\177\000\000\000\000\000\000\000\000\000", __u6_addr16 = {9105, 51985, 32560, 0, 0, 0, 0, 0},
            __u6_addr32 = {3406898065, 32560, 0, 0}}}}}, port = 0}, remote = {address = {family = 0x6e6f6870656c6574, {ipv4 = {s_addr = 1986342245}, ipv6 = {__in6_u = {__u6_addr8 = "e-event\000E\000\000\000\000\000\000",
            __u6_addr16 = {11621, 30309, 28261, 116, 69, 0, 0, 0}, __u6_addr32 = {1986342245, 7630437, 69, 0}}}}}, port = 1697849300}}
(gdb) p *m
$2 = {msg_name = 0x0, msg_namelen = 0, msg_iov = 0x7f375af9b7e8, msg_iovlen = 10, msg_control = 0x0, msg_controllen = 0, msg_flags = 0}
(gdb) p *e
$3 = {address = {family = 0x55fe380cebe0 <__socket_families.lto_priv.0>, {ipv4 = {s_addr = 3513627296}, ipv6 = {__in6_u = {__u6_addr8 = "\240\262m\321", '\000' <repeats 11 times>, __u6_addr16 = {45728, 53613, 0, 0, 0, 0, 0, 0},
          __u6_addr32 = {3513627296, 0, 0, 0}}}}}, port = 60933}

[rfuchs]

A few more please

p *(stream_fd *) (((char *) s) - (long) &(((stream_fd *) 0)->socket))
frame 2
p *sfd
frame 3
p *ag
p *ag->call
p *ag->media
p ag->media->streams

[ccppprogrammer]

(gdb) p *(stream_fd *) (((char *) s) - (long) &(((stream_fd *) 0)->socket))
$5 = {obj = {ref = 995539792, clear_func = 0x0, free_func = 0x0}, lock = {__data = {__lock = 37, __count = 0, __owner = 811068032, __nusers = 32567, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x25}},
    __size = "%\000\000\000\000\000\000\000\200\352W07\177", '\000' <repeats 18 times>, "%\000\000\000\000\000\000", __align = 37}, unique_id = 8, {socket = {fd = 0, family = 0x0, local = {address = {family = 0x35, {ipv4 = {
              s_addr = 3406898065}, ipv6 = {__in6_u = {__u6_addr8 = "\221#\021\3130\177\000\000\000\000\000\000\000\000\000", __u6_addr16 = {9105, 51985, 32560, 0, 0, 0, 0, 0}, __u6_addr32 = {3406898065, 32560, 0, 0}}}}}, port = 0},
      remote = {address = {family = 0x6e6f6870656c6574, {ipv4 = {s_addr = 1986342245}, ipv6 = {__in6_u = {__u6_addr8 = "e-event\000E\000\000\000\000\000\000", __u6_addr16 = {11621, 30309, 28261, 116, 69, 0, 0, 0}, __u6_addr32 = {
                  1986342245, 7630437, 69, 0}}}}}, port = 1697849300}}, spl = {socket = {fd = 0, family = 0x0, local = {address = {family = 0x35, {ipv4 = {s_addr = 3406898065}, ipv6 = {__in6_u = {
                  __u6_addr8 = "\221#\021\3130\177\000\000\000\000\000\000\000\000\000", __u6_addr16 = {9105, 51985, 32560, 0, 0, 0, 0, 0}, __u6_addr32 = {3406898065, 32560, 0, 0}}}}}, port = 0}, remote = {address = {
            family = 0x6e6f6870656c6574, {ipv4 = {s_addr = 1986342245}, ipv6 = {__in6_u = {__u6_addr8 = "e-event\000E\000\000\000\000\000\000", __u6_addr16 = {11621, 30309, 28261, 116, 69, 0, 0, 0}, __u6_addr32 = {1986342245, 7630437,
                    69, 0}}}}}, port = 1697849300}}, links = {q = {head = 0x7e696a263ec77e69, tail = 0xedc03ec7, length = 980591700}, {head = 0x7e696a263ec77e69, tail = 0xedc03ec7, length = 980591700}, __t = 0x7e696a263ec77e69,
        __ct = 0x7e696a263ec77e69}, pp = 0x6215f8887e696a26, link = {prev = 0x9f06000000fe7, next = 0x0}}}, local_intf = 0x25, call = 0x7f373bc780a0, stream = 0x7f3730522a00, crypto = {params = {crypto_suite = 0x0,
      master_key = "5\000\000\000\000\000\000\000\270\035\360d6\177\000\000\017", '\000' <repeats 14 times>, master_salt = "telephone-even", mki = 0x35 <error: Cannot access memory at address 0x35>, mki_len = 3695917647,
      session_params = {unencrypted_srtcp = 0, unencrypted_srtp = 1, unauthenticated_srtp = 1}}, session_key = "xP\361\207\235\a;\356\335d\033i\000\000\000\000\017\f\003\034", '\000' <repeats 11 times>,
    session_salt = "u\000\000\000\000\000\000\000\b\000\000\000\000", session_auth_key = "\000\000\a\000\000\000\a\000\000\000\001\000\000\000\001\000\000\000\003", session_key_ctx = {0x7f373b1cf7f0, 0x7f3667462630},
    have_session_key = 0}, dtls = {ssl_ctx = 0x55fe3803a290 <str_hash>, ssl = 0x55fe3803a490 <str_equal>, r_bio = 0x100000001, w_bio = 0x0, ps = 0x0, fsin = {address = {family = 0x70, {ipv4 = {s_addr = 53}, ipv6 = {__in6_u = {
              __u6_addr8 = "5\000\000\000\000\000\000\000\221\255\023\3130\177\000", __u6_addr16 = {53, 0, 0, 0, 44433, 51987, 32560, 0}, __u6_addr32 = {53, 0, 3407064465, 32560}}}}}, port = 0}, sfd = 0x6500000008,
    tls_id = '\000' <repeats 15 times>, init = 1, active = 0, connected = 1}, error_strikes = 947250224, active_read_events = 32567, poller = 0x7f373875e430, users = 1, kernelized = false, confirmed = false}
(gdb) frame 2
#2  0x000055fe37fd0883 in __do_ice_check (pair=<optimized out>) at ./daemon/ice.c:797
warning: 797    ./daemon/ice.c: No such file or directory
(gdb) p *sfd
value has been optimized out
(gdb) frame 3
#3  0x000055fe37fd0f25 in __do_ice_checks (ag=0x7f366ea26cf0) at ./daemon/ice.c:958
958     in ./daemon/ice.c
(gdb) p *ag
$6 = {tt_obj = {obj = {ref = 3, clear_func = 0x55fe37fcdab0 <__ice_agent_free>, free_func = 0x7f37692da820 <g_free>}, tt = 0x55fe380d3aa0 <ice_agents_timer_thread.lto_priv>, thread = 0x55fe6432c670, next_check = 1779810375790266,
    last_run = 1779810375590266}, call = 0x7f3669e2a410, media = 0x7f37227969d8, logical_intf = 0x55fe643232c0, desired_family = 0x55fe380cebe0 <__socket_families.lto_priv.0>, last_activity = {a = 1779810375590266}, lock = {__data = {
      __lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, remote_candidates = {q = {
      head = 0x7f36195797a0, tail = 0x7f3651133eb0, length = 2}, {head = 0x7f36195797a0, tail = 0x7f3651133eb0, length = 2}, __t = 0x7f36195797a0, __ct = 0x7f36195797a0}, candidate_pairs = {q = {head = 0x7f3665f15430,
      tail = 0x7f366b2b1620, length = 2}, {head = 0x7f3665f15430, tail = 0x7f366b2b1620, length = 2}, __t = 0x7f3665f15430, __ct = 0x7f3665f15430}, triggered = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0,
      length = 0}, __t = 0x0, __ct = 0x0}, candidate_hash = {ht = 0x7f36698a2bc0, __key = 0x7f36698a2bc0, __ckey = 0x7f36698a2bc0, __value = 0x7f36698a2bc0, __iter = 0x7f36698a2bc0}, cand_prio_hash = {ht = 0x7f366ad960f0,
    __key = 0x7f366ad960f0, __ckey = 0x7f366ad960f0, __value = 0x7f366ad960f0, __iter = 0x7f366ad960f0}, pair_hash = {ht = 0x7f366a0da490, __key = 0x7f366a0da490, __ckey = 0x7f366a0da490, __value = 0x7f366a0da490,
    __iter = 0x7f366a0da490}, transaction_hash = {ht = 0x7f366b4b4260, __key = 0x7f366b4b4260, __ckey = 0x7f366b4b4260, __value = 0x7f366b4b4260, __iter = 0x7f366b4b4260}, foundation_hash = {ht = 0x7f3650db8f50,
    __key = 0x7f3650db8f50, __ckey = 0x7f3650db8f50, __value = 0x7f3650db8f50, __iter = 0x7f3650db8f50}, all_pairs = 0x7f36696f60f0, all_pairs_list = {q = {head = 0x7f373a8d6910, tail = 0x7f36682bbfc0, length = 2}, {
      head = 0x7f373a8d6910, tail = 0x7f36682bbfc0, length = 2}, __t = 0x7f373a8d6910, __ct = 0x7f373a8d6910}, nominated_pairs = 0x7f3722e18d90, succeeded_pairs = 0x7f36502d7af0, valid_pairs = 0x7f3722d094a0, active_components = 1,
  start_nominating = 0, recalc_needed = 0, ufrag = {{
      s = 0x7f36691d14e6 "Lb+z\r\na=ice-pwd:G/ZWx2+xpQVfw8sYX+OCzdhh\r\na=ice-options:trickle\r\na=fingerprint:sha-256 42:D8:19:CE:EA:1C:11:D1:CA:33:85:9E:11:DA:1F:EA:E5:5B:D6:F4:A9:46:96:F4:E7:C3:CE:ED:D7:F0:44:68\r\na=setup:active\r\n"..., len = 4, arena = 0x7f3669e2a428}, {s = 0x7f372039e920 "jIKSSAWF", len = 8, arena = 0x7f3669e2a428}}, pwd = {{
      s = 0x7f36691d14f6 "G/ZWx2+xpQVfw8sYX+OCzdhh\r\na=ice-options:trickle\r\na=fingerprint:sha-256 42:D8:19:CE:EA:1C:11:D1:CA:33:85:9E:11:DA:1F:EA:E5:5B:D6:F4:A9:46:96:F4:E7:C3:CE:ED:D7:F0:44:68\r\na=setup:active\r\na=mid:0\r\na=sendr"..., len = 24, arena = 0x7f3669e2a428}, {s = 0x7f372039e930 "3axr8PdzOloV7UUdRxZnjvKwAE", len = 26, arena = 0x7f3669e2a428}}, agent_flags = {a = 28}}
(gdb) p *ag->call
$7 = {obj = {ref = 2, clear_func = 0x55fe37f6efe0 <__call_free>, free_func = 0x7f37692da820 <g_free>}, buffer = {pieces = 0x7f365082a310, error = 0}, poller = 0x55fe6453ad50, master_lock = {__data = {__readers = 8, __writers = 0,
      __wrphase_futex = 0, __writers_futex = 0, __pad3 = 0, __pad4 = 0, __cur_writer = 0, __shared = 0, __rwelision = 0 '\000', __pad1 = "\000\000\000\000\000\000", __pad2 = 0, __flags = 0}, __size = "\b", '\000' <repeats 54 times>,
    __align = 8}, monologues = {q = {head = 0x7f3668302280, tail = 0x7f36505334e0, length = 2}, {head = 0x7f3668302280, tail = 0x7f36505334e0, length = 2}, __t = 0x7f3668302280, __ct = 0x7f3668302280}, medias = {q = {
      head = 0x7f3650952500, tail = 0x7f3650dae8a0, length = 2}, {head = 0x7f3650952500, tail = 0x7f3650dae8a0, length = 2}, __t = 0x7f3650952500, __ct = 0x7f3650952500}, tags = {ht = 0x7f3650766e70, __key = 0x7f3650766e70,
    __ckey = 0x7f3650766e70, __value = 0x7f3650766e70, __iter = 0x7f3650766e70}, viabranches = {ht = 0x7f3687feab90, __key = 0x7f3687feab90, __ckey = 0x7f3687feab90, __value = 0x7f3687feab90, __iter = 0x7f3687feab90}, labels = {
    ht = 0x7f3687feac20, __key = 0x7f3687feac20, __ckey = 0x7f3687feac20, __value = 0x7f3687feac20, __iter = 0x7f3687feac20}, sdps = {ht = 0x7f3722bbee50, __key = 0x7f3722bbee50, __ckey = 0x7f3722bbee50, __value = 0x7f3722bbee50,
    __iter = 0x7f3722bbee50}, endpoints = {ht = 0x7f37205af660, __key = 0x7f37205af660, __ckey = 0x7f37205af660, __value = 0x7f37205af660, __iter = 0x7f37205af660}, sdp_fragments = {ht = 0x7f37205af760, __key = 0x7f37205af760,
    __ckey = 0x7f37205af760, __value = 0x7f37205af760, __iter = 0x7f37205af760}, streams = {q = {head = 0x7f366b1df010, tail = 0x7f366bd51920, length = 4}, {head = 0x7f366b1df010, tail = 0x7f366bd51920, length = 4},
    __t = 0x7f366b1df010, __ct = 0x7f366b1df010}, stream_fds = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, endpoint_maps = {q = {head = 0x7f37217c44c0,
      tail = 0x7f3650babce0, length = 2}, {head = 0x7f37217c44c0, tail = 0x7f3650babce0, length = 2}, __t = 0x7f37217c44c0, __ct = 0x7f37217c44c0}, dtls_cert = 0x55fe6432d000, mqtt_timer = 0x0, callid = {
    s = 0x7f3720b05fe8 "deb06463-d3bc-123f-4ba3-0a2c15959079", len = 36, arena = 0x7f3669e2a428}, callid_aliases = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0},
  created = 1779810375205278, destroyed = 1779810375489040, last_signal_us = 1779810375347652, deleted_us = 0, ml_deleted_us = 0, tos = 184 '\270', created_from = {s = 0x7f3720b069e8 "10.0.0.4:42758", len = 14,
    arena = 0x7f3669e2a428}, xmlrpc_callback = {s = 0x0, len = 0, arena = 0x0}, dtmf_log_dest = {address = {family = 0x0, {ipv4 = {s_addr = 0}, ipv6 = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0,
              0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}}}, port = 0}, redis_hosted_db = -1, last_redis_update_us = {a = 0}, recording = 0x0, metadata = {s = 0x0, len = 0, arena = 0x0}, recording_meta_prefix = {s = 0x0, len = 0,
    arena = 0x0}, recording_file = {s = 0x0, len = 0, arena = 0x0}, recording_random_tag = {s = 0x0, len = 0, arena = 0x0}, recording_path = {s = 0x0, len = 0, arena = 0x0}, recording_pattern = {s = 0x0, len = 0, arena = 0x0},
  iterator = {{link = {l = {data = 0x0, next = 0x0, prev = 0x0}, {{data = 0x0, __ct = 0x0}, next = 0x0, prev = 0x0}}, next_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0,
          __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, prev_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {
            __prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}, {link = {l = {data = 0x0, next = 0x0, prev = 0x0}, {{data = 0x0, __ct = 0x0}, next = 0x0, prev = 0x0}}, next_lock = {__data = {__lock = 0,
          __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, prev_lock = {__data = {__lock = 0, __count = 0,
          __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}, {link = {l = {data = 0x0, next = 0x0, prev = 0x0}, {{data = 0x0,
            __ct = 0x0}, next = 0x0, prev = 0x0}}, next_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
        __size = '\000' <repeats 39 times>, __align = 0}, prev_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
        __size = '\000' <repeats 39 times>, __align = 0}}, {link = {l = {data = 0x0, next = 0x0, prev = 0x0}, {{data = 0x0, __ct = 0x0}, next = 0x0, prev = 0x0}}, next_lock = {__data = {__lock = 0, __count = 0, __owner = 0,
          __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, prev_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0,
          __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}}, cpu_affinity = -1, block_dtmf = BLOCK_DTMF_OFF, call_flags = {a = 327680}, update_iter = 3,
  media_rec_slots = 0}
(gdb) p *ag->media
$8 = {monologue = 0x7f3720e01db8, call = 0x7f3669e2a410, index = 1, unique_id = 1, sp = {index = 1, type = {
      s = 0x7f36691d1392 "audio 60933 UDP/TLS/RTP/SAVP 8 0 9 102 101 103\r\nc=IN IP4 160.178.109.209\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:3872729472 1 udp 2122260223 192.168.100.48 60933 typ host generation 0 network-id 1\r\na=ca"..., len = 5, arena = 0x7f3669e2a428}, type_id = MT_AUDIO, rtp_endpoint = {address = {family = 0x55fe380cebe0 <__socket_families.lto_priv.0>, {ipv4 = {s_addr = 3513627296}, ipv6 = {__in6_u = {
              __u6_addr8 = "\240\262m\321", '\000' <repeats 11 times>, __u6_addr16 = {45728, 53613, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {3513627296, 0, 0, 0}}}}}, port = 60933}, rtcp_endpoint = {address = {
        family = 0x55fe380cebe0 <__socket_families.lto_priv.0>, {ipv4 = {s_addr = 3513627296}, ipv6 = {__in6_u = {__u6_addr8 = "\240\262m\321", '\000' <repeats 11 times>, __u6_addr16 = {45728, 53613, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {
                3513627296, 0, 0, 0}}}}}, port = 9}, consecutive_ports = 1, num_ports = 2, protocol_str = {
      s = 0x7f36691d139e "UDP/TLS/RTP/SAVP 8 0 9 102 101 103\r\nc=IN IP4 160.178.109.209\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:3872729472 1 udp 2122260223 192.168.100.48 60933 typ host generation 0 network-id 1\r\na=candidate:1235"..., len = 16, arena = 0x7f3669e2a428}, protocol = 0x55fe380bcc20 <transport_protocols+128>, format_str = {
      s = 0x7f36691d13af "8 0 9 102 101 103\r\nc=IN IP4 160.178.109.209\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:3872729472 1 udp 2122260223 192.168.100.48 60933 typ host generation 0 network-id 1\r\na=candidate:1235208382 1 udp 1686"..., len = 17, arena = 0x7f3669e2a428}, sdes_params = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, generic_attributes = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0,
        tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, all_attributes = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, direction = {{s = 0x7f37228c063d "", len = 8,
        arena = 0x0}, {s = 0x7f37228c0647 "", len = 8, arena = 0x0}}, fingerprint = {digest = "B\330\031\316\352\034\021\321\3123\205\236\021\332\037\352\345[\326\364\251F\226\364\347\303\316\355\327\360Dh", '\000' <repeats 31 times>,
      digest_len = 32, hash_func = 0x55fe380bcce0 <hash_funcs.lto_priv>}, sp_flags = {a = 1212}, codecs = {codecs = {ht = 0x0, __key = 0x0, __ckey = 0x0, __value = 0x0, __iter = 0x0}, codec_names = {ht = 0x0, __key = 0x0,
        __ckey = 0x0, __value = 0x0, __iter = 0x0}, codec_prefs = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, supp_link = 0x0, tracker = 0x0, media = 0x0, strip_all = 0,
      strip_full = 0}, ice_candidates = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, ice_ufrag = {
      s = 0x7f36691d14e6 "Lb+z\r\na=ice-pwd:G/ZWx2+xpQVfw8sYX+OCzdhh\r\na=ice-options:trickle\r\na=fingerprint:sha-256 42:D8:19:CE:EA:1C:11:D1:CA:33:85:9E:11:DA:1F:EA:E5:5B:D6:F4:A9:46:96:F4:E7:C3:CE:ED:D7:F0:44:68\r\na=setup:active\r\n"..., len = 4, arena = 0x7f3669e2a428}, ice_pwd = {
      s = 0x7f36691d14f6 "G/ZWx2+xpQVfw8sYX+OCzdhh\r\na=ice-options:trickle\r\na=fingerprint:sha-256 42:D8:19:CE:EA:1C:11:D1:CA:33:85:9E:11:DA:1F:EA:E5:5B:D6:F4:A9:46:96:F4:E7:C3:CE:ED:D7:F0:44:68\r\na=setup:active\r\na=mid:0\r\na=sendr"..., len = 24, arena = 0x7f3669e2a428}, ptime = 0, maxptime = 0, media_id = {
      s = 0x7f36691d15b4 "0\r\na=sendrecv\r\na=msid:b6dd5444-adec-42da-9b64-703c86211ad7 2594888f-4787-443f-a795-f3721620e681\r\na=rtcp-mux\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:9 G722/8000\r\na=rtpmap:102 opus/48000/2"..., len = 1, arena = 0x7f3669e2a428}, t38_options = {version = 0, fec_span = 0, min_ec_entries = 3, max_ec_entries = 3, max_ifp = 0, max_datagram = 0, local_tcf = 0, fill_bit_removal = 0, transcoding_mmr = 0,
      transcoding_jbig = 0, no_ecm = 0, no_v17 = 0, no_v27ter = 0, no_v29 = 0, no_v34 = 0, no_iaf = 0}, tls_id = {s = 0x0, len = 0, arena = 0x0}, media_sdp_id = 0, media_session_bandiwdth = {as = -1, rr = -1, rs = -1, ct = -1,
      tias = -1}, sdp_information = {s = 0x0, len = 0, arena = 0x0}, extmap = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, label = {s = 0x0, len = 0, arena = 0x0}}, type = {
    s = 0x7f3720b06075 "audio 25808 RTP/AVP 8 0 9 102 101 103\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:9 G722/8000\r\na=rtpmap:102 opus/48000/2\r\na=fmtp:102 useinbandfec=1; maxaveragebitrate=30000; maxplaybackrate="..., len = 5, arena = 0x7f3669e2a428}, type_id = MT_AUDIO, protocol_str = {
    s = 0x7f36691d139e "UDP/TLS/RTP/SAVP 8 0 9 102 101 103\r\nc=IN IP4 160.178.109.209\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:3872729472 1 udp 2122260223 192.168.100.48 60933 typ host generation 0 network-id 1\r\na=candidate:1235"..., len = 16, arena = 0x7f3669e2a428}, protocol = 0x55fe380bcc20 <transport_protocols+128>, format_str = {
    s = 0x7f36691d13af "8 0 9 102 101 103\r\nc=IN IP4 160.178.109.209\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:3872729472 1 udp 2122260223 192.168.100.48 60933 typ host generation 0 network-id 1\r\na=candidate:1235208382 1 udp 1686"..., len = 17, arena = 0x7f3669e2a428}, desired_family = 0x55fe380cebe0 <__socket_families.lto_priv.0>, logical_intf = 0x55fe643232c0, ice_agent = 0x0, extmap = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0,
      length = 0}, __t = 0x0, __ct = 0x0}, extmap_a = {0x0 <repeats 14 times>}, extmap_id = {0x0}, extmap_ht = {ht = 0x7f37225a8a70, __key = 0x7f37225a8a70, __ckey = 0x7f37225a8a70, __value = 0x7f37225a8a70, __iter = 0x7f37225a8a70},
  extmap_ops = 0x55fe380bcae0 <extmap_ops_short.lto_priv>, media_id = {s = 0x0, len = 0, arena = 0x0}, label = {s = 0x0, len = 0, arena = 0x0}, bundle = 0x0, pt_media = {ht = 0x0, __key = 0x0, __ckey = 0x0, __value = 0x0,
    __iter = 0x0}, sdes_in = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, sdes_out = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0},
    __t = 0x0, __ct = 0x0}, fingerprint = {digest = "B\330\031\316\352\034\021\321\3123\205\236\021\332\037\352\345[\326\364\251F\226\364\347\303\316\355\327\360Dh", '\000' <repeats 31 times>, digest_len = 32,
    hash_func = 0x55fe380bcce0 <hash_funcs.lto_priv>}, fp_hash_func = 0x55fe380bcce0 <hash_funcs.lto_priv>, tls_id = {s = 0x0, len = 0, arena = 0x0}, ice_candidates = {q = {head = 0x7f365071a960, tail = 0x7f37204a4410, length = 2}, {
      head = 0x7f365071a960, tail = 0x7f37204a4410, length = 2}, __t = 0x7f365071a960, __ct = 0x7f365071a960}, media_rec_slot = 0, streams = {q = {head = 0x7f3720811460, tail = 0x7f3668bd86b0, length = 2}, {head = 0x7f3720811460,
      tail = 0x7f3668bd86b0, length = 2}, __t = 0x7f3720811460, __ct = 0x7f3720811460}, endpoint_map = 0x7f366bab8628, endpoint_maps = {q = {head = 0x7f3722fd1660, tail = 0x7f3722fd1660, length = 1}, {head = 0x7f3722fd1660,
      tail = 0x7f3722fd1660, length = 1}, __t = 0x7f3722fd1660, __ct = 0x7f3722fd1660}, ssrc_hash_in = {nq = {head = 0x7f366c35c9c8, tail = 0x7f366c35c9c8, length = 1}, lock = {__data = {__lock = 0, __count = 0, __owner = 0,
        __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, create_func = 0x55fe37ff71c0 <create_ssrc_entry_call>, uptr = 0x0, precreat = 0x0,
    iters = 0}, ssrc_hash_out = {nq = {head = 0x0, tail = 0x0, length = 0}, lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
      __size = '\000' <repeats 39 times>, __align = 0}, create_func = 0x55fe37ff71c0 <create_ssrc_entry_call>, uptr = 0x0, precreat = 0x7f37259907b0, iters = 0}, codecs = {codecs = {ht = 0x7f37219fe010, __key = 0x7f37219fe010,
      __ckey = 0x7f37219fe010, __value = 0x7f37219fe010, __iter = 0x7f37219fe010}, codec_names = {ht = 0x7f3723d84210, __key = 0x7f3723d84210, __ckey = 0x7f3723d84210, __value = 0x7f3723d84210, __iter = 0x7f3723d84210}, codec_prefs = {
      q = {head = 0x7f366a746eb0, tail = 0x7f366a795c50, length = 6}, {head = 0x7f366a746eb0, tail = 0x7f366a795c50, length = 6}, __t = 0x7f366a746eb0, __ct = 0x7f366a746eb0}, supp_link = 0x7f366b647710, tracker = 0x7f3720037e30,
    media = 0x7f37227969d8, strip_all = 0, strip_full = 0}, offered_codecs = {codecs = {ht = 0x7f3669ad8190, __key = 0x7f3669ad8190, __ckey = 0x7f3669ad8190, __value = 0x7f3669ad8190, __iter = 0x7f3669ad8190}, codec_names = {
      ht = 0x7f3669b54960, __key = 0x7f3669b54960, __ckey = 0x7f3669b54960, __value = 0x7f3669b54960, __iter = 0x7f3669b54960}, codec_prefs = {q = {head = 0x7f3720614f20, tail = 0x7f3720ff7e70, length = 6}, {head = 0x7f3720614f20,
        tail = 0x7f3720ff7e70, length = 6}, __t = 0x7f3720614f20, __ct = 0x7f3720614f20}, supp_link = 0x7f3650b404a0, tracker = 0x7f3722777d90, media = 0x7f37227969d8, strip_all = 0, strip_full = 0}, generic_attributes = {q = {
      head = 0x7f365001ee30, tail = 0x7f366a356530, length = 2}, {head = 0x7f365001ee30, tail = 0x7f366a356530, length = 2}, __t = 0x7f365001ee30, __ct = 0x7f365001ee30}, all_attributes = {q = {head = 0x7f366a3559e0,
      tail = 0x7f37239ee010, length = 20}, {head = 0x7f366a3559e0, tail = 0x7f37239ee010, length = 20}, __t = 0x7f366a3559e0, __ct = 0x7f366a3559e0}, sdp_attr_print = 0x55fe37f86ca0 <sdp_insert_media_attributes>, codec_handlers = {
    ht = 0x7f37222b7840, __key = 0x7f37222b7840, __ckey = 0x7f37222b7840, __value = 0x7f37222b7840, __iter = 0x7f37222b7840}, codec_handlers_store = {q = {head = 0x7f36501d2ee0, tail = 0x7f3723c92780, length = 6}, {
      head = 0x7f36501d2ee0, tail = 0x7f3723c92780, length = 6}, __t = 0x7f36501d2ee0, __ct = 0x7f36501d2ee0}, codec_handler_cache = 0x7f372003e2e0, rtcp_handler = 0x0, rtcp_timer = 0x0, mqtt_timer = 0x0, t38_gateway = 0x0,
  audio_player = 0x0, t38_handler = 0x0, buffer_delay = 0, media_subscriptions_ht = {ht = 0x7f366b09e8e0, __key = 0x7f366b09e8e0, __ckey = 0x7f366b09e8e0, __value = 0x7f366b09e8e0, __iter = 0x7f366b09e8e0}, media_subscribers_ht = {
    ht = 0x7f3723736df0, __key = 0x7f3723736df0, __ckey = 0x7f3723736df0, __value = 0x7f3723736df0, __iter = 0x7f3723736df0}, media_subscribers = {{{head = 0x7f372392d840, tail = 0x7f372392d840, length = 2}, offset = 0x7f372392d840,
      const_ele = 0x7f372392d840}}, media_subscriptions = {{{head = 0x7f366a79d5f0, tail = 0x7f366a79d5f0, length = 2}, offset = 0x7f366a79d5f0, const_ele = 0x7f366a79d5f0}}, dtmf_lock = {__data = {__lock = 0, __count = 0,
      __owner = 0, __nusers = 0, __kind = -1, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 16 times>, "\377\377\377\377", '\000' <repeats 19 times>, __align = 0}, dtmf_ts = 0,
  dtmf_count = 0, dtmf_recv = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0}, __t = 0x0, __ct = 0x0}, dtmf_send = {q = {head = 0x0, tail = 0x0, length = 0}, {head = 0x0, tail = 0x0, length = 0},
    __t = 0x0, __ct = 0x0}, media_sdp_id = 0, sdp_media_bandwidth = {as = -1, rr = -1, rs = -1, ct = -1, tias = -1}, sdp_information = {s = 0x0, len = 0, arena = 0x0}, encoder_callback = {amr = {cmr_in_ts = 0, cmr_in = 0,
      cmr_out_ts = 0, cmr_out = 0}, evs = {cmr_in_ts = 0, cmr_in = 0}}, ptime = 20, maxptime = 0, media_flags = {a = 274880464092}, update_iter = 3}
(gdb) p ag->media->streams
$9 = {q = {head = 0x7f3720811460, tail = 0x7f3668bd86b0, length = 2}, {head = 0x7f3720811460, tail = 0x7f3668bd86b0, length = 2}, __t = 0x7f3720811460, __ct = 0x7f3720811460}

[rfuchs]

I'm assuming that this was a random occurrence and not something you know how to reproduce? (I have a possible fix, just would be nice to fully confirm)

[ccppprogrammer]

Yes, that’s exactly right. rtpengine ran under heavy load for several days, and then started crashing randomly at the end of the day. Unfortunately, I have no idea why this happened or how to reproduce the issue.
I can install the fixed version, but the main thing is to get the deb package with the fix into the dfx repository. I’ll be tracking it in the REL=beta branch of the dfx repository.