(gdb) bt full
#0 0x00007f6d613e7f7c in pthread_kill@@GLIBC_2.34 () from /usr/drte/v5/lib64/libc.so.6
No symbol table info available.
#1 0x00007f6d61396f22 in raise () from /usr/drte/v5/lib64/libc.so.6
No symbol table info available.
#2 0x000056028969833f in Envoy::SignalAction::sigHandler (sig=11, info=<optimized out>, context=0x7f6d2c28cb80) at external/envoy/source/common/signal/signal_action.cc:53
tracer = {<Envoy::Logger::Loggable<(Envoy::Logger::Id)4>> = {<No data fields>}, static log_to_stderr_ = false, static MaxStackDepth = 64, stack_trace_ = {0x5602894ceb8a <quic::HttpDecoder::ReadFrameType(quic::QuicDataReader*)+58>, 0x5602894d06c3 <quic::HttpDecoder::ProcessInput(char const*, unsigned long)+787>, 0x56028942cb2b <quic::QuicSpdyStream::OnDataAvailable()+315>, 0x560289437afb <quic::QpackDecodedHeadersAccumulator::OnDecodingCompleted()+107>, 0x5602895cec4a <quic::QpackDecoderHeaderTable::InsertEntry(std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >)+90>, 0x560289439b6d <quic::QpackEncoderStreamReceiver::OnInstructionDecoded(quic::QpackInstruction const*)+109>, 0x56028943f5f2 <quic::QpackInstructionDecoder::DoStartField()+626>, 0x56028943f750 <quic::QpackInstructionDecoder::Decode(std::basic_string_view<char, std::char_traits<char> >)+288>, 0x560289442035 <quic::QpackReceiveStream::OnDataAvailable()+117>, 0x56028947e4eb <quic::QuicStreamSequencer::OnFrameData(unsigned long, unsigned long, char const*)+843>, 0x56028949b8a2 <quic::QuicConnection::OnStreamFrame(quic::QuicStreamFrame const&)+626>, 0x5602894edc6c <quic::QuicFramer::ProcessIetfFrameData(quic::QuicDataReader*, quic::QuicPacketHeader const&, quic::EncryptionLevel)+460>, 0x5602894ef18b <quic::QuicFramer::ProcessIetfDataPacket(quic::QuicDataReader*, quic::QuicPacketHeader*, quic::QuicEncryptedPacket const&, char*, unsigned long)+2491>, 0x5602894ef49a <quic::QuicFramer::ProcessPacketInternal(quic::QuicEncryptedPacket const&)+458>, 0x5602894ef5d8 <quic::QuicFramer::ProcessPacket(quic::QuicEncryptedPacket const&)+24>, 0x5602894a61d2 <quic::QuicConnection::ProcessUdpPacket(quic::QuicSocketAddress const&, quic::QuicSocketAddress const&, quic::QuicReceivedPacket const&)+1010>, 0x560289469224 <quic::QuicSession::ProcessUdpPacket(quic::QuicSocketAddress const&, quic::QuicSocketAddress const&, quic::QuicReceivedPacket const&)+84>, 0x560289403d2d <Envoy::Quic::EnvoyQuicServerSession::ProcessUdpPacket(quic::QuicSocketAddress const&, quic::QuicSocketAddress const&, quic::QuicReceivedPacket const&)+61>, 0x5602893c55fd <quic::QuicDispatcher::MaybeDispatchPacket(quic::ReceivedPacketInfo const&)+429>, 0x5602893c8094 <quic::QuicDispatcher::ProcessPacket(quic::QuicSocketAddress const&, quic::QuicSocketAddress const&, quic::QuicReceivedPacket const&)+308>, 0x5602893b5df9 <Envoy::Quic::EnvoyQuicDispatcher::processPacket(quic::QuicSocketAddress const&, quic::QuicSocketAddress const&, quic::QuicReceivedPacket const&)+25>, 0x5602893b1beb <Envoy::Quic::ActiveQuicListener::onDataWorker(Envoy::Network::UdpRecvData&&)+235>, 0x5602893d8179 <Envoy::Network::UdpListenerImpl::processPacket(std::shared_ptr<Envoy::Network::Address::Instance const>, std::shared_ptr<Envoy::Network::Address::Instance const>, std::unique_ptr<Envoy::Buffer::Instance, std::default_delete<Envoy::Buffer::Instance> >, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >)+121>, 0x56028968b661 <Envoy::Network::passPayloadToProcessor(unsigned long, std::unique_ptr<Envoy::Buffer::Instance, std::default_delete<Envoy::Buffer::Instance> >, std::shared_ptr<Envoy::Network::Address::Instance const>, std::shared_ptr<Envoy::Network::Address::Instance const>, Envoy::Network::UdpPacketProcessor&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >)+689>, 0x56028968c541 <Envoy::Network::Utility::readFromSocket(Envoy::Network::IoHandle&, Envoy::Network::Address::Instance const&, Envoy::Network::UdpPacketProcessor&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >, bool, unsigned int*)+2801>, 0x56028968d1b4 <Envoy::Network::Utility::readPacketsFromSocket(Envoy::Network::IoHandle&, Envoy::Network::Address::Instance const&, Envoy::Network::UdpPacketProcessor&, Envoy::TimeSource&, bool, unsigned int&)+276>, 0x5602893d9625 <Envoy::Network::UdpListenerImpl::handleReadCallback()+293>, 0x5602893d9b2d <Envoy::Network::UdpListenerImpl::onSocketEvent(short)+381>, 0x56028952cb5e <std::_Function_handler<void(unsigned int), Envoy::Event::DispatcherImpl::createFileEvent(os_fd_t, Envoy::Event::FileReadyCb, Envoy::Event::FileTriggerType, uint32_t)::<lambda(uint32_t)> >::_M_invoke(const std::_Any_data &, unsigned int &&)+62>, 0x560289530921 <Envoy::Event::FileEventImpl::mergeInjectedEventsAndRunCb(unsigned int)+97>, 0x5602896bc922 <event_process_active_single_queue+1346>, 0x5602896bce47 <event_base_loop+1079>, 0x560288b48e72 <Envoy::Server::WorkerImpl::threadRoutine(Envoy::OptRef<Envoy::Server::GuardDog>, std::function<void ()> const&)+418>, 0x5602898531a5 <Envoy::Thread::ThreadImplPosix::ThreadImplPosix(std::function<void ()>, std::optional<Envoy::Thread::Options> const&)::{lambda(void*)#1}::_FUN(void*)+21>, 0x7f6d613e6173 <start_thread+723>, 0x4, 0x7f6d2c28cb50, 0x7f6d2c28cb60, 0x7f6d2c28cb30, 0x56028967f424 <Envoy::Http::HeaderMapImpl::HeaderEntryImpl::HeaderEntryImpl(Envoy::Http::LowerCaseString const&)+52>, 0x17c8455e8508, 0x17c8455e8508, 0x56028a805dd0 <guard variable for Envoy::ConstSingleton<Envoy::Runtime::RuntimeFeatures>::get()::instance>, 0x17c814f2a3c0, 0x17c8455e8508, 0x56028b11b9f8 <tcmalloc::tcmalloc_internal::Static::cpu_cache_active_>, 0x1000, 0xffffffffffffff98, 0x756e3a006e696769, 0x17c7ff997c10, 0x0, 0x56028a724640 <vtable for Envoy::Router::UpstreamRequest::DownstreamWatermarkManager+16>, 0x17c8455e8500, 0x40fee9e7a03fec00, 0x0, 0x40fee9e7a03fec00, 0x0, 0xfe, 0x17c86c0e9400, 0x5602896b5170 <Envoy::Buffer::OwnedImpl::add(void const*, unsigned long)>, 0x7f6d2c28cc30, 0x5602896b4ae4 <Envoy::Buffer::OwnedImpl::addImpl(void const*, unsigned long)+228>, 0x56028847f330 <non-virtual thunk to Envoy::Http::RequestHeaderMapImpl::setGrpcTimeout(std::basic_string_view<char, std::char_traits<char> >)>, 0x7f6d2c28cb60}, stack_depth_ = 35}
status = <optimized out>
__func__ = "sigHandler"
#3 <signal handler called>
No symbol table info available.
#4 quiche::QuicheDataReader::PeekVarInt62Length (this=this@entry=0x7f6d2c28d990) at external/com_github_google_quiche/quiche/common/quiche_data_reader.cc:153
next = 0x2ae0dedee4fc2702 <error: Cannot access memory at address 0x2ae0dedee4fc2702>
#5 0x00005602894ceb8a in quic::HttpDecoder::ReadFrameType (this=0x17c7eaff7140, reader=0x7f6d2c28d990) at external/com_github_google_quiche/quiche/quic/core/http/http_decoder.cc:151
success = <optimized out>
#6 0x00005602894d06c3 in quic::HttpDecoder::ProcessInput (this=this@entry=0x17c7eaff7140, data=<optimized out>, len=18446744073709551377) at external/com_github_google_quiche/quiche/quic/core/http/http_decoder.cc:114
reader = {<quiche::QuicheDataReader> = {data_ = 0x17c7e4eae0ef "", len_ = 18446744073709551377, pos_ = 3089688245975467539, endianness_ = quiche::NETWORK_BYTE_ORDER}, <No data fields>}
continue_processing = <optimized out>
#7 0x000056028942cb2b in quic::QuicSpdyStream::OnDataAvailable (this=0x17c7eaff6e00) at external/com_github_google_quiche/quiche/quic/core/http/quic_spdy_stream.cc:859
processed_bytes = <optimized out>
iov = {iov_base = 0x17c7e4eae0ef, iov_len = 18446744073709551377}
#8 0x0000560289437afb in quic::QpackDecodedHeadersAccumulator::OnDecodingCompleted (this=0x17c9ba089710) at external/com_github_google_quiche/quiche/quic/core/qpack/qpack_decoded_headers_accumulator.cc:63
No locals.
#9 0x00005602895cec4a in quic::QpackDecoderHeaderTable::InsertEntry (this=0x17c90e6f2a40, name=..., value=...) at external/com_github_google_quiche/quiche/quic/core/qpack/qpack_header_table.cc:189
it = <optimized out>
observer = 0x17ca518768c8
index = 38
#10 0x0000560289439b6d in quic::QpackEncoderStreamReceiver::OnInstructionDecoded (this=0x17c90e6f2920, instruction=0x17c851f9ea20) at external/com_github_google_quiche/quiche/quic/core/qpack/qpack_encoder_stream_receiver.cc:40
No locals.
#11 0x000056028943f5f2 in quic::QpackInstructionDecoder::DoStartField (this=0x17c90e6f2930) at external/com_github_google_quiche/quiche/quic/core/qpack/qpack_instruction_decoder.cc:109
No locals.
#12 0x000056028943f750 in quic::QpackInstructionDecoder::Decode (this=0x17c90e6f2930, data=...) at external/com_github_google_quiche/quiche/quic/core/qpack/qpack_instruction_decoder.cc:49
success = true
bytes_consumed = 0
#13 0x0000560289442035 in quic::QpackReceiveStream::OnDataAvailable (this=0x17c8f5e04000) at external/com_github_google_quiche/quiche/quic/core/qpack/qpack_receive_stream.cc:27
iov = {iov_base = 0x17c86cb185fa, iov_len = 226}
#14 0x000056028947e4eb in quic::QuicStreamSequencer::OnFrameData (this=0x17c8f5e04008, byte_offset=<optimized out>, data_len=226, data_buffer=0x7f6d2c28ee86 "[redacted]", <incomplete sequence \354\233>) at external/com_github_google_quiche/quiche/quic/core/quic_stream_sequencer.cc:124
previous_readable_bytes = 0
bytes_written = 226
error_details = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f6d2c28df90 ""}, _M_string_length = 0, {_M_local_buf = "\000p\367T\233\265X\346\337@\002\b\016\201Ob", _M_allocated_capacity = 16598216105424023552}}
result = quic::QUIC_NO_ERROR
__func__ = "OnFrameData"
stream_unblocked = true
#15 0x000056028949b8a2 in quic::QuicConnection::OnStreamFrame (this=0x17c8b5ade000, frame=...) at external/com_github_google_quiche/quiche/quic/core/quic_connection.cc:1379
__func__ = "OnStreamFrame"
This crash happens when QUIC stream buffers near 64MB worth of request/response data in its circular receive buffer and then starts to push data to the decoder. The excessive buffering is likely due to head of line blocking in Qpack decoding in QUICHE or slow Envoy decoding pipeline. The receive buffer is a circular buffer of 1024 memory blocks of 8KB each. The following access to the receive buffer will be located in the same memory block which the last received byte lands. In this case the peeked memory length will be calculated wrongly with a lower end offset - higher start offset which causes integer under flow.
Envoy users who has configured HTTP/3 upstream or downstream are vulnerable.
Summary
Following crash at
QuicheDataReader::PeekVarInt62Length()
are reported. It is caused by integer underflow inQuicStreamSequencerBuffer::PeekRegion()
implementation.Details
This crash happens when QUIC stream buffers near 64MB worth of request/response data in its circular receive buffer and then starts to push data to the decoder. The excessive buffering is likely due to head of line blocking in Qpack decoding in QUICHE or slow Envoy decoding pipeline. The receive buffer is a circular buffer of 1024 memory blocks of 8KB each. The following access to the receive buffer will be located in the same memory block which the last received byte lands. In this case the peeked memory length will be calculated wrongly with a lower end offset - higher start offset which causes integer under flow.
PoC
Hack a QUIC client implementation to send a large
POST
request (> 128MB) which reference to a request header in Qpack dynamic table. Do not send Qpack table update instructions. After the stream gets flow control blocked, send the Qpack table update.Impact
Envoy users who has configured HTTP/3 upstream or downstream are vulnerable.