diff --git a/profiling/build.rs b/profiling/build.rs index 77f5349b8f..66c6b63266 100644 --- a/profiling/build.rs +++ b/profiling/build.rs @@ -358,11 +358,13 @@ fn cfg_php_feature_flags(vernum: u64) { if vernum >= 80400 { println!("cargo:rustc-cfg=php_frameless"); println!("cargo:rustc-cfg=php_opcache_restart_hook"); + println!("cargo:rustc-cfg=php_zend_mm_set_custom_handlers_ex"); } } fn cfg_zts() { let output = Command::new("php") + .arg("-n") .arg("-r") .arg("echo PHP_ZTS, PHP_EOL;") .output() diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs new file mode 100644 index 0000000000..6be8ba5fcc --- /dev/null +++ b/profiling/src/allocation/allocation_ge84.rs @@ -0,0 +1,444 @@ +use crate::allocation::ALLOCATION_PROFILING_COUNT; +use crate::allocation::ALLOCATION_PROFILING_SIZE; +use crate::allocation::ALLOCATION_PROFILING_STATS; +use crate::bindings::{self as zend}; +use crate::PROFILER_NAME; +use libc::{c_char, c_void, size_t}; +use log::{debug, trace, warn}; +use std::cell::UnsafeCell; +use std::ptr; +use std::sync::atomic::Ordering::SeqCst; + +struct ZendMMState { + /// The heap we create and set as the current heap in ZendMM + heap: *mut zend::zend_mm_heap, + /// The heap installed in ZendMM at the time we install our custom handlers + prev_heap: *mut zend::zend_mm_heap, + /// The engine's previous custom allocation function, if there is one. + prev_custom_mm_alloc: Option, + /// The engine's previous custom reallocation function, if there is one. + prev_custom_mm_realloc: Option, + /// The engine's previous custom free function, if there is one. + prev_custom_mm_free: Option, + /// The engine's previous custom gc function, if there is one. + prev_custom_mm_gc: Option, + /// The engine's previous custom shutdown function, if there is one. + prev_custom_mm_shutdown: Option, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_alloc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_alloc` is initialised to a valid function + /// pointer, otherwise there will be dragons. + alloc: unsafe fn(size_t) -> *mut c_void, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_realloc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid + /// function pointer, otherwise there will be dragons. + realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_free()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_free` is initialised to a valid function + /// pointer, otherwise there will be dragons. + free: unsafe fn(*mut c_void), + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_gc()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_gc` is initialised to a valid function + /// pointer, otherwise there will be dragons. + gc: unsafe fn() -> size_t, + /// Safety: this function pointer is only allowed to point to + /// `alloc_prof_prev_shutdown()` when at the same time the + /// `ZEND_MM_STATE.prev_custom_mm_shutdown` is initialised to a valid function + /// pointer, otherwise there will be dragons. + shutdown: unsafe fn(bool, bool), +} + +impl ZendMMState { + const fn new() -> ZendMMState { + ZendMMState { + // Safety: Using `ptr::null_mut()` might seem dangerous but actually it is okay in this + // case. The `heap` and `prev_heap` fields will be initialized in the first call to + // RINIT and only used after that. By using this "trick" we can get rid of all + // `unwrap()` calls when using the `heap` or `prev_heap` field. Alternatively we could + // use `unwrap_unchecked()` for the same performance characteristics. + heap: ptr::null_mut(), + prev_heap: ptr::null_mut(), + prev_custom_mm_alloc: None, + prev_custom_mm_realloc: None, + prev_custom_mm_free: None, + prev_custom_mm_gc: None, + prev_custom_mm_shutdown: None, + alloc: alloc_prof_orig_alloc, + realloc: alloc_prof_orig_realloc, + free: alloc_prof_orig_free, + gc: alloc_prof_orig_gc, + shutdown: alloc_prof_orig_shutdown, + } + } +} + +impl ZendMMState {} + +thread_local! { + /// Using an `UnsafeCell` here should be okay. There might not be any + /// synchronisation issues, as it is used in as thread local and only + /// mutated in RINIT and RSHUTDOWN. + static ZEND_MM_STATE: UnsafeCell = const { + UnsafeCell::new(ZendMMState::new()) + }; +} + +macro_rules! tls_zend_mm_state { + ($x:ident) => { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + (*zend_mm_state).$x + }) + }; +} + +/// This initializes the thread locale variable `ZEND_MM_STATE` with respect to the currently +/// installed `zend_mm_heap` in ZendMM. It guarantees compliance with the safety guarantees +/// described in the `ZendMMState` structure, specifically for `ZendMMState::alloc`, +/// `ZendMMState::realloc`, `ZendMMState::free`, `ZendMMState::gc` and `ZendMMState::shutdown`. +/// This function may panic if called out of order! +pub fn alloc_prof_ginit() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + + // Only need to create an observed heap once per thread. When we have it, we can just + // install the observed heap via `zend::zend_mm_set_heap()` + if unsafe { !(*zend_mm_state).heap.is_null() } { + // This can only happen if either MINIT or GINIT is being called out of order. + panic!("MINIT/GINIT was called with an already initialized allocation profiler. Most likely the SAPI did this without going through MSHUTDOWN/GSHUTDOWN before."); + } + + // Safety: `zend_mm_get_heap()` always returns a non-null pointer to a valid heap structure + let prev_heap = unsafe { zend::zend_mm_get_heap() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(prev_heap) }; + + if !is_zend_mm() { + // Neighboring custom memory handlers found in the currently used ZendMM heap + debug!("Found another extension using the ZendMM custom handler hook"); + unsafe { + zend::zend_mm_get_custom_handlers_ex( + prev_heap, + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc), + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown), + ); + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_prev_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_prev_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_prev_realloc); + // `gc` handler can be NULL + if (*zend_mm_state).prev_custom_mm_gc.is_none() { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + } else { + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_prev_gc); + } + // `shutdown` handler can be NULL + if (*zend_mm_state).prev_custom_mm_shutdown.is_none() { + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_orig_shutdown); + } else { + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_prev_shutdown); + } + } + } + + // Create a new (to be observed) heap and prepare custom handlers + let heap = unsafe { zend::zend_mm_startup() }; + unsafe { ptr::addr_of_mut!((*zend_mm_state).heap).write(heap) }; + + // install our custom handler to ZendMM + unsafe { + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap, + Some(alloc_prof_malloc), + Some(alloc_prof_free), + Some(alloc_prof_realloc), + Some(alloc_prof_gc), + Some(alloc_prof_shutdown), + ); + } + debug!("New observed heap created"); + }); +} + +/// This resets the thread locale variable `ZEND_MM_STATE` and frees allocated memory. It +/// guarantees compliance with the safety guarantees described in the `ZendMMState` structure, +/// specifically for `ZendMMState::alloc`, `ZendMMState::realloc`, `ZendMMState::free`, +/// `ZendMMState::gc` and `ZendMMState::shutdown`. +pub fn alloc_prof_gshutdown() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + unsafe { + // Remove custom handlers to allow for ZendMM internal shutdown + zend::zend_mm_set_custom_handlers_ex( + (*zend_mm_state).heap, + None, + None, + None, + None, + None, + ); + + // Reset ZEND_MM_STATE to defaults, now that the pointer are not know to the observed + // heap anymore. + ptr::addr_of_mut!((*zend_mm_state).alloc).write(alloc_prof_orig_alloc); + ptr::addr_of_mut!((*zend_mm_state).free).write(alloc_prof_orig_free); + ptr::addr_of_mut!((*zend_mm_state).realloc).write(alloc_prof_orig_realloc); + ptr::addr_of_mut!((*zend_mm_state).gc).write(alloc_prof_orig_gc); + ptr::addr_of_mut!((*zend_mm_state).shutdown).write(alloc_prof_orig_shutdown); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_alloc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_free).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_realloc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_gc).write(None); + ptr::addr_of_mut!((*zend_mm_state).prev_custom_mm_shutdown).write(None); + + // This shutdown call will free the observed heap we created in `alloc_prof_custom_heap_init` + zend::zend_mm_shutdown((*zend_mm_state).heap, true, true); + + // Now that the heap is gone, we need to NULL the pointer + ptr::addr_of_mut!((*zend_mm_state).heap).write(ptr::null_mut()); + ptr::addr_of_mut!((*zend_mm_state).prev_heap).write(ptr::null_mut()); + } + trace!("Observed heap was freed and `zend_mm_state` reset"); + }); +} + +pub fn alloc_prof_rinit() { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `zend_mm_state.heap` got initialized in `MINIT` and is guaranteed to + // be a non null pointer to a valid `zend::zend_mm_heap` struct + unsafe { + // Install our observed heap into ZendMM + zend::zend_mm_set_heap((*zend_mm_state).heap); + } + }); + + // `is_zend_mm()` should be false now, as we installed our custom handlers + if is_zend_mm() { + // Can't proceed with it being disabled, because that's a system-wide + // setting, not per-request. + panic!("Memory allocation profiling could not be enabled. Please feel free to fill an issue stating the PHP version and installed modules. Most likely the reason is your PHP binary was compiled with `ZEND_MM_CUSTOM` being disabled."); + } + trace!("Memory allocation profiling enabled.") +} + +pub fn alloc_prof_rshutdown() { + // If `is_zend_mm()` is true, the custom handlers have been reset to `None` or our observed + // heap has been uninstalled. This is unexpected, therefore we will not touch the ZendMM + // handlers anymore as resetting to prev handlers might result in segfaults and other undefined + // behaviour. + if is_zend_mm() { + return; + } + + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + + // Do a sanity check and see if something played with our heap + let mut custom_mm_malloc: Option = None; + let mut custom_mm_free: Option = None; + let mut custom_mm_realloc: Option = None; + let mut custom_mm_gc: Option = None; + let mut custom_mm_shutdown: Option = None; + + // Safety: `unwrap()` is safe here, as `heap` is initialized in `MINIT` + let heap = unsafe { (*zend_mm_state).heap }; + unsafe { + zend::zend_mm_get_custom_handlers_ex( + heap, + &mut custom_mm_malloc, + &mut custom_mm_free, + &mut custom_mm_realloc, + &mut custom_mm_gc, + &mut custom_mm_shutdown, + ); + } + if custom_mm_free != Some(alloc_prof_free) + || custom_mm_malloc != Some(alloc_prof_malloc) + || custom_mm_realloc != Some(alloc_prof_realloc) + || custom_mm_gc != Some(alloc_prof_gc) + || custom_mm_shutdown != Some(alloc_prof_shutdown) + { + // Custom handlers are installed, but it's not us. Someone, somewhere might have + // function pointers to our custom handlers. Best bet to avoid segfaults is to not + // touch custom handlers in ZendMM and make sure our extension will not be + // `dlclose()`-ed so the pointers stay valid + let zend_extension = + unsafe { zend::zend_get_extension(PROFILER_NAME.as_ptr() as *const c_char) }; + if !zend_extension.is_null() { + // Safety: Checked for null pointer above. + unsafe { ptr::addr_of_mut!((*zend_extension).handle).write(ptr::null_mut()) }; + } + warn!("Found another extension using the custom heap which is unexpected at this point, so the extension handle was `null`'ed to avoid being `dlclose()`'ed."); + } else { + // This is the happy path. Restore previous heap. + unsafe { + zend::zend_mm_set_heap( + (*zend_mm_state).prev_heap + ); + } + trace!("Memory allocation profiling shutdown gracefully."); + } + }); +} + +unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { + ALLOCATION_PROFILING_COUNT.fetch_add(1, SeqCst); + ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, SeqCst); + + let ptr = tls_zend_mm_state!(alloc)(len); + + // during startup, minit, rinit, ... current_execute_data is null + // we are only interested in allocations during userland operations + if zend::ddog_php_prof_get_current_execute_data().is_null() { + return ptr; + } + + ALLOCATION_PROFILING_STATS.with(|cell| { + let mut allocations = cell.borrow_mut(); + allocations.track_allocation(len) + }); + + ptr +} + +unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); + // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_alloc` is also initialised + let ptr = ((*zend_mm_state).prev_custom_mm_alloc.unwrap_unchecked())(len); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap); + ptr + }) +} + +unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { + let ptr: *mut c_void = zend::_zend_mm_alloc(tls_zend_mm_state!(prev_heap), len); + ptr +} + +/// This function exists because when calling `zend_mm_set_custom_handlers()`, +/// you need to pass a pointer to a `free()` function as well, otherwise your +/// custom handlers won't be installed. We can not just point to the original +/// `zend::_zend_mm_free()` as the function definitions differ. +unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { + tls_zend_mm_state!(free)(ptr); +} + +unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); + // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_free` is also initialised + ((*zend_mm_state).prev_custom_mm_free.unwrap_unchecked())(ptr); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap); + }) +} + +unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { + zend::_zend_mm_free(tls_zend_mm_state!(prev_heap), ptr); +} + +unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + ALLOCATION_PROFILING_COUNT.fetch_add(1, SeqCst); + ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, SeqCst); + + let ptr = tls_zend_mm_state!(realloc)(prev_ptr, len); + + // during startup, minit, rinit, ... current_execute_data is null + // we are only interested in allocations during userland operations + if zend::ddog_php_prof_get_current_execute_data().is_null() || ptr == prev_ptr { + return ptr; + } + + ALLOCATION_PROFILING_STATS.with(|cell| { + let mut allocations = cell.borrow_mut(); + allocations.track_allocation(len) + }); + + ptr +} + +unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); + // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_realloc` is also initialised + let ptr = ((*zend_mm_state).prev_custom_mm_realloc.unwrap_unchecked())(prev_ptr, len); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap); + ptr + }) +} + +unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + zend::_zend_mm_realloc(tls_zend_mm_state!(prev_heap), prev_ptr, len) +} + +unsafe extern "C" fn alloc_prof_gc() -> size_t { + tls_zend_mm_state!(gc)() +} + +unsafe fn alloc_prof_prev_gc() -> size_t { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); + // Safety: `ZEND_MM_STATE.prev_custom_mm_gc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_gc` is also initialised + let freed = ((*zend_mm_state).prev_custom_mm_gc.unwrap_unchecked())(); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap); + freed + }) +} + +unsafe fn alloc_prof_orig_gc() -> size_t { + zend::zend_mm_gc(tls_zend_mm_state!(prev_heap)) +} + +unsafe extern "C" fn alloc_prof_shutdown(full: bool, silent: bool) { + tls_zend_mm_state!(shutdown)(full, silent); +} + +unsafe fn alloc_prof_prev_shutdown(full: bool, silent: bool) { + ZEND_MM_STATE.with(|cell| { + let zend_mm_state = cell.get(); + // Safety: `ZEND_MM_STATE.prev_heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).prev_heap); + // Safety: `ZEND_MM_STATE.prev_custom_mm_shutdown` will be initialised in + // `alloc_prof_rinit()` and only point to this function when + // `prev_custom_mm_shutdown` is also initialised + ((*zend_mm_state).prev_custom_mm_shutdown.unwrap_unchecked())(full, silent); + // Safety: `ZEND_MM_STATE.heap` got initialised in `alloc_prof_rinit()` + zend::zend_mm_set_heap((*zend_mm_state).heap); + }) +} + +unsafe fn alloc_prof_orig_shutdown(full: bool, silent: bool) { + zend::zend_mm_shutdown(tls_zend_mm_state!(prev_heap), full, silent) +} + +/// safe wrapper for `zend::is_zend_mm()`. +/// `true` means the internal ZendMM is being used, `false` means that a custom memory manager is +/// installed +fn is_zend_mm() -> bool { + unsafe { zend::is_zend_mm() } +} diff --git a/profiling/src/allocation.rs b/profiling/src/allocation/allocation_le83.rs similarity index 83% rename from profiling/src/allocation.rs rename to profiling/src/allocation/allocation_le83.rs index 8c05881a94..7f8b12617d 100644 --- a/profiling/src/allocation.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -1,41 +1,20 @@ +use crate::allocation::{ + ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE, ALLOCATION_PROFILING_STATS, +}; use crate::bindings::{ self as zend, datadog_php_install_handler, datadog_php_zif_handler, ddog_php_prof_copy_long_into_zval, }; -use crate::profiling::Profiler; use crate::{PROFILER_NAME, REQUEST_LOCALS}; use lazy_static::lazy_static; use libc::{c_char, c_int, c_void, size_t}; use log::{debug, error, trace, warn}; -use rand::rngs::ThreadRng; -use rand_distr::{Distribution, Poisson}; -use std::cell::{RefCell, UnsafeCell}; -use std::sync::atomic::AtomicU64; +use std::cell::UnsafeCell; use std::sync::atomic::Ordering::{Relaxed, SeqCst}; use std::{ffi, ptr}; static mut GC_MEM_CACHES_HANDLER: zend::InternalFunctionHandler = None; -/// take a sample every 4096 KiB -pub const ALLOCATION_PROFILING_INTERVAL: f64 = 1024.0 * 4096.0; - -/// This will store the count of allocations (including reallocations) during -/// a profiling period. This will overflow when doing more than u64::MAX -/// allocations, which seems big enough to ignore. -pub static ALLOCATION_PROFILING_COUNT: AtomicU64 = AtomicU64::new(0); - -/// This will store the accumulated size of all allocations in bytes during the -/// profiling period. This will overflow when allocating more than 18 exabyte -/// of memory (u64::MAX) which might not happen, so we can ignore this. -pub static ALLOCATION_PROFILING_SIZE: AtomicU64 = AtomicU64::new(0); - -pub struct AllocationProfilingStats { - /// number of bytes until next sample collection - next_sample: i64, - poisson: Poisson, - rng: ThreadRng, -} - type ZendHeapPrepareFn = unsafe fn(heap: *mut zend::_zend_mm_heap) -> c_int; type ZendHeapRestoreFn = unsafe fn(heap: *mut zend::_zend_mm_heap, custom_heap: c_int); @@ -69,49 +48,7 @@ struct ZendMMState { free: unsafe fn(*mut c_void), } -impl AllocationProfilingStats { - fn new() -> AllocationProfilingStats { - // Safety: this will only error if lambda <= 0 - let poisson = Poisson::new(ALLOCATION_PROFILING_INTERVAL).unwrap(); - let mut stats = AllocationProfilingStats { - next_sample: 0, - poisson, - rng: rand::thread_rng(), - }; - stats.next_sampling_interval(); - stats - } - - fn next_sampling_interval(&mut self) { - self.next_sample = self.poisson.sample(&mut self.rng) as i64; - } - - fn track_allocation(&mut self, len: size_t) { - self.next_sample -= len as i64; - - if self.next_sample > 0 { - return; - } - - self.next_sampling_interval(); - - if let Some(profiler) = Profiler::get() { - // Safety: execute_data was provided by the engine, and the profiler doesn't mutate it. - unsafe { - profiler.collect_allocations( - zend::ddog_php_prof_get_current_execute_data(), - 1_i64, - len as i64, - ) - }; - } - } -} - thread_local! { - static ALLOCATION_PROFILING_STATS: RefCell = - RefCell::new(AllocationProfilingStats::new()); - /// Using an `UnsafeCell` here should be okay. There might not be any /// synchronisation issues, as it is used in as thread local and only /// mutated in RINIT and RSHUTDOWN. @@ -150,7 +87,7 @@ lazy_static! { static ref JIT_ENABLED: bool = unsafe { zend::ddog_php_jit_enabled() }; } -pub fn alloc_prof_minit() { +pub fn alloc_prof_ginit() { unsafe { zend::ddog_php_opcache_init_handle() }; } @@ -167,23 +104,6 @@ pub fn first_rinit_should_disable_due_to_jit() -> bool { } pub fn alloc_prof_rinit() { - let allocation_profiling: bool = REQUEST_LOCALS.with(|cell| { - match cell.try_borrow() { - Ok(locals) => { - let system_settings = locals.system_settings(); - system_settings.profiling_allocation_enabled - }, - Err(_err) => { - error!("Memory allocation was not initialized correctly due to a borrow error. Please report this to Datadog."); - false - } - } - }); - - if !allocation_profiling { - return; - } - ZEND_MM_STATE.with(|cell| { let zend_mm_state = cell.get(); @@ -247,16 +167,6 @@ pub fn alloc_prof_rinit() { } pub fn alloc_prof_rshutdown() { - let allocation_profiling = REQUEST_LOCALS.with(|cell| { - cell.try_borrow() - .map(|locals| locals.system_settings().profiling_allocation_enabled) - .unwrap_or(false) - }); - - if !allocation_profiling { - return; - } - // If `is_zend_mm()` is true, the custom handlers have been reset to `None` // already. This is unexpected, therefore we will not touch the ZendMM // handlers anymore as resetting to prev handlers might result in segfaults @@ -401,7 +311,7 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { } unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { - // Safety: `ALLOCATION_PROFILING_ALLOC` will be initialised in + // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_alloc` is also initialised let alloc = tls_zend_mm_state!(prev_custom_mm_alloc).unwrap(); @@ -426,8 +336,8 @@ unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { } unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { - // Safety: `ALLOCATION_PROFILING_FREE` will be initialised in - // `alloc_prof_free()` and only point to this function when + // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in + // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_free` is also initialised let free = tls_zend_mm_state!(prev_custom_mm_free).unwrap(); free(ptr) @@ -459,8 +369,8 @@ unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> * } unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ALLOCATION_PROFILING_REALLOC` will be initialised in - // `alloc_prof_realloc()` and only point to this function when + // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in + // `alloc_prof_rinit()` and only point to this function when // `prev_custom_mm_realloc` is also initialised let realloc = tls_zend_mm_state!(prev_custom_mm_realloc).unwrap(); realloc(prev_ptr, len) diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs new file mode 100644 index 0000000000..f20e2e31be --- /dev/null +++ b/profiling/src/allocation/mod.rs @@ -0,0 +1,138 @@ +use crate::bindings::{self as zend}; +use crate::profiling::Profiler; +use crate::REQUEST_LOCALS; +use libc::size_t; +use log::{error, trace}; +use rand::rngs::ThreadRng; +use rand_distr::{Distribution, Poisson}; +use std::cell::RefCell; +use std::sync::atomic::AtomicU64; + +#[cfg(php_zend_mm_set_custom_handlers_ex)] +mod allocation_ge84; +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] +pub mod allocation_le83; + +/// take a sample every 4096 KiB +pub const ALLOCATION_PROFILING_INTERVAL: f64 = 1024.0 * 4096.0; + +/// This will store the count of allocations (including reallocations) during +/// a profiling period. This will overflow when doing more than u64::MAX +/// allocations, which seems big enough to ignore. +pub static ALLOCATION_PROFILING_COUNT: AtomicU64 = AtomicU64::new(0); + +/// This will store the accumulated size of all allocations in bytes during the +/// profiling period. This will overflow when allocating more than 18 exabyte +/// of memory (u64::MAX) which might not happen, so we can ignore this. +pub static ALLOCATION_PROFILING_SIZE: AtomicU64 = AtomicU64::new(0); + +pub struct AllocationProfilingStats { + /// number of bytes until next sample collection + next_sample: i64, + poisson: Poisson, + rng: ThreadRng, +} + +impl AllocationProfilingStats { + fn new() -> AllocationProfilingStats { + // Safety: this will only error if lambda <= 0 + let poisson = Poisson::new(ALLOCATION_PROFILING_INTERVAL).unwrap(); + let mut stats = AllocationProfilingStats { + next_sample: 0, + poisson, + rng: rand::thread_rng(), + }; + stats.next_sampling_interval(); + stats + } + + fn next_sampling_interval(&mut self) { + self.next_sample = self.poisson.sample(&mut self.rng) as i64; + } + + fn track_allocation(&mut self, len: size_t) { + self.next_sample -= len as i64; + + if self.next_sample > 0 { + return; + } + + self.next_sampling_interval(); + + if let Some(profiler) = Profiler::get() { + // Safety: execute_data was provided by the engine, and the profiler doesn't mutate it. + unsafe { + profiler.collect_allocations( + zend::ddog_php_prof_get_current_execute_data(), + 1_i64, + len as i64, + ) + }; + } + } +} + +thread_local! { + static ALLOCATION_PROFILING_STATS: RefCell = + RefCell::new(AllocationProfilingStats::new()); +} + +pub fn alloc_prof_ginit() { + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] + allocation_le83::alloc_prof_ginit(); + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_ginit(); +} + +pub fn alloc_prof_gshutdown() { + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_gshutdown(); +} + +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] +pub fn alloc_prof_startup() { + allocation_le83::alloc_prof_startup(); +} + +pub fn alloc_prof_rinit() { + let allocation_profiling: bool = REQUEST_LOCALS.with(|cell| { + match cell.try_borrow() { + Ok(locals) => { + let system_settings = locals.system_settings(); + system_settings.profiling_allocation_enabled + }, + Err(_err) => { + error!("Memory allocation was not initialized correctly due to a borrow error. Please report this to Datadog."); + false + } + } + }); + + if !allocation_profiling { + return; + } + + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] + allocation_le83::alloc_prof_rinit(); + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_rinit(); + + trace!("Memory allocation profiling enabled.") +} + +pub fn alloc_prof_rshutdown() { + let allocation_profiling = REQUEST_LOCALS.with(|cell| { + cell.try_borrow() + .map(|locals| locals.system_settings().profiling_allocation_enabled) + .unwrap_or(false) + }); + + if !allocation_profiling { + return; + } + + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] + allocation_le83::alloc_prof_rshutdown(); + #[cfg(php_zend_mm_set_custom_handlers_ex)] + allocation_ge84::alloc_prof_rshutdown(); +} diff --git a/profiling/src/bindings/mod.rs b/profiling/src/bindings/mod.rs index 8258ec3da6..5638dc8e91 100644 --- a/profiling/src/bindings/mod.rs +++ b/profiling/src/bindings/mod.rs @@ -46,6 +46,10 @@ pub type VmMmCustomAllocFn = unsafe extern "C" fn(size_t) -> *mut c_void; pub type VmMmCustomReallocFn = unsafe extern "C" fn(*mut c_void, size_t) -> *mut c_void; #[cfg(feature = "allocation_profiling")] pub type VmMmCustomFreeFn = unsafe extern "C" fn(*mut c_void); +#[cfg(all(feature = "allocation_profiling", php_zend_mm_set_custom_handlers_ex))] +pub type VmMmCustomGcFn = unsafe extern "C" fn() -> size_t; +#[cfg(all(feature = "allocation_profiling", php_zend_mm_set_custom_handlers_ex))] +pub type VmMmCustomShutdownFn = unsafe extern "C" fn(bool, bool); // todo: this a lie on some PHP versions; is it a problem even though zend_bool // was always supposed to be 0 or 1 anyway? @@ -175,13 +179,13 @@ pub struct ModuleEntry { /// thread for module globals. The function pointers in [`ModuleEntry::globals_ctor`] and /// [`ModuleEntry::globals_dtor`] will only be called if this is a non-zero. pub globals_size: size_t, - #[cfg(php_zts)] /// Pointer to a `ts_rsrc_id` (which is a [`i32`]). For C-Extension this is created using the /// `ZEND_DECLARE_MODULE_GLOBALS(module_name)` macro. /// See + #[cfg(php_zts)] pub globals_id_ptr: *mut ts_rsrc_id, - #[cfg(not(php_zts))] /// Pointer to the module globals struct in NTS mode + #[cfg(not(php_zts))] pub globals_ptr: *mut c_void, /// Constructor for module globals. /// Be aware this will only be called in case [`ModuleEntry::globals_size`] is non-zero and for diff --git a/profiling/src/config.rs b/profiling/src/config.rs index d8a9c4fb18..c21185e838 100644 --- a/profiling/src/config.rs +++ b/profiling/src/config.rs @@ -1,3 +1,4 @@ +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] use crate::allocation; use crate::bindings::zai_config_type::*; use crate::bindings::{ @@ -101,7 +102,8 @@ impl SystemSettings { } // Work around version-specific issues. - if allocation::first_rinit_should_disable_due_to_jit() { + #[cfg(not(php_zend_mm_set_custom_handlers_ex))] + if allocation::allocation_le83::first_rinit_should_disable_due_to_jit() { system_settings.profiling_allocation_enabled = false; } swap(&mut system_settings, SYSTEM_SETTINGS.assume_init_mut()); diff --git a/profiling/src/lib.rs b/profiling/src/lib.rs index 3b3db288a0..f8cf7c44be 100644 --- a/profiling/src/lib.rs +++ b/profiling/src/lib.rs @@ -31,7 +31,6 @@ use core::ptr; use ddcommon::{cstr, tag, tag::Tag}; use lazy_static::lazy_static; use libc::c_char; -#[cfg(php_zts)] use libc::c_void; use log::{debug, error, info, trace, warn}; use once_cell::sync::{Lazy, OnceCell}; @@ -184,14 +183,13 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { version: PROFILER_VERSION.as_ptr(), post_deactivate_func: Some(prshutdown), deps: DEPS.as_ptr(), - #[cfg(php_zts)] globals_ctor: Some(ginit), - #[cfg(php_zts)] globals_dtor: Some(gshutdown), - #[cfg(php_zts)] globals_size: 1, #[cfg(php_zts)] globals_id_ptr: unsafe { &mut GLOBALS_ID_PTR }, + #[cfg(not(php_zts))] + globals_ptr: ptr::null_mut(), ..Default::default() }); @@ -199,16 +197,20 @@ pub extern "C" fn get_module() -> &'static mut zend::ModuleEntry { unsafe { &mut *ptr::addr_of_mut!(MODULE) } } -#[cfg(php_zts)] unsafe extern "C" fn ginit(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_ginit(); + + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_ginit(); } -#[cfg(php_zts)] unsafe extern "C" fn gshutdown(_globals_ptr: *mut c_void) { #[cfg(all(feature = "timeline", php_zts))] timeline::timeline_gshutdown(); + + #[cfg(feature = "allocation_profiling")] + allocation::alloc_prof_gshutdown(); } /* Important note on the PHP lifecycle: @@ -337,9 +339,6 @@ extern "C" fn minit(_type: c_int, module_number: c_int) -> ZendResult { */ unsafe { zend::zend_register_extension(&extension, handle) }; - #[cfg(feature = "allocation_profiling")] - allocation::alloc_prof_minit(); - #[cfg(feature = "timeline")] timeline::timeline_minit(); @@ -875,7 +874,7 @@ extern "C" fn startup(extension: *mut ZendExtension) -> ZendResult { timeline::timeline_startup(); } - #[cfg(feature = "allocation_profiling")] + #[cfg(all(feature = "allocation_profiling", not(php_zend_mm_set_custom_handlers_ex)))] allocation::alloc_prof_startup(); ZendResult::Success