Flame Graph
unix`rw_enter (1,485 samples, 0.05%)
unix`mutex_owner_running (1,175 samples, 0.04%)
unix`mul32 (702 samples, 0.03%)
unix`atomic_add_64 (14,688 samples, 0.53%)
unix`mutex_exit (438 samples, 0.02%)
genunix`fsflush_do_pages (2,319 samples, 0.08%)
unix`tsc_read (3,545 samples, 0.13%)
genunix`copyin_args32 (281 samples, 0.01%)
genunix`disp_lock_exit (40,780 samples, 1.46%)
c2audit`auditme (372 samples, 0.01%)
genunix`audit_getstate (536 samples, 0.02%)
genunix`avl_numnodes (845 samples, 0.03%)
zfs`zfs_range_unlock (3,230 samples, 0.12%)
unix`0xfffffffffb800ee8 (8,249 samples, 0.30%)
zfs`zio_null (7,875 samples, 0.28%)
zfs`zio_destroy (2,122 samples, 0.08%)
unix`tsc_gethrtimeunscaled (294 samples, 0.01%)
genunix`copyin_args32 (10,994 samples, 0.39%)
unix`tsc_read (3,248 samples, 0.12%)
zfs`dmu_read_uio_dnode (2,436,315 samples, 87.25%)
zfs`dmu_read_uio_dnode
unix`mutex_exit (936 samples, 0.03%)
zfs`arc_buf_add_ref (1,331,740 samples, 47.69%)
zfs`arc_buf_add_ref
zfs`dbuf_read (53,579 samples, 1.92%)
z..
genunix`list_remove (8,546 samples, 0.31%)
unix`mutex_adaptive_tryenter (13,998 samples, 0.50%)
unix`clock_tick_process (415 samples, 0.01%)
genunix`gethrtime_unscaled (4,045 samples, 0.14%)
unix`idle (11,255 samples, 0.40%)
genunix`turnstile_lookup (570 samples, 0.02%)
unix`mutex_vector_exit (374 samples, 0.01%)
genunix`audit_update_context (470 samples, 0.02%)
zfs`dbuf_update_data (1,245 samples, 0.04%)
unix`tsc_gethrtime (660 samples, 0.02%)
unix`mutex_delay_default (523,760 samples, 18.76%)
unix`mutex_delay_default
unix`default_lock_delay (160,565 samples, 5.75%)
unix`de..
unix`audit_start (485 samples, 0.02%)
unix`rw_exit (1,410 samples, 0.05%)
genunix`fs_rwlock (715 samples, 0.03%)
all (2,792,247 samples, 100%)
unix`bcopy_ck_size (309 samples, 0.01%)
genunix`clear_stale_fd (1,100 samples, 0.04%)
zfs`dmu_buf_rele_array (850,358 samples, 30.45%)
zfs`dmu_buf_rele_array
unix`atomic_add_64 (21,686 samples, 0.78%)
unix`splr (11,006 samples, 0.39%)
zfs`dmu_zfetch_stream_reclaim (9,204 samples, 0.33%)
unix`do_splx (557 samples, 0.02%)
unix`copyin (583 samples, 0.02%)
unix`switch_sp_and_call (1,561 samples, 0.06%)
unix`mutex_enter (383 samples, 0.01%)
unix`0xfffffffffb800ecb (7,414 samples, 0.27%)
genunix`gethrestime (2,210 samples, 0.08%)
unix`disp_anywork (804 samples, 0.03%)
zfs`zrl_add (440 samples, 0.02%)
genunix`lbolt_cyclic_driven (355 samples, 0.01%)
unix`lock_try (2,655 samples, 0.10%)
unix`mutex_exit (748 samples, 0.03%)
unix`membar_enter (457 samples, 0.02%)
genunix`ddi_get_lbolt (373 samples, 0.01%)
unix`mutex_owner_running (8,046 samples, 0.29%)
genunix`fop_rwlock (399 samples, 0.01%)
genunix`disp_lock_enter (548 samples, 0.02%)
unix`atomic_cas_32 (1,774 samples, 0.06%)
genunix`post_syscall (73,297 samples, 2.63%)
ge..
unix`mutex_owner_running (5,100 samples, 0.18%)
unix`bcopy (401 samples, 0.01%)
unix`cbe_softclock (520 samples, 0.02%)
c2audit`audit_start (405 samples, 0.01%)
zfs`zfs_tstamp_update_setup (3,153 samples, 0.11%)
zfs`dmu_zfetch_colinear (10,829 samples, 0.39%)
unix`ddi_io_put8 (322 samples, 0.01%)
unix`mutex_delay_default (509,190 samples, 18.24%)
unix`mutex_delay_default
unix`lwp_getdatamodel (504 samples, 0.02%)
unix`disp_getwork (1,058 samples, 0.04%)
genunix`clock_highres_gettime (880 samples, 0.03%)
genunix`taskq_thread (255 samples, 0.01%)
unix`mutex_vector_enter (702,996 samples, 25.18%)
unix`mutex_vector_enter
zfs`zio_wait_for_children (322 samples, 0.01%)
unix`tsc_read (1,294 samples, 0.05%)
genunix`pre_syscall (32,670 samples, 1.17%)
genunix`kmem_cache_free (503 samples, 0.02%)
unix`mutex_adaptive_tryenter (13,297 samples, 0.48%)
zfs`vdev_stat_update (340 samples, 0.01%)
unix`tsc_read (3,652 samples, 0.13%)
genunix`clock_gettime (277 samples, 0.01%)
genunix`hrt2ts (1,817 samples, 0.07%)
zfs`arc_buf_remove_ref (840,103 samples, 30.09%)
zfs`arc_buf_remove_ref
genunix`getf (262 samples, 0.01%)
unix`mutex_enter (14,843 samples, 0.53%)
unix`tsc_read (1,203 samples, 0.04%)
unix`bcopy (1,542 samples, 0.06%)
genunix`kmem_cache_alloc (391 samples, 0.01%)
unix`_sys_sysenter_post_swapgs (2,577,333 samples, 92.30%)
unix`_sys_sysenter_post_swapgs
zfs`arc_access (368 samples, 0.01%)
unix`mutex_enter (1,014 samples, 0.04%)
genunix`avl_insert (347 samples, 0.01%)
unix`get_udatamodel (365 samples, 0.01%)
unix`get_udatamodel (240 samples, 0.01%)
c2audit`auditme (1,800 samples, 0.06%)
genunix`syscall_exit (411 samples, 0.01%)
unix`splr (1,341 samples, 0.05%)
zfs`dmu_buf_hold_array_by_dnode (1,474,600 samples, 52.81%)
zfs`dmu_buf_hold_array_by_dnode
unix`tsc_gethrtimeunscaled (413 samples, 0.01%)
genunix`uiomove (105,991 samples, 3.80%)
genu..
unix`bcopy (316 samples, 0.01%)
unix`copyin (327 samples, 0.01%)
unix`mutex_vector_enter (724,998 samples, 25.96%)
unix`mutex_vector_enter
unix`default_lock_backoff (301 samples, 0.01%)
unix`mutex_delay_default (308 samples, 0.01%)
zfs`arc_access (608 samples, 0.02%)
unix`mutex_enter (4,569 samples, 0.16%)
genunix`syscall_exit (77,855 samples, 2.79%)
ge..
unix`tsc_read (6,732 samples, 0.24%)
genunix`gethrtime_unscaled (3,603 samples, 0.13%)
zfs`dmu_zfetch (34,285 samples, 1.23%)
genunix`uiomove (251 samples, 0.01%)
unix`switch_sp_and_call (375 samples, 0.01%)
genunix`watch_disable_addr (1,347 samples, 0.05%)
unix`mutex_exit (443 samples, 0.02%)
genunix`avl_numnodes (283 samples, 0.01%)
unix`mutex_enter (1,096 samples, 0.04%)
unix`mutex_enter (1,162 samples, 0.04%)
unix`atomic_add_64 (17,562 samples, 0.63%)
zfs`dbuf_rele_and_unlock (846,241 samples, 30.31%)
zfs`dbuf_rele_and_unlock
genunix`set_active_fd (302 samples, 0.01%)
genunix`set_active_fd (844 samples, 0.03%)
unix`mutex_enter (640 samples, 0.02%)
unix`default_lock_backoff (320 samples, 0.01%)
genunix`list_next (970 samples, 0.03%)
zfs`arc_released (352 samples, 0.01%)
genunix`pre_syscall (451 samples, 0.02%)
unix`dispatch_hilevel (375 samples, 0.01%)
genunix`syscall_entry (275 samples, 0.01%)
zfs`remove_reference (729,523 samples, 26.13%)
zfs`remove_reference
genunix`lbolt_cyclic_driven (464 samples, 0.02%)
genunix`audit_update_context (4,026 samples, 0.14%)
zfs`buf_hash (3,280 samples, 0.12%)
genunix`clock_tick (325 samples, 0.01%)
unix`mutex_vector_exit (380 samples, 0.01%)
genunix`disp_lock_enter (619 samples, 0.02%)
genunix`turnstile_lookup (649 samples, 0.02%)
zfs`add_reference (754,506 samples, 27.02%)
zfs`add_reference
zfs`dbuf_find (32,297 samples, 1.16%)
genunix`crgetauinfo (282 samples, 0.01%)
unix`atomic_add_64 (3,167 samples, 0.11%)
genunix`audit_getstate (506 samples, 0.02%)
unix`mutex_vector_enter (513,764 samples, 18.40%)
unix`mutex_vector_enter
genunix`kmem_cache_alloc (279 samples, 0.01%)
genunix`fop_rwlock (5,681 samples, 0.20%)
unix`mutex_enter (22,062 samples, 0.79%)
unix`mutex_vector_enter (72,212 samples, 2.59%)
un..
genunix`crgetauinfo (313 samples, 0.01%)
genunix`list_next (4,797 samples, 0.17%)
unix`mutex_enter (785 samples, 0.03%)
unix`mutex_enter (1,765 samples, 0.06%)
unix`tsc_gethrtimeunscaled (270 samples, 0.01%)
zfs`dbuf_find (885 samples, 0.03%)
unix`default_lock_delay (153,823 samples, 5.51%)
unix`de..
genunix`copyin_nowatch (8,153 samples, 0.29%)
unix`mutex_owner_running (5,186 samples, 0.19%)
unix`rw_enter (674 samples, 0.02%)
genunix`clock_gettime (11,324 samples, 0.41%)
genunix`clock (463 samples, 0.02%)
zfs`zfs_range_unlock_reader (2,828 samples, 0.10%)
unix`atomic_cas_64 (299 samples, 0.01%)
genunix`save_syscall_args (799 samples, 0.03%)
unix`mutex_vector_exit (344 samples, 0.01%)
unix`mutex_enter (1,070 samples, 0.04%)
unix`mutex_destroy (256 samples, 0.01%)
unix`tsc_read (2,690 samples, 0.10%)
unix`clear_int_flag (263 samples, 0.01%)
genunix`kmem_alloc (1,733 samples, 0.06%)
genunix`list_insert_head (6,650 samples, 0.24%)
unix`mutex_enter (715 samples, 0.03%)
unix`mutex_delay_default (45,293 samples, 1.62%)
c2audit`audit_finish (2,341 samples, 0.08%)
unix`lwp_getdatamodel (355 samples, 0.01%)
genunix`kmem_free (1,928 samples, 0.07%)
unix`mutex_enter (696 samples, 0.02%)
unix`default_lock_delay (2,442 samples, 0.09%)
unix`lock_clear_splx (808 samples, 0.03%)
c2audit`audit_finish (336 samples, 0.01%)
unix`atomic_add_64_nv (404 samples, 0.01%)
unix`default_lock_delay (330 samples, 0.01%)
genunix`syscall_mstate (8,073 samples, 0.29%)
unix`mutex_enter (12,249 samples, 0.44%)
zfs`dbuf_hold_impl (1,392,223 samples, 49.86%)
zfs`dbuf_hold_impl
unix`mutex_exit (489 samples, 0.02%)
unix`kcopy (1,477 samples, 0.05%)
unix`kcopy (1,231 samples, 0.04%)
zfs`dbuf_prefetch (1,044 samples, 0.04%)
unix`mutex_delay_default (323,606 samples, 11.59%)
unix`mutex_delay_..
unix`kcopy (240 samples, 0.01%)
unix`mutex_init (260 samples, 0.01%)
unix`0xfffffffffb800ed9 (78,342 samples, 2.81%)
un..
unix`tsc_gethrtimeunscaled (708 samples, 0.03%)
zfs`zio_ready (2,191 samples, 0.08%)
unix`atomic_add_64_nv (417 samples, 0.01%)
unix`pc_gethrestime (2,074 samples, 0.07%)
asy`asysoftintr (541 samples, 0.02%)
zfs`zfs_read (2,473,305 samples, 88.58%)
zfs`zfs_read
genunix`copyin_nowatch (567 samples, 0.02%)
genunix`disp_lock_exit (789 samples, 0.03%)
unix`0xfffffffffb800ec1 (1,341 samples, 0.05%)
genunix`avl_find (414 samples, 0.01%)
zfs`dmu_zfetch_find (6,644 samples, 0.24%)
unix`tsc_gethrtimeunscaled (677 samples, 0.02%)
unix`mutex_enter (883 samples, 0.03%)
genunix`fsflush (2,989 samples, 0.11%)
unix`tsc_gethrtime (1,520 samples, 0.05%)
unix`lwp_getdatamodel (519 samples, 0.02%)
zfs`zio_root (8,185 samples, 0.29%)
genunix`kmem_zalloc (1,387 samples, 0.05%)
unix`lwp_getdatamodel (292 samples, 0.01%)
genunix`save_syscall_args (15,222 samples, 0.55%)
unix`mutex_enter (909 samples, 0.03%)
genunix`avl_numnodes (249 samples, 0.01%)
genunix`ddi_get_lbolt (543 samples, 0.02%)
unix`copyin (273 samples, 0.01%)
unix`default_lock_delay (21,603 samples, 0.77%)
unix`atomic_cas_64 (432 samples, 0.02%)
unix`0xfffffffffb853eea (103,349 samples, 3.70%)
unix..
genunix`cv_destroy (275 samples, 0.01%)
genunix`kmem_free (1,709 samples, 0.06%)
unix`mutex_enter (1,823 samples, 0.07%)
genunix`hrt2ts (595 samples, 0.02%)
zfs`dmu_zfetch_dofetch (1,059 samples, 0.04%)
genunix`list_next (556 samples, 0.02%)
unix`atomic_add_64_nv (370 samples, 0.01%)
genunix`clock_highres_gettime (7,864 samples, 0.28%)
genunix`kmem_cache_free (834 samples, 0.03%)
genunix`watch_disable_addr (670 samples, 0.02%)
genunix`pread64 (2,493,865 samples, 89.31%)
genunix`pread64
unix`_interrupt (378 samples, 0.01%)
unix`av_dispatch_softvect (1,086 samples, 0.04%)
zfs`rrm_exit (5,652 samples, 0.20%)
genunix`kmem_cache_free (565 samples, 0.02%)
unix`copyout (271 samples, 0.01%)
zfs`rrw_enter_read (424 samples, 0.02%)
unix`membar_enter (451 samples, 0.02%)
zfs`rrw_exit (453 samples, 0.02%)
unix`tsc_read (857 samples, 0.03%)
genunix`copyin_args32 (292 samples, 0.01%)
zfs`dbuf_hold (340 samples, 0.01%)
unix`mutex_exit (530 samples, 0.02%)
asy`async_softint (538 samples, 0.02%)
unix`mutex_enter (854 samples, 0.03%)
unix`tsc_read (27,134 samples, 0.97%)
unix`av_dispatch_autovect (326 samples, 0.01%)
asy`async_nstart (387 samples, 0.01%)
genunix`ioctl (247 samples, 0.01%)
genunix`kmem_cache_alloc (291 samples, 0.01%)
unix`atomic_cas_64 (451 samples, 0.02%)
unix`copyout (437 samples, 0.02%)
unix`tsc_gethrtimeunscaled (274 samples, 0.01%)
unix`atomic_cas_32 (1,554 samples, 0.06%)
unix`do_interrupt (378 samples, 0.01%)
unix`mach_cpu_idle (8,734 samples, 0.31%)
genunix`syscall_mstate (15,520 samples, 0.56%)
genunix`syscall_mstate (7,212 samples, 0.26%)
unix`bzero (2,393 samples, 0.09%)
unix`mutex_enter (4,203 samples, 0.15%)
unix`rw_tryenter (320 samples, 0.01%)
unix`clear_int_flag (297 samples, 0.01%)
unix`kcopy (249 samples, 0.01%)
zfs`zio_create (4,466 samples, 0.16%)
unix`lwp_getdatamodel (789 samples, 0.03%)
unix`atomic_cas_64 (332 samples, 0.01%)
zfs`dmu_read_uio_dbuf (2,444,377 samples, 87.54%)
zfs`dmu_read_uio_dbuf
unix`do_splx (39,714 samples, 1.42%)
genunix`gethrtime (1,562 samples, 0.06%)
genunix`fop_read (2,477,219 samples, 88.72%)
genunix`fop_read
zfs`zio_wait_for_children (584 samples, 0.02%)
unix`default_lock_delay (165,657 samples, 5.93%)
unix`de..
zfs`rrm_enter_read (5,033 samples, 0.18%)
genunix`releasef (596 samples, 0.02%)
genunix`avl_add (902 samples, 0.03%)
genunix`copyin_nowatch (3,524 samples, 0.13%)
zfs`zrl_add (2,574 samples, 0.09%)
unix`page_nextn (271 samples, 0.01%)
zfs`dbuf_hold (1,393,319 samples, 49.90%)
zfs`dbuf_hold
genunix`gethrtime_unscaled (7,588 samples, 0.27%)
genunix`au_zone_getstate (414 samples, 0.01%)
genunix`syscall_mstate (1,218 samples, 0.04%)
genunix`au_zone_getstate (924 samples, 0.03%)
genunix`fop_rwunlock (3,503 samples, 0.13%)
genunix`list_create (338 samples, 0.01%)
zfs`zio_wait (14,411 samples, 0.52%)
genunix`list_next (3,278 samples, 0.12%)
genunix`watch_disable_addr (1,938 samples, 0.07%)
unix`mutex_adaptive_tryenter (377 samples, 0.01%)
unix`lock_set_spl_spin (440 samples, 0.02%)
zfs`zio_wait_for_children (282 samples, 0.01%)
genunix`gethrtime (4,984 samples, 0.18%)
unix`dispatch_hardint (256 samples, 0.01%)
zfs`buf_hash (2,070 samples, 0.07%)
unix`rw_exit (951 samples, 0.03%)
genunix`thread_lock (13,235 samples, 0.47%)
genunix`post_syscall (309 samples, 0.01%)
genunix`syscall_entry (43,198 samples, 1.55%)
unix`atomic_add_64_nv (1,997 samples, 0.07%)
unix`atomic_dec_32_nv (703 samples, 0.03%)
unix`dispatch_softint (1,305 samples, 0.05%)
zfs`zrl_add (1,392 samples, 0.05%)
unix`sys_syscall (380 samples, 0.01%)
genunix`avl_remove (441 samples, 0.02%)
genunix`getf (1,546 samples, 0.06%)
unix`rw_tryenter (369 samples, 0.01%)
zfs`zio_execute (10,975 samples, 0.39%)
unix`do_splx (380 samples, 0.01%)
unix`cpu_idle (9,999 samples, 0.36%)
zfs`dbuf_whichblock (281 samples, 0.01%)
unix`mutex_exit (906 samples, 0.03%)
unix`mutex_exit (293 samples, 0.01%)
unix`clock_tick_schedule (438 samples, 0.02%)
zfs`zio_done (5,498 samples, 0.20%)
genunix`copyin_args32 (5,679 samples, 0.20%)
genunix`save_syscall_args (259 samples, 0.01%)
unix`lwp_getdatamodel (743 samples, 0.03%)
genunix`clear_stale_fd (688 samples, 0.02%)
unix`lock_set_spl_spin (321 samples, 0.01%)
zfs`dmu_zfetch_fetch (1,056 samples, 0.04%)
unix`clock_tick_execute_common (438 samples, 0.02%)
genunix`thread_lock (480 samples, 0.02%)
asy`async_start (531 samples, 0.02%)
unix`prunstop (661 samples, 0.02%)
zfs`dbuf_hold_impl (351 samples, 0.01%)
zfs`dbuf_find (425 samples, 0.02%)
c2audit`audit_start (11,261 samples, 0.40%)
zfs`dbuf_rele (847,105 samples, 30.34%)
zfs`dbuf_rele
zfs`zfs_range_lock (3,325 samples, 0.12%)
zfs`dbuf_hash (1,419 samples, 0.05%)
genunix`cyclic_softint (517 samples, 0.02%)
genunix`nbl_need_check (298 samples, 0.01%)
unix`mutex_enter (2,784 samples, 0.10%)
unix`atomic_dec_32_nv (2,772 samples, 0.10%)
unix`tsc_gethrtime (518 samples, 0.02%)
unix`thread_start (14,626 samples, 0.52%)