From 787187dceca599c5b9c3a6f63fb45198f8dc3e35 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 28 Apr 2021 11:55:54 +1000 Subject: [PATCH 01/65] Add InSpace/InOldSpace checks --- v8/third_party/heap/mmtk/mmtk.cc | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index dc3babc..c660c8b 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -117,6 +117,7 @@ AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, Alloc int space = (type == AllocationType::kCode) ? 3 : (type == AllocationType::kReadOnly) ? 4 : (large_object) ? 2 : 0; Address result = reinterpret_cast
(alloc(tph_mutator_, size_in_bytes, align_bytes, 0, space)); + if (type == AllocationType::kCode && large_object) space = 5; tph_archive_insert(tph_data_->archive(), reinterpret_cast(result), tph_data_->isolate(), uint8_t(space)); HeapObject rtn = HeapObject::FromAddress(result); return rtn; @@ -137,6 +138,40 @@ bool Heap::CollectGarbage() { return true; } +bool Heap::InSpace(Address address, AllocationSpace space) { + switch (space) { + case OLD_SPACE: return InOldSpace(address); + case CODE_SPACE: return InCodeSpace(address); + case LO_SPACE: return InLargeObjectSpace(address); + case CODE_LO_SPACE: + for (size_t i = 0; i < tph_data_list->size(); i++) { + TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); + uint8_t space = tph_archive_obj_to_space( + tph_data_->archive(), reinterpret_cast(address)); + if (space == 255) continue; + if (space == 5) return true; + else return false; + } + UNREACHABLE(); + case RO_SPACE: return InReadOnlySpace(address); + default: return false; + } +} + +bool Heap::InOldSpace(Address address) { + for (size_t i = 0; i < tph_data_list->size(); i++) + { + TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); + uint8_t space = tph_archive_obj_to_space( + tph_data_->archive(), reinterpret_cast(address)); + if (space == 255) continue; + if (space == 0) return true; + else return false; + } + UNREACHABLE(); +} + + bool Heap::InCodeSpace(Address address) { for (size_t i = 0; i < tph_data_list->size(); i++) { From 07f9eb412d00943e0e5e35da1161ede61ece72e6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 28 Apr 2021 14:24:25 +1000 Subject: [PATCH 02/65] Fix space check --- v8/third_party/heap/mmtk/mmtk.cc | 80 +++++++++----------------------- 1 file changed, 21 insertions(+), 59 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index c660c8b..f67dd99 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -117,8 +117,15 @@ AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, Alloc int space = (type == AllocationType::kCode) ? 3 : (type == AllocationType::kReadOnly) ? 4 : (large_object) ? 2 : 0; Address result = reinterpret_cast
(alloc(tph_mutator_, size_in_bytes, align_bytes, 0, space)); - if (type == AllocationType::kCode && large_object) space = 5; - tph_archive_insert(tph_data_->archive(), reinterpret_cast(result), tph_data_->isolate(), uint8_t(space)); + AllocationSpace allocation_space; + if (type == AllocationType::kCode) { + allocation_space = large_object ? CODE_LO_SPACE : CODE_SPACE; + } else if (type == AllocationType::kReadOnly) { + allocation_space = RO_SPACE; + } else { + allocation_space = large_object ? LO_SPACE : OLD_SPACE; + } + tph_archive_insert(tph_data_->archive(), reinterpret_cast(result), tph_data_->isolate(), uint8_t(allocation_space)); HeapObject rtn = HeapObject::FromAddress(result); return rtn; } @@ -138,77 +145,32 @@ bool Heap::CollectGarbage() { return true; } -bool Heap::InSpace(Address address, AllocationSpace space) { - switch (space) { - case OLD_SPACE: return InOldSpace(address); - case CODE_SPACE: return InCodeSpace(address); - case LO_SPACE: return InLargeObjectSpace(address); - case CODE_LO_SPACE: - for (size_t i = 0; i < tph_data_list->size(); i++) { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - uint8_t space = tph_archive_obj_to_space( - tph_data_->archive(), reinterpret_cast(address)); - if (space == 255) continue; - if (space == 5) return true; - else return false; - } - UNREACHABLE(); - case RO_SPACE: return InReadOnlySpace(address); - default: return false; +constexpr AllocationSpace kNoSpace = AllocationSpace(255); + +bool Heap::InSpace(Address address, AllocationSpace allocation_space) { + for (auto tph_data : *tph_data_list) { + auto space = AllocationSpace(tph_archive_obj_to_space(tph_data->archive(), reinterpret_cast(address))); + if (space == kNoSpace) continue; + return space == allocation_space; } + UNREACHABLE(); } bool Heap::InOldSpace(Address address) { - for (size_t i = 0; i < tph_data_list->size(); i++) - { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - uint8_t space = tph_archive_obj_to_space( - tph_data_->archive(), reinterpret_cast(address)); - if (space == 255) continue; - if (space == 0) return true; - else return false; - } - UNREACHABLE(); + return InSpace(address, OLD_SPACE); } bool Heap::InCodeSpace(Address address) { - for (size_t i = 0; i < tph_data_list->size(); i++) - { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - uint8_t space = tph_archive_obj_to_space( - tph_data_->archive(), reinterpret_cast(address)); - if (space == 255) continue; - if (space == 3) return true; - else return false; - } - UNREACHABLE(); + return InSpace(address, CODE_SPACE); } bool Heap::InReadOnlySpace(Address address) { - for (size_t i = 0; i < tph_data_list->size(); i++) - { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - uint8_t space = tph_archive_obj_to_space( - tph_data_->archive(), reinterpret_cast(address)); - if (space == 255) continue; - if (space == 4) return true; - else return false; - } - UNREACHABLE(); + return InSpace(address, RO_SPACE); } bool Heap::InLargeObjectSpace(Address address) { - for (size_t i = 0; i < tph_data_list->size(); i++) - { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - uint8_t space = tph_archive_obj_to_space( - tph_data_->archive(), reinterpret_cast(address)); - if (space == 255) continue; - if (space == 2) return true; - else return false; - } - UNREACHABLE(); + return InSpace(address, LO_SPACE); } bool Heap::IsValidHeapObject(HeapObject object) { From f028b8d37f8302a17a13b7c658aaa0620f8dd430 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 30 Apr 2021 19:58:26 +1000 Subject: [PATCH 03/65] Paging-GC can do one collection and pass mk-snapshot now --- mmtk/Cargo.toml | 6 +- mmtk/src/api.rs | 25 +++- mmtk/src/collection.rs | 15 +- mmtk/src/lib.rs | 16 ++- mmtk/src/object_model.rs | 4 +- mmtk/src/scanning.rs | 48 ++++++- v8/third_party/heap/mmtk/mmtk.cc | 31 ++--- v8/third_party/heap/mmtk/mmtk.h | 42 ++++++ v8/third_party/heap/mmtk/mmtkUpcalls.cc | 175 +++++++++++++++++++++++- 9 files changed, 314 insertions(+), 48 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 910abd9..b8d2822 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -8,7 +8,11 @@ name = "mmtk_v8" # be careful - LTO is only allowed for certain crate types crate-type = ["cdylib"] +[profile.dev] +panic = "abort" + [profile.release] +panic = "abort" lto = true [dependencies] @@ -27,5 +31,5 @@ mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "c2132e36c0f0db188 # mmtk = { path = "../repos/mmtk-core" } [features] -default = ["mmtk/code_space", "mmtk/ro_space"] +default = ["mmtk/code_space", "mmtk/ro_space", "mmtk/side_gc_header"] nogc = [] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 7464c36..00acf55 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -6,20 +6,26 @@ use mmtk::util::{Address, ObjectReference, OpaquePointer}; use mmtk::AllocationSemantics; use mmtk::Mutator; use mmtk::MMTK; +use mmtk::policy::space::Space; use std::ffi::CStr; use V8_Upcalls; use UPCALLS; use V8; +#[no_mangle] +pub unsafe extern "C" fn release_buffer(ptr: *mut Address, length: usize, capacity: usize) { + let _vec = Vec::
::from_raw_parts(ptr, length, capacity); +} + #[no_mangle] pub extern "C" fn v8_new_heap(calls: *const V8_Upcalls, heap_size: usize) -> *mut c_void { unsafe { UPCALLS = calls; }; - let mmtk: Box> = Box::new(MMTK::new()); - let mmtk: *mut MMTK = Box::into_raw(mmtk); - memory_manager::gc_init(unsafe { &mut *mmtk }, heap_size); + let mmtk: *const MMTK = &*crate::SINGLETON; + memory_manager::gc_init(unsafe { &mut *(mmtk as *mut MMTK) }, heap_size); + enable_collection(unsafe { &mut *(mmtk as *mut MMTK) }, OpaquePointer::UNINITIALIZED); mmtk as *mut c_void } @@ -37,6 +43,15 @@ pub extern "C" fn bind_mutator( Box::into_raw(memory_manager::bind_mutator(mmtk, tls)) } +#[no_mangle] +pub unsafe extern "C" fn mmtk_in_space(mmtk: &'static MMTK, object: ObjectReference, space: AllocationSemantics) -> i32 { + let unsync = &*mmtk.plan.base().unsync.get(); + match space { + AllocationSemantics::ReadOnly => unsync.ro_space.in_space(object) as _, + _ => unreachable!() + } +} + #[no_mangle] // It is fine we turn the pointer back to box, as we turned a boxed value to the raw pointer in bind_mutator() #[allow(clippy::not_unsafe_ptr_arg_deref)] @@ -52,7 +67,9 @@ pub extern "C" fn alloc( offset: isize, semantics: AllocationSemantics, ) -> Address { - memory_manager::alloc::(mutator, size, align, offset, semantics) + let a = memory_manager::alloc::(mutator, size, align, offset, semantics); + unsafe { memory_manager::post_alloc::(mutator, a.to_object_reference(), size, semantics); } + a } #[no_mangle] diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index e6e470c..3a13c78 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -2,7 +2,7 @@ use mmtk::scheduler::gc_work::ProcessEdgesWork; use mmtk::scheduler::GCWorker; use mmtk::util::OpaquePointer; use mmtk::vm::Collection; -use mmtk::MutatorContext; +use mmtk::{MutatorContext, MMTK}; use UPCALLS; use V8; @@ -33,10 +33,15 @@ impl Collection for VMCollection { r as *const GCWorker as *mut GCWorker } else { std::ptr::null_mut() - }; - unsafe { - ((*UPCALLS).spawn_worker_thread)(tls, ctx_ptr as usize as _); - } + } as usize; + std::thread::spawn(move || { + let mmtk: *mut MMTK = &*crate::SINGLETON as *const MMTK as *mut MMTK; + if ctx_ptr == 0 { + crate::api::start_control_collector(unsafe { &mut *mmtk }, OpaquePointer::UNINITIALIZED); + } else { + crate::api::start_worker(unsafe { &mut *mmtk }, OpaquePointer::UNINITIALIZED, unsafe { &mut *(ctx_ptr as *mut GCWorker) }); + } + }); } fn prepare_mutator>(_tls: OpaquePointer, _m: &T) { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index f9bbd99..1aa5516 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(vec_into_raw_parts)] + extern crate libc; extern crate mmtk; #[macro_use] @@ -22,6 +24,15 @@ mod object_archive; pub mod object_model; pub mod reference_glue; pub mod scanning; +use mmtk::util::{Address}; + +#[repr(C)] +pub struct NewBuffer { + pub ptr: *mut Address, + pub capacity: usize, +} + +type ProcessEdgesFn = *const extern "C" fn(buf: *mut Address, size: usize, cap: usize) -> NewBuffer; #[repr(C)] pub struct V8_Upcalls { @@ -40,6 +51,8 @@ pub struct V8_Upcalls { pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, pub get_mmtk_mutator: extern "C" fn(tls: OpaquePointer) -> *mut Mutator, pub is_mutator: extern "C" fn(tls: OpaquePointer) -> bool, + pub scan_roots: extern "C" fn(process_edges: ProcessEdgesFn), + pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn), } pub static mut UPCALLS: *const V8_Upcalls = null_mut(); @@ -59,9 +72,6 @@ impl VMBinding for V8 { lazy_static! { pub static ref SINGLETON: MMTK = { - #[cfg(feature = "nogc")] - std::env::set_var("MMTK_PLAN", "NoGC"); - MMTK::new() }; } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 7b5b655..a3a3f8c 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -44,7 +44,7 @@ impl ObjectModel for VMObjectModel { } } - fn ref_to_address(_object: ObjectReference) -> Address { - unimplemented!() + fn ref_to_address(object: ObjectReference) -> Address { + unsafe { Address::from_usize(object.to_address().as_usize() & !0b1) } } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 1443b35..33f1fdd 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -5,6 +5,11 @@ use mmtk::util::OpaquePointer; use mmtk::vm::Scanning; use mmtk::{Mutator, TransitiveClosure}; use V8; +use mmtk::scheduler::gc_work::*; +use mmtk::scheduler::*; +use crate::*; + +use std::marker::PhantomData; pub struct VMScanning {} @@ -25,10 +30,14 @@ impl Scanning for VMScanning { } fn scan_objects>( - _objects: &[ObjectReference], - _worker: &mut GCWorker, + objects: &[ObjectReference], + worker: &mut GCWorker, ) { - unimplemented!() + unsafe { + let buf = objects.as_ptr(); + let len = objects.len(); + ((*UPCALLS).scan_objects)(buf, len, create_process_edges_work:: as _); + } } fn scan_thread_roots>() { @@ -43,10 +52,41 @@ impl Scanning for VMScanning { } fn scan_vm_specific_roots>() { - unimplemented!() + SINGLETON.scheduler.work_buckets[WorkBucketStage::Prepare] + .add(ScanRoots::::new()); } fn supports_return_barrier() -> bool { unimplemented!() } } + +pub struct ScanRoots>(PhantomData); + +impl> ScanRoots { + pub fn new() -> Self { + Self(PhantomData) + } +} + +impl> GCWork for ScanRoots { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + unsafe { + ((*UPCALLS).scan_roots)(create_process_edges_work:: as _); + } + } +} + +pub(crate) extern "C" fn create_process_edges_work>( + ptr: *mut Address, + length: usize, + capacity: usize, +) -> NewBuffer { + if !ptr.is_null() { + let buf = unsafe { Vec::
::from_raw_parts(ptr, length, capacity) }; + SINGLETON.scheduler.work_buckets[WorkBucketStage::Closure] + .add(W::new(buf, false, &SINGLETON)); + } + let (ptr, _, capacity) = Vec::with_capacity(W::CAPACITY).into_raw_parts(); + NewBuffer { ptr, capacity } +} \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index f67dd99..5c21f48 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -8,30 +8,11 @@ namespace v8 { namespace internal { namespace third_party_heap { -class TPHData { - Heap* v8_tph_; - MMTk_Heap mmtk_heap_; - v8::internal::Isolate* isolate_; - MMTk_Heap_Archive tph_archive_; - - public: - Heap* v8_tph() { return v8_tph_; } - MMTk_Heap mmtk_heap() { return mmtk_heap_; } - v8::internal::Isolate * isolate() { return isolate_; } - MMTk_Heap_Archive archive() { return tph_archive_; } - - TPHData(Heap* v8_tph, MMTk_Heap mmtk_heap, Isolate* isolate, MMTk_Heap_Archive tph_archive): - v8_tph_(v8_tph), mmtk_heap_(mmtk_heap), isolate_(isolate), tph_archive_(tph_archive) {} -}; + // Data structure required for Rust-MMTK -class BumpAllocator { - public: - TPHData* tph_data; - uintptr_t cursor; - uintptr_t limit; - void* space; -}; + +v8::internal::Heap* v8_heap = nullptr; base::AddressRegion code_range_; @@ -53,12 +34,15 @@ TPHData* get_tph_data(Heap* tph) { UNREACHABLE(); } +std::vector* all_mutators = new std::vector(); + inline void CheckMutator(Heap* tph) { TPHData* tph_data_ = get_tph_data(tph); if (tph_mutator_ == nullptr) { tph_mutator_ = reinterpret_cast( bind_mutator(tph_data_->mmtk_heap(), &tph_mutator_)); tph_mutator_->tph_data = tph_data_; + all_mutators->push_back(tph_mutator_); } } @@ -78,6 +62,8 @@ MMTk_Heap GetMMTkHeap(Address object_pointer) { static std::atomic_bool IsolateCreated { false }; std::unique_ptr Heap::New(v8::internal::Isolate* isolate) { + DCHECK(!v8_heap); + v8_heap = isolate->heap(); // MMTK current default maximum heap size is 1GB. auto isolate_created = IsolateCreated.exchange(true); DCHECK_WITH_MSG(!isolate_created, "Multiple isolates are not supported."); @@ -142,6 +128,7 @@ const v8::base::AddressRegion& Heap::GetCodeRange() { } bool Heap::CollectGarbage() { + handle_user_collection_request(get_tph_data(this)->mmtk_heap(), (void*) 0); return true; } diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 19e49e0..54ba0a1 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -20,6 +20,36 @@ typedef void* MMTk_TraceLocal; typedef void* MMTk_Heap; typedef void* MMTk_Heap_Archive; + +namespace v8 { +namespace internal { + class Isolate; +namespace third_party_heap { + class Heap; +class TPHData { + Heap* v8_tph_; + MMTk_Heap mmtk_heap_; + v8::internal::Isolate* isolate_; + MMTk_Heap_Archive tph_archive_; + + public: + Heap* v8_tph() { return v8_tph_; } + MMTk_Heap mmtk_heap() { return mmtk_heap_; } + v8::internal::Isolate * isolate() { return isolate_; } + MMTk_Heap_Archive archive() { return tph_archive_; } + + TPHData(Heap* v8_tph, MMTk_Heap mmtk_heap, Isolate* isolate, MMTk_Heap_Archive tph_archive): + v8_tph_(v8_tph), mmtk_heap_(mmtk_heap), isolate_(isolate), tph_archive_(tph_archive) {} +}; +class BumpAllocator { + public: + TPHData* tph_data; + uintptr_t cursor; + uintptr_t limit; + void* space; +}; + +}}} /** * Allocation */ @@ -69,6 +99,16 @@ extern void* tph_archive_iter_next(void* arch); extern void* tph_archive_inner_to_obj(void* arch, void* inner_ptr); extern void* tph_archive_obj_to_isolate(void* arch, void* obj_ptr); extern uint8_t tph_archive_obj_to_space(void* arch, void* obj_ptr); +extern int mmtk_in_space(void* mmtk, void* object, size_t space); + +extern void release_buffer(void** buffer, size_t len, size_t cap); + +typedef struct { + void** buf; + size_t cap; +} NewBuffer; + +typedef NewBuffer (*ProcessEdgesFn)(void** buf, size_t len, size_t cap); typedef struct { void (*stop_all_mutators) (void *tls); @@ -86,6 +126,8 @@ typedef struct { size_t (*get_object_size) (void* object); void* (*get_mmtk_mutator) (void* tls); bool (*is_mutator) (void* tls); + void (*scan_roots) (ProcessEdgesFn process_edges); + void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges); } V8_Upcalls; /** diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 7ce4a25..75c2e19 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -1,25 +1,44 @@ #include "src/base/logging.h" #include "mmtkUpcalls.h" - +#include +#include +#include "src/objects/slots-inl.h" +#include "src/heap/safepoint.h" namespace v8 { namespace internal { namespace third_party_heap { +extern v8::internal::Heap* v8_heap; + +std::mutex m; +std::condition_variable* cv = new std::condition_variable(); +bool gcInProgress = false; +SafepointScope* scope; + static void mmtk_stop_all_mutators(void *tls) { - UNIMPLEMENTED(); + printf("mmtk_stop_all_mutators\n"); + scope = new SafepointScope(v8_heap); } static void mmtk_resume_mutators(void *tls) { - UNIMPLEMENTED(); + printf("mmtk_resume_mutators\n"); + delete scope; + scope = nullptr; + std::unique_lock lock(m); + gcInProgress = false; + cv->notify_all(); } static void mmtk_spawn_collector_thread(void* tls, void* ctx) { UNIMPLEMENTED(); } + static void mmtk_block_for_gc() { - UNIMPLEMENTED(); + gcInProgress = true; + std::unique_lock lock(m); + cv->wait(lock, []{ return !gcInProgress; }); } static void* mmtk_active_collector(void* tls) { @@ -34,12 +53,16 @@ static bool mmtk_is_mutator(void* tls) { return false; } +extern std::vector* all_mutators; +size_t index = 0; + static void* mmtk_get_next_mutator() { - UNIMPLEMENTED(); + if (index >= all_mutators->size()) return nullptr; + return (*all_mutators)[index++]; } static void mmtk_reset_mutator_iterator() { - UNIMPLEMENTED(); + index = 0; } @@ -67,6 +90,142 @@ static size_t mmtk_get_object_size(void* object) { UNIMPLEMENTED(); } +class MMTkRootVisitor: public RootVisitor { +public: + explicit MMTkRootVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { + NewBuffer buf = process_edges(NULL, 0, 0); + buffer_ = buf.buf; + cap_ = buf.cap; + } + + virtual ~MMTkRootVisitor() { + if (cursor_ > 0) flush(); + if (buffer_ != NULL) { + release_buffer(buffer_, cursor_, cap_); + } + } + + void VisitRootPointer(Root root, const char* description, FullObjectSlot p) final { + // DCHECK(!MapWord::IsPacked(p.Relaxed_Load().ptr())); + ProcessRootEdge(root, p); + } + + void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) final { + for (FullObjectSlot p = start; p < end; ++p) { + ProcessRootEdge(root, p); + } + } + + virtual void VisitRootPointers(Root root, const char* description, + OffHeapObjectSlot start, + OffHeapObjectSlot end) { + for (OffHeapObjectSlot p = start; p < end; ++p) { + ProcessRootEdge(root, p); + } + } + + private: + V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { + if (!(*p).IsHeapObject()) return; + buffer_[cursor_++] = (void*) p.address(); + if (cursor_ >= cap_) { + flush(); + } + } + + void flush() { + if (cursor_ > 0) { + NewBuffer buf = process_edges_(buffer_, cursor_, cap_); + buffer_ = buf.buf; + cap_ = buf.cap; + cursor_ = 0; + } + } + + ProcessEdgesFn process_edges_; + void** buffer_ = nullptr; + size_t cap_ = 0; + size_t cursor_ = 0; +}; + +class MMTkObjectVisitor: public ObjectVisitor { +public: + explicit MMTkObjectVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { + NewBuffer buf = process_edges(NULL, 0, 0); + buffer_ = buf.buf; + cap_ = buf.cap; + } + + virtual ~MMTkObjectVisitor() { + if (cursor_ > 0) flush(); + if (buffer_ != NULL) { + release_buffer(buffer_, cursor_, cap_); + } + } + + void VisitPointers(HeapObject host, ObjectSlot start, + ObjectSlot end) override { + for (ObjectSlot p = start; p < end; ++p) { + ProcessEdge(p); + } + } + + void VisitPointers(HeapObject host, MaybeObjectSlot start, + MaybeObjectSlot end) override { + for (MaybeObjectSlot p = start; p < end; ++p) { + ProcessEdge(p); + } + } + + void VisitCodeTarget(Code host, RelocInfo* rinfo) final { + // Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); + // MarkHeapObject(target); + } + void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final { + // MarkHeapObject(rinfo->target_object()); + } + + void VisitMapPointer(HeapObject object) override {} + + private: + template + V8_INLINE void ProcessEdge(T p) { + if (!(*p).IsHeapObject()) return; + buffer_[cursor_++] = (void*) p.address(); + if (cursor_ >= cap_) { + flush(); + } + } + + void flush() { + if (cursor_ > 0) { + NewBuffer buf = process_edges_(buffer_, cursor_, cap_); + buffer_ = buf.buf; + cap_ = buf.cap; + cursor_ = 0; + } + } + + ProcessEdgesFn process_edges_; + void** buffer_ = nullptr; + size_t cap_ = 0; + size_t cursor_ = 0; +}; + +static void mmtk_scan_roots(ProcessEdgesFn process_edges) { + MMTkRootVisitor root_visitor(process_edges); + v8_heap->IterateRoots(&root_visitor, base::EnumSet{}); +} + +static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { + MMTkObjectVisitor visitor(process_edges); + for (size_t i = 0; i < count; i++) { + auto ptr = *(objects + i); + auto obj = HeapObject::FromAddress(((Address) ptr)); + obj.Iterate(&visitor); + } +} + V8_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -83,8 +242,10 @@ V8_Upcalls mmtk_upcalls = { mmtk_get_object_size, mmtk_get_mmtk_mutator, mmtk_is_mutator, + mmtk_scan_roots, + mmtk_scan_objects, }; } // namespace third_party_heap } // namespace internal -} // namespace v8 \ No newline at end of file +} // namespace v8 \ No newline at end of file From f4a762261acd41a134cb4be8b6b79268ca00612d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 30 Apr 2021 20:10:49 +1000 Subject: [PATCH 04/65] cleanup --- v8/third_party/heap/mmtk/mmtk.h | 43 +++++---- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 121 ++++++++++++------------ 2 files changed, 84 insertions(+), 80 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 54ba0a1..b2427d4 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -23,24 +23,29 @@ typedef void* MMTk_Heap_Archive; namespace v8 { namespace internal { - class Isolate; + +class Isolate; + namespace third_party_heap { - class Heap; + +class Heap; + class TPHData { - Heap* v8_tph_; - MMTk_Heap mmtk_heap_; - v8::internal::Isolate* isolate_; - MMTk_Heap_Archive tph_archive_; - - public: - Heap* v8_tph() { return v8_tph_; } - MMTk_Heap mmtk_heap() { return mmtk_heap_; } - v8::internal::Isolate * isolate() { return isolate_; } - MMTk_Heap_Archive archive() { return tph_archive_; } - - TPHData(Heap* v8_tph, MMTk_Heap mmtk_heap, Isolate* isolate, MMTk_Heap_Archive tph_archive): - v8_tph_(v8_tph), mmtk_heap_(mmtk_heap), isolate_(isolate), tph_archive_(tph_archive) {} + Heap* v8_tph_; + MMTk_Heap mmtk_heap_; + v8::internal::Isolate* isolate_; + MMTk_Heap_Archive tph_archive_; + + public: + Heap* v8_tph() { return v8_tph_; } + MMTk_Heap mmtk_heap() { return mmtk_heap_; } + v8::internal::Isolate * isolate() { return isolate_; } + MMTk_Heap_Archive archive() { return tph_archive_; } + + TPHData(Heap* v8_tph, MMTk_Heap mmtk_heap, Isolate* isolate, MMTk_Heap_Archive tph_archive): + v8_tph_(v8_tph), mmtk_heap_(mmtk_heap), isolate_(isolate), tph_archive_(tph_archive) {} }; + class BumpAllocator { public: TPHData* tph_data; @@ -49,7 +54,9 @@ class BumpAllocator { void* space; }; -}}} +} // third_party_heap +} // internal +} // v8 /** * Allocation */ @@ -104,8 +111,8 @@ extern int mmtk_in_space(void* mmtk, void* object, size_t space); extern void release_buffer(void** buffer, size_t len, size_t cap); typedef struct { - void** buf; - size_t cap; + void** buf; + size_t cap; } NewBuffer; typedef NewBuffer (*ProcessEdgesFn)(void** buf, size_t len, size_t cap); diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 75c2e19..a1fa677 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -17,17 +17,17 @@ bool gcInProgress = false; SafepointScope* scope; static void mmtk_stop_all_mutators(void *tls) { - printf("mmtk_stop_all_mutators\n"); - scope = new SafepointScope(v8_heap); + printf("mmtk_stop_all_mutators\n"); + scope = new SafepointScope(v8_heap); } static void mmtk_resume_mutators(void *tls) { - printf("mmtk_resume_mutators\n"); - delete scope; - scope = nullptr; - std::unique_lock lock(m); - gcInProgress = false; - cv->notify_all(); + printf("mmtk_resume_mutators\n"); + delete scope; + scope = nullptr; + std::unique_lock lock(m); + gcInProgress = false; + cv->notify_all(); } static void mmtk_spawn_collector_thread(void* tls, void* ctx) { @@ -36,9 +36,9 @@ static void mmtk_spawn_collector_thread(void* tls, void* ctx) { static void mmtk_block_for_gc() { - gcInProgress = true; - std::unique_lock lock(m); - cv->wait(lock, []{ return !gcInProgress; }); + gcInProgress = true; + std::unique_lock lock(m); + cv->wait(lock, []{ return !gcInProgress; }); } static void* mmtk_active_collector(void* tls) { @@ -57,8 +57,8 @@ extern std::vector* all_mutators; size_t index = 0; static void* mmtk_get_next_mutator() { - if (index >= all_mutators->size()) return nullptr; - return (*all_mutators)[index++]; + if (index >= all_mutators->size()) return nullptr; + return (*all_mutators)[index++]; } static void mmtk_reset_mutator_iterator() { @@ -67,33 +67,33 @@ static void mmtk_reset_mutator_iterator() { static void mmtk_compute_global_roots(void* trace, void* tls) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } static void mmtk_compute_static_roots(void* trace, void* tls) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } static void mmtk_compute_thread_roots(void* trace, void* tls) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } static void mmtk_scan_object(void* trace, void* object, void* tls) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } static void mmtk_dump_object(void* object) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } static size_t mmtk_get_object_size(void* object) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } class MMTkRootVisitor: public RootVisitor { -public: + public: explicit MMTkRootVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { - NewBuffer buf = process_edges(NULL, 0, 0); + NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; cap_ = buf.cap; } @@ -116,13 +116,11 @@ class MMTkRootVisitor: public RootVisitor { } } - virtual void VisitRootPointers(Root root, const char* description, - OffHeapObjectSlot start, - OffHeapObjectSlot end) { - for (OffHeapObjectSlot p = start; p < end; ++p) { + virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) { + for (OffHeapObjectSlot p = start; p < end; ++p) { ProcessRootEdge(root, p); } - } + } private: V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { @@ -149,9 +147,9 @@ class MMTkRootVisitor: public RootVisitor { }; class MMTkObjectVisitor: public ObjectVisitor { -public: + public: explicit MMTkObjectVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { - NewBuffer buf = process_edges(NULL, 0, 0); + NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; cap_ = buf.cap; } @@ -163,27 +161,26 @@ class MMTkObjectVisitor: public ObjectVisitor { } } - void VisitPointers(HeapObject host, ObjectSlot start, - ObjectSlot end) override { + void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override { for (ObjectSlot p = start; p < end; ++p) { ProcessEdge(p); } } - void VisitPointers(HeapObject host, MaybeObjectSlot start, - MaybeObjectSlot end) override { + void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override { for (MaybeObjectSlot p = start; p < end; ++p) { ProcessEdge(p); } } void VisitCodeTarget(Code host, RelocInfo* rinfo) final { - // Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); - // MarkHeapObject(target); - } - void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final { - // MarkHeapObject(rinfo->target_object()); - } + // Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); + // MarkHeapObject(target); + } + + void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final { + // MarkHeapObject(rinfo->target_object()); + } void VisitMapPointer(HeapObject object) override {} @@ -213,37 +210,37 @@ class MMTkObjectVisitor: public ObjectVisitor { }; static void mmtk_scan_roots(ProcessEdgesFn process_edges) { - MMTkRootVisitor root_visitor(process_edges); - v8_heap->IterateRoots(&root_visitor, base::EnumSet{}); + MMTkRootVisitor root_visitor(process_edges); + v8_heap->IterateRoots(&root_visitor, base::EnumSet{}); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { MMTkObjectVisitor visitor(process_edges); - for (size_t i = 0; i < count; i++) { - auto ptr = *(objects + i); - auto obj = HeapObject::FromAddress(((Address) ptr)); - obj.Iterate(&visitor); - } + for (size_t i = 0; i < count; i++) { + auto ptr = *(objects + i); + auto obj = HeapObject::FromAddress(((Address) ptr)); + obj.Iterate(&visitor); + } } V8_Upcalls mmtk_upcalls = { - mmtk_stop_all_mutators, - mmtk_resume_mutators, - mmtk_spawn_collector_thread, - mmtk_block_for_gc, - mmtk_active_collector, - mmtk_get_next_mutator, - mmtk_reset_mutator_iterator, - mmtk_compute_static_roots, - mmtk_compute_global_roots, - mmtk_compute_thread_roots, - mmtk_scan_object, - mmtk_dump_object, - mmtk_get_object_size, - mmtk_get_mmtk_mutator, - mmtk_is_mutator, - mmtk_scan_roots, - mmtk_scan_objects, + mmtk_stop_all_mutators, + mmtk_resume_mutators, + mmtk_spawn_collector_thread, + mmtk_block_for_gc, + mmtk_active_collector, + mmtk_get_next_mutator, + mmtk_reset_mutator_iterator, + mmtk_compute_static_roots, + mmtk_compute_global_roots, + mmtk_compute_thread_roots, + mmtk_scan_object, + mmtk_dump_object, + mmtk_get_object_size, + mmtk_get_mmtk_mutator, + mmtk_is_mutator, + mmtk_scan_roots, + mmtk_scan_objects, }; } // namespace third_party_heap From 0d1e12cf90d8bdd18b6b872095c1079086251cd8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 3 May 2021 16:23:13 +1000 Subject: [PATCH 05/65] Update visitors --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 39 +++++++++++++++++++------ 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index a1fa677..94d48f5 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -4,6 +4,14 @@ #include #include "src/objects/slots-inl.h" #include "src/heap/safepoint.h" +#include "src/codegen/reloc-info.h" +#include "src/objects/code.h" +#include "src/objects/code-inl.h" +#include "src/codegen/assembler-inl.h" +#include "src/codegen/interface-descriptors.h" +#include "src/codegen/macro-assembler-inl.h" +#include "src/codegen/macro-assembler.h" +#include "src/compiler/code-assembler.h" namespace v8 { namespace internal { @@ -39,6 +47,7 @@ static void mmtk_block_for_gc() { gcInProgress = true; std::unique_lock lock(m); cv->wait(lock, []{ return !gcInProgress; }); + printf("mmtk_block_for_gc end\n"); } static void* mmtk_active_collector(void* tls) { @@ -161,28 +170,40 @@ class MMTkObjectVisitor: public ObjectVisitor { } } - void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override { + virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override { for (ObjectSlot p = start; p < end; ++p) { ProcessEdge(p); } } - void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override { + virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override { for (MaybeObjectSlot p = start; p < end; ++p) { ProcessEdge(p); } } - void VisitCodeTarget(Code host, RelocInfo* rinfo) final { - // Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); - // MarkHeapObject(target); + virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override { + // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. + Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); + auto p = new HeapObject(target); + buffer_[cursor_++] = (void*) p; + if (cursor_ >= cap_) { + flush(); + } } - void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final { - // MarkHeapObject(rinfo->target_object()); + virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override { + // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. + auto p = new HeapObject(rinfo->target_object()); + buffer_[cursor_++] = (void*) p; + if (cursor_ >= cap_) { + flush(); + } } - void VisitMapPointer(HeapObject object) override {} + virtual void VisitMapPointer(HeapObject host) override { + ProcessEdge(host.map_slot()); + } private: template @@ -211,7 +232,7 @@ class MMTkObjectVisitor: public ObjectVisitor { static void mmtk_scan_roots(ProcessEdgesFn process_edges) { MMTkRootVisitor root_visitor(process_edges); - v8_heap->IterateRoots(&root_visitor, base::EnumSet{}); + v8_heap->IterateRoots(&root_visitor, {}); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { From 30ef6c9dbe433c4f1949c89fbd4bbe56448518d8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 4 May 2021 10:28:33 +1000 Subject: [PATCH 06/65] Refactor --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 84 ++++++------------------- 1 file changed, 18 insertions(+), 66 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 94d48f5..9c07821 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -99,90 +99,42 @@ static size_t mmtk_get_object_size(void* object) { UNIMPLEMENTED(); } -class MMTkRootVisitor: public RootVisitor { +class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { public: - explicit MMTkRootVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { + explicit MMTkEdgeVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; cap_ = buf.cap; } - virtual ~MMTkRootVisitor() { + virtual ~MMTkEdgeVisitor() { if (cursor_ > 0) flush(); if (buffer_ != NULL) { release_buffer(buffer_, cursor_, cap_); } } - void VisitRootPointer(Root root, const char* description, FullObjectSlot p) final { - // DCHECK(!MapWord::IsPacked(p.Relaxed_Load().ptr())); - ProcessRootEdge(root, p); + virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { + ProcessEdge(p); } - void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) final { - for (FullObjectSlot p = start; p < end; ++p) { - ProcessRootEdge(root, p); - } + virtual void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override final { + for (FullObjectSlot p = start; p < end; ++p) ProcessEdge(p); } - virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) { - for (OffHeapObjectSlot p = start; p < end; ++p) { - ProcessRootEdge(root, p); - } - } - - private: - V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { - if (!(*p).IsHeapObject()) return; - buffer_[cursor_++] = (void*) p.address(); - if (cursor_ >= cap_) { - flush(); - } + virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) override final { + for (OffHeapObjectSlot p = start; p < end; ++p) ProcessEdge(p); } - void flush() { - if (cursor_ > 0) { - NewBuffer buf = process_edges_(buffer_, cursor_, cap_); - buffer_ = buf.buf; - cap_ = buf.cap; - cursor_ = 0; - } + virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { + for (ObjectSlot p = start; p < end; ++p) ProcessEdge(p); } - ProcessEdgesFn process_edges_; - void** buffer_ = nullptr; - size_t cap_ = 0; - size_t cursor_ = 0; -}; - -class MMTkObjectVisitor: public ObjectVisitor { - public: - explicit MMTkObjectVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { - NewBuffer buf = process_edges(NULL, 0, 0); - buffer_ = buf.buf; - cap_ = buf.cap; - } - - virtual ~MMTkObjectVisitor() { - if (cursor_ > 0) flush(); - if (buffer_ != NULL) { - release_buffer(buffer_, cursor_, cap_); - } - } - - virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override { - for (ObjectSlot p = start; p < end; ++p) { - ProcessEdge(p); - } - } - - virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override { - for (MaybeObjectSlot p = start; p < end; ++p) { - ProcessEdge(p); - } + virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override final { + for (MaybeObjectSlot p = start; p < end; ++p) ProcessEdge(p); } - virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override { + virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); auto p = new HeapObject(target); @@ -192,7 +144,7 @@ class MMTkObjectVisitor: public ObjectVisitor { } } - virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override { + virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. auto p = new HeapObject(rinfo->target_object()); buffer_[cursor_++] = (void*) p; @@ -201,7 +153,7 @@ class MMTkObjectVisitor: public ObjectVisitor { } } - virtual void VisitMapPointer(HeapObject host) override { + virtual void VisitMapPointer(HeapObject host) override final { ProcessEdge(host.map_slot()); } @@ -231,12 +183,12 @@ class MMTkObjectVisitor: public ObjectVisitor { }; static void mmtk_scan_roots(ProcessEdgesFn process_edges) { - MMTkRootVisitor root_visitor(process_edges); + MMTkEdgeVisitor root_visitor(process_edges); v8_heap->IterateRoots(&root_visitor, {}); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { - MMTkObjectVisitor visitor(process_edges); + MMTkEdgeVisitor visitor(process_edges); for (size_t i = 0; i < count; i++) { auto ptr = *(objects + i); auto obj = HeapObject::FromAddress(((Address) ptr)); From 72bc857a8e625143eef1aaa39e2733dc22ce482f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 4 May 2021 10:40:12 +1000 Subject: [PATCH 07/65] Fix build error --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 9c07821..99856ee 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -115,15 +115,15 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { } virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { - ProcessEdge(p); + ProcessRootEdge(root, p); } virtual void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override final { - for (FullObjectSlot p = start; p < end; ++p) ProcessEdge(p); + for (FullObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); } virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) override final { - for (OffHeapObjectSlot p = start; p < end; ++p) ProcessEdge(p); + for (OffHeapObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); } virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { @@ -158,6 +158,10 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { } private: + V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { + ProcessEdge(p); + } + template V8_INLINE void ProcessEdge(T p) { if (!(*p).IsHeapObject()) return; From 58d1871d4dd41a6f9c6c652d851062faa5afd04a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 4 May 2021 10:45:05 +1000 Subject: [PATCH 08/65] Refactor --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 96 +------------------- v8/third_party/heap/mmtk/mmtkVisitors.h | 113 ++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 95 deletions(-) create mode 100644 v8/third_party/heap/mmtk/mmtkVisitors.h diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 99856ee..a580888 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -4,14 +4,7 @@ #include #include "src/objects/slots-inl.h" #include "src/heap/safepoint.h" -#include "src/codegen/reloc-info.h" -#include "src/objects/code.h" -#include "src/objects/code-inl.h" -#include "src/codegen/assembler-inl.h" -#include "src/codegen/interface-descriptors.h" -#include "src/codegen/macro-assembler-inl.h" -#include "src/codegen/macro-assembler.h" -#include "src/compiler/code-assembler.h" +#include "mmtkVisitors.h" namespace v8 { namespace internal { @@ -99,93 +92,6 @@ static size_t mmtk_get_object_size(void* object) { UNIMPLEMENTED(); } -class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { - public: - explicit MMTkEdgeVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { - NewBuffer buf = process_edges(NULL, 0, 0); - buffer_ = buf.buf; - cap_ = buf.cap; - } - - virtual ~MMTkEdgeVisitor() { - if (cursor_ > 0) flush(); - if (buffer_ != NULL) { - release_buffer(buffer_, cursor_, cap_); - } - } - - virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { - ProcessRootEdge(root, p); - } - - virtual void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override final { - for (FullObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); - } - - virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) override final { - for (OffHeapObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); - } - - virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { - for (ObjectSlot p = start; p < end; ++p) ProcessEdge(p); - } - - virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override final { - for (MaybeObjectSlot p = start; p < end; ++p) ProcessEdge(p); - } - - virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { - // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. - Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); - auto p = new HeapObject(target); - buffer_[cursor_++] = (void*) p; - if (cursor_ >= cap_) { - flush(); - } - } - - virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { - // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. - auto p = new HeapObject(rinfo->target_object()); - buffer_[cursor_++] = (void*) p; - if (cursor_ >= cap_) { - flush(); - } - } - - virtual void VisitMapPointer(HeapObject host) override final { - ProcessEdge(host.map_slot()); - } - - private: - V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { - ProcessEdge(p); - } - - template - V8_INLINE void ProcessEdge(T p) { - if (!(*p).IsHeapObject()) return; - buffer_[cursor_++] = (void*) p.address(); - if (cursor_ >= cap_) { - flush(); - } - } - - void flush() { - if (cursor_ > 0) { - NewBuffer buf = process_edges_(buffer_, cursor_, cap_); - buffer_ = buf.buf; - cap_ = buf.cap; - cursor_ = 0; - } - } - - ProcessEdgesFn process_edges_; - void** buffer_ = nullptr; - size_t cap_ = 0; - size_t cursor_ = 0; -}; - static void mmtk_scan_roots(ProcessEdgesFn process_edges) { MMTkEdgeVisitor root_visitor(process_edges); v8_heap->IterateRoots(&root_visitor, {}); diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h new file mode 100644 index 0000000..8e63417 --- /dev/null +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -0,0 +1,113 @@ +#ifndef MMTK_VISITORS_H +#define MMTK_VISITORS_H + +#include "src/base/logging.h" +#include "mmtkUpcalls.h" +#include +#include +#include "src/objects/slots-inl.h" +#include "src/heap/safepoint.h" +#include "src/codegen/reloc-info.h" +#include "src/objects/code.h" +#include "src/objects/code-inl.h" +#include "src/codegen/assembler-inl.h" + + +namespace v8 { +namespace internal { +namespace third_party_heap { + + +class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { + public: + explicit MMTkEdgeVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { + NewBuffer buf = process_edges(NULL, 0, 0); + buffer_ = buf.buf; + cap_ = buf.cap; + } + + virtual ~MMTkEdgeVisitor() { + if (cursor_ > 0) flush(); + if (buffer_ != NULL) { + release_buffer(buffer_, cursor_, cap_); + } + } + + virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { + ProcessRootEdge(root, p); + } + + virtual void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override final { + for (FullObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); + } + + virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) override final { + for (OffHeapObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); + } + + virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { + for (ObjectSlot p = start; p < end; ++p) ProcessEdge(p); + } + + virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override final { + for (MaybeObjectSlot p = start; p < end; ++p) ProcessEdge(p); + } + + virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { + // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. + Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); + auto p = new HeapObject(target); + buffer_[cursor_++] = (void*) p; + if (cursor_ >= cap_) { + flush(); + } + } + + virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { + // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. + auto p = new HeapObject(rinfo->target_object()); + buffer_[cursor_++] = (void*) p; + if (cursor_ >= cap_) { + flush(); + } + } + + virtual void VisitMapPointer(HeapObject host) override final { + ProcessEdge(host.map_slot()); + } + + private: + V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { + ProcessEdge(p); + } + + template + V8_INLINE void ProcessEdge(T p) { + if (!(*p).IsHeapObject()) return; + buffer_[cursor_++] = (void*) p.address(); + if (cursor_ >= cap_) { + flush(); + } + } + + void flush() { + if (cursor_ > 0) { + NewBuffer buf = process_edges_(buffer_, cursor_, cap_); + buffer_ = buf.buf; + cap_ = buf.cap; + cursor_ = 0; + } + } + + ProcessEdgesFn process_edges_; + void** buffer_ = nullptr; + size_t cap_ = 0; + size_t cursor_ = 0; +}; + +} +} +} + + +#endif // MMTK_VISITORS_H \ No newline at end of file From 47decc099bed2196d34a97ed806a512887038ffd Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 5 May 2021 16:35:04 +1000 Subject: [PATCH 09/65] Verifiers --- mmtk/src/collection.rs | 12 +++- v8/third_party/heap/mmtk/mmtk.cc | 13 ++++ v8/third_party/heap/mmtk/mmtkUpcalls.cc | 8 +++ v8/third_party/heap/mmtk/mmtkVisitors.h | 84 +++++++++++++++++++++++-- 4 files changed, 112 insertions(+), 5 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 3a13c78..83b0441 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,6 +1,6 @@ use mmtk::scheduler::gc_work::ProcessEdgesWork; use mmtk::scheduler::GCWorker; -use mmtk::util::OpaquePointer; +use mmtk::util::*; use mmtk::vm::Collection; use mmtk::{MutatorContext, MMTK}; @@ -47,4 +47,14 @@ impl Collection for VMCollection { fn prepare_mutator>(_tls: OpaquePointer, _m: &T) { unimplemented!() } + + fn sweep(addr: Address) { + unsafe { + mmtk_delete_object(addr) + } + } +} + +extern { + fn mmtk_delete_object(addr: Address); } diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 5c21f48..dfd5b35 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -178,6 +178,19 @@ HeapObject Heap::NextObject() { return HeapObject(); } } +extern "C" { + void mmtk_delete_object(Address addr) { + for (auto tph_data : *tph_data_list) { + auto space = AllocationSpace(tph_archive_obj_to_space(tph_data->archive(), reinterpret_cast(addr))); + if (space == kNoSpace) continue; + tph_archive_remove(tph_data->archive(), (void*) addr); + return; + } + printf("Unknown object %p\n", (void*) addr); + UNREACHABLE(); + } + +} } } // namespace internal diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index a580888..f84435b 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -20,9 +20,17 @@ SafepointScope* scope; static void mmtk_stop_all_mutators(void *tls) { printf("mmtk_stop_all_mutators\n"); scope = new SafepointScope(v8_heap); + printf("mmtk_stop_all_mutators: heap verify start\n"); + MMTkHeapVerifier visitor; + v8_heap->IterateRoots(&visitor, {}); + printf("mmtk_stop_all_mutators: heap verify end\n"); } static void mmtk_resume_mutators(void *tls) { + printf("mmtk_resume_mutators: heap verify start\n"); + MMTkHeapVerifier visitor; + v8_heap->IterateRoots(&visitor, {}); + printf("mmtk_resume_mutators: heap verify end\n"); printf("mmtk_resume_mutators\n"); delete scope; scope = nullptr; diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index 8e63417..9911ea0 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -11,6 +11,7 @@ #include "src/objects/code.h" #include "src/objects/code-inl.h" #include "src/codegen/assembler-inl.h" +#include namespace v8 { @@ -83,10 +84,14 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { template V8_INLINE void ProcessEdge(T p) { - if (!(*p).IsHeapObject()) return; - buffer_[cursor_++] = (void*) p.address(); - if (cursor_ >= cap_) { - flush(); + HeapObject object; + if ((*p).GetHeapObject(&object)) { + auto s = new HeapObject(object); + + buffer_[cursor_++] = (void*) s; + if (cursor_ >= cap_) { + flush(); + } } } @@ -105,6 +110,77 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { size_t cursor_ = 0; }; + + +class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { + public: + explicit MMTkHeapVerifier() {} + + virtual ~MMTkHeapVerifier() {} + + virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { + VerifyRootEdge(root, p); + } + + virtual void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override final { + for (FullObjectSlot p = start; p < end; ++p) VerifyRootEdge(root, p); + } + + virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) override final { + for (OffHeapObjectSlot p = start; p < end; ++p) VerifyRootEdge(root, p); + } + + virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { + for (ObjectSlot p = start; p < end; ++p) VerifyEdge(p); + } + + virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override final { + for (MaybeObjectSlot p = start; p < end; ++p) VerifyEdge(p); + } + + virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { + Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); + VerifyHeapObject(0, target); + } + + virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { + VerifyHeapObject(0, rinfo->target_object()); + } + + virtual void VisitMapPointer(HeapObject host) override final { + VerifyEdge(host.map_slot()); + } + + private: + V8_INLINE void VerifyRootEdge(Root root, FullObjectSlot p) { + VerifyEdge(p); + } + + template + V8_INLINE void VerifyEdge(T p) { + HeapObject object; + if ((*p).GetHeapObject(&object)) { + VerifyHeapObject(p.address(), object); + } + } + + V8_INLINE void VerifyHeapObject(Address edge, HeapObject o) { + if (marked_objects_.find(o.ptr()) == marked_objects_.end()) { + marked_objects_.insert(o.ptr()); + if (!third_party_heap::Heap::IsValidHeapObject(o)) { + printf("Dead edge %p -> %p\n", (void*) edge, (void*) o.ptr()); + } + CHECK(third_party_heap::Heap::IsValidHeapObject(o)); + o.Iterate(this); + } + } + + std::unordered_set
marked_objects_; +}; + + + + } } } From 3b21fb28dea1d524ee81b9cb79df2be6e4757db4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 10 May 2021 16:03:23 +1000 Subject: [PATCH 10/65] Update --- mmtk/src/scanning.rs | 1 - v8/third_party/heap/mmtk/mmtkUpcalls.cc | 15 ++++--- v8/third_party/heap/mmtk/mmtkVisitors.h | 59 ++++++++++++------------- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 33f1fdd..6eb3c56 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -5,7 +5,6 @@ use mmtk::util::OpaquePointer; use mmtk::vm::Scanning; use mmtk::{Mutator, TransitiveClosure}; use V8; -use mmtk::scheduler::gc_work::*; use mmtk::scheduler::*; use crate::*; diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index f84435b..702967e 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -18,20 +18,20 @@ bool gcInProgress = false; SafepointScope* scope; static void mmtk_stop_all_mutators(void *tls) { - printf("mmtk_stop_all_mutators\n"); + fprintf(stderr, "mmtk_stop_all_mutators\n"); scope = new SafepointScope(v8_heap); - printf("mmtk_stop_all_mutators: heap verify start\n"); + fprintf(stderr, "mmtk_stop_all_mutators: heap verify start\n"); MMTkHeapVerifier visitor; v8_heap->IterateRoots(&visitor, {}); - printf("mmtk_stop_all_mutators: heap verify end\n"); + fprintf(stderr, "mmtk_stop_all_mutators: heap verify end\n"); } static void mmtk_resume_mutators(void *tls) { - printf("mmtk_resume_mutators: heap verify start\n"); + fprintf(stderr, "mmtk_resume_mutators: heap verify start\n"); MMTkHeapVerifier visitor; v8_heap->IterateRoots(&visitor, {}); - printf("mmtk_resume_mutators: heap verify end\n"); - printf("mmtk_resume_mutators\n"); + fprintf(stderr, "mmtk_resume_mutators: heap verify end\n"); + fprintf(stderr, "mmtk_resume_mutators\n"); delete scope; scope = nullptr; std::unique_lock lock(m); @@ -48,7 +48,7 @@ static void mmtk_block_for_gc() { gcInProgress = true; std::unique_lock lock(m); cv->wait(lock, []{ return !gcInProgress; }); - printf("mmtk_block_for_gc end\n"); + fprintf(stderr, "mmtk_block_for_gc end\n"); } static void* mmtk_active_collector(void* tls) { @@ -109,6 +109,7 @@ static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn proce MMTkEdgeVisitor visitor(process_edges); for (size_t i = 0; i < count; i++) { auto ptr = *(objects + i); + DCHECK_EQ(((Address) ptr) & 1, 0); auto obj = HeapObject::FromAddress(((Address) ptr)); obj.Iterate(&visitor); } diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index 9911ea0..f438d62 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -57,20 +57,12 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); - auto p = new HeapObject(target); - buffer_[cursor_++] = (void*) p; - if (cursor_ >= cap_) { - flush(); - } + AddEdge(target); } virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. - auto p = new HeapObject(rinfo->target_object()); - buffer_[cursor_++] = (void*) p; - if (cursor_ >= cap_) { - flush(); - } + AddEdge(rinfo->target_object()); } virtual void VisitMapPointer(HeapObject host) override final { @@ -86,12 +78,16 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { V8_INLINE void ProcessEdge(T p) { HeapObject object; if ((*p).GetHeapObject(&object)) { - auto s = new HeapObject(object); + AddEdge(object); + } + } - buffer_[cursor_++] = (void*) s; - if (cursor_ >= cap_) { - flush(); - } + V8_INLINE void AddEdge(HeapObject o) { + HeapObject* edge = new HeapObject(); + *edge = o; + buffer_[cursor_++] = (void*) edge; + if (cursor_ >= cap_) { + flush(); } } @@ -114,7 +110,8 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { public: - explicit MMTkHeapVerifier() {} + explicit MMTkHeapVerifier() { + } virtual ~MMTkHeapVerifier() {} @@ -131,47 +128,47 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { } virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { - for (ObjectSlot p = start; p < end; ++p) VerifyEdge(p); + for (ObjectSlot p = start; p < end; ++p) VerifyEdge(host, p); } virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override final { - for (MaybeObjectSlot p = start; p < end; ++p) VerifyEdge(p); + for (MaybeObjectSlot p = start; p < end; ++p) VerifyEdge(host, p); } virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); - VerifyHeapObject(0, target); + VerifyHeapObject(host, 0, target); } virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { - VerifyHeapObject(0, rinfo->target_object()); + VerifyHeapObject(host, 0, rinfo->target_object()); } virtual void VisitMapPointer(HeapObject host) override final { - VerifyEdge(host.map_slot()); + VerifyEdge(host, host.map_slot()); } private: V8_INLINE void VerifyRootEdge(Root root, FullObjectSlot p) { - VerifyEdge(p); + VerifyEdge(HeapObject(), p); } template - V8_INLINE void VerifyEdge(T p) { + V8_INLINE void VerifyEdge(HeapObject host, T p) { HeapObject object; if ((*p).GetHeapObject(&object)) { - VerifyHeapObject(p.address(), object); + VerifyHeapObject(host, p.address(), object); } } - V8_INLINE void VerifyHeapObject(Address edge, HeapObject o) { + V8_INLINE void VerifyHeapObject(HeapObject host, Address edge, HeapObject o) { if (marked_objects_.find(o.ptr()) == marked_objects_.end()) { - marked_objects_.insert(o.ptr()); - if (!third_party_heap::Heap::IsValidHeapObject(o)) { - printf("Dead edge %p -> %p\n", (void*) edge, (void*) o.ptr()); - } - CHECK(third_party_heap::Heap::IsValidHeapObject(o)); - o.Iterate(this); + marked_objects_.insert(o.ptr()); + if (!third_party_heap::Heap::IsValidHeapObject(o)) { + printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); + } + CHECK(third_party_heap::Heap::IsValidHeapObject(o)); + o.Iterate(this); } } From f8373b990af5e069b565704e9f1687f47ced3cc7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 12 May 2021 09:30:56 +1000 Subject: [PATCH 11/65] WIP: Drop stub cache --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index bcf055f..dd87c2f 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -4,6 +4,8 @@ #include #include "src/objects/slots-inl.h" #include "src/heap/safepoint.h" +#include "src/deoptimizer/deoptimizer.h" +#include "src/execution/frames-inl.h" #include "mmtkVisitors.h" namespace v8 { @@ -32,6 +34,17 @@ static void mmtk_resume_mutators(void *tls) { v8_heap->IterateRoots(&visitor, {}); fprintf(stderr, "mmtk_resume_mutators: heap verify end\n"); fprintf(stderr, "mmtk_resume_mutators\n"); + + v8_heap->isolate()->inner_pointer_to_code_cache()->Flush(); + // The stub caches are not traversed during GC; clear them to force + // their lazy re-initialization. This must be done after the + // GC, because it relies on the new address of certain old space + // objects (empty string, illegal builtin). + v8_heap->isolate()->load_stub_cache()->Clear(); + v8_heap->isolate()->store_stub_cache()->Clear(); + // Some code objects were marked for deoptimization during the GC. + // Deoptimizer::DeoptimizeMarkedCode(v8_heap->isolate()); + delete scope; scope = nullptr; std::unique_lock lock(m); From 172b85e6188a0823c3af0600aafcbf82e2ce3e5b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 14 May 2021 09:44:13 +1000 Subject: [PATCH 12/65] WIP: Update weak refs --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 7 +++---- v8/third_party/heap/mmtk/mmtkVisitors.h | 25 +++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index dd87c2f..0291319 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -23,15 +23,13 @@ static void mmtk_stop_all_mutators(void *tls) { fprintf(stderr, "mmtk_stop_all_mutators\n"); scope = new SafepointScope(v8_heap); fprintf(stderr, "mmtk_stop_all_mutators: heap verify start\n"); - MMTkHeapVerifier visitor; - v8_heap->IterateRoots(&visitor, {}); + MMTkHeapVerifier::Verify(v8_heap); fprintf(stderr, "mmtk_stop_all_mutators: heap verify end\n"); } static void mmtk_resume_mutators(void *tls) { fprintf(stderr, "mmtk_resume_mutators: heap verify start\n"); - MMTkHeapVerifier visitor; - v8_heap->IterateRoots(&visitor, {}); + MMTkHeapVerifier::Verify(v8_heap); fprintf(stderr, "mmtk_resume_mutators: heap verify end\n"); fprintf(stderr, "mmtk_resume_mutators\n"); @@ -112,6 +110,7 @@ static size_t mmtk_get_object_size(void* object) { static void mmtk_scan_roots(ProcessEdgesFn process_edges) { MMTkEdgeVisitor root_visitor(process_edges); v8_heap->IterateRoots(&root_visitor, {}); + v8_heap->ProcessAllWeakReferences(&root_visitor); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index f438d62..3ddae55 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -19,7 +19,7 @@ namespace internal { namespace third_party_heap { -class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { +class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor, public WeakObjectRetainer { public: explicit MMTkEdgeVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { NewBuffer buf = process_edges(NULL, 0, 0); @@ -69,6 +69,12 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { ProcessEdge(host.map_slot()); } + virtual Object RetainAs(Object object) override final { + HeapObject heap_object = HeapObject::cast(object); + AddEdge(heap_object); + return object; + } + private: V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { ProcessEdge(p); @@ -148,6 +154,20 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { VerifyEdge(host, host.map_slot()); } + void TransitiveClosure() { + while (mark_stack_.size() != 0) { + auto o = mark_stack_.back(); + mark_stack_.pop_back(); + o.Iterate(this); + } + } + + static void Verify(v8::internal::Heap* heap) { + MMTkHeapVerifier visitor; + heap->IterateRoots(&visitor, {}); + visitor.TransitiveClosure(); + } + private: V8_INLINE void VerifyRootEdge(Root root, FullObjectSlot p) { VerifyEdge(HeapObject(), p); @@ -168,11 +188,12 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } CHECK(third_party_heap::Heap::IsValidHeapObject(o)); - o.Iterate(this); + mark_stack_.push_back(o); } } std::unordered_set
marked_objects_; + std::vector mark_stack_; }; From 4bac4f5c47bd85ba2d700e2b633857ba618051da Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 14 May 2021 11:28:55 +1000 Subject: [PATCH 13/65] Update weakrefs --- mmtk/src/collection.rs | 6 ++++++ mmtk/src/lib.rs | 1 + v8/third_party/heap/mmtk/mmtk.h | 1 + v8/third_party/heap/mmtk/mmtkUpcalls.cc | 7 ++++++- v8/third_party/heap/mmtk/mmtkVisitors.h | 22 +++++++++++++++------- 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 6c14d8a..0116ba2 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -57,6 +57,12 @@ impl Collection for VMCollection { mmtk_delete_object(addr) } } + + fn process_weak_refs() { + unsafe { + ((*UPCALLS).process_weak_refs)(); + } + } } extern { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 7f7a56f..70c2bdb 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -52,6 +52,7 @@ pub struct V8_Upcalls { pub is_mutator: extern "C" fn(tls: OpaquePointer) -> bool, pub scan_roots: extern "C" fn(process_edges: ProcessEdgesFn), pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn), + pub process_weak_refs: extern "C" fn(), } pub static mut UPCALLS: *const V8_Upcalls = null_mut(); diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 3fafc41..0602039 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -134,6 +134,7 @@ typedef struct { bool (*is_mutator) (void* tls); void (*scan_roots) (ProcessEdgesFn process_edges); void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges); + void (*process_weak_refs) (); } V8_Upcalls; /** diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 0291319..9d34a51 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -27,6 +27,11 @@ static void mmtk_stop_all_mutators(void *tls) { fprintf(stderr, "mmtk_stop_all_mutators: heap verify end\n"); } +static void mmtk_process_weak_refs() { + MMTkWeakObjectRetainer retainer; + v8_heap->ProcessAllWeakReferences(&retainer); +} + static void mmtk_resume_mutators(void *tls) { fprintf(stderr, "mmtk_resume_mutators: heap verify start\n"); MMTkHeapVerifier::Verify(v8_heap); @@ -110,7 +115,6 @@ static size_t mmtk_get_object_size(void* object) { static void mmtk_scan_roots(ProcessEdgesFn process_edges) { MMTkEdgeVisitor root_visitor(process_edges); v8_heap->IterateRoots(&root_visitor, {}); - v8_heap->ProcessAllWeakReferences(&root_visitor); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { @@ -140,6 +144,7 @@ V8_Upcalls mmtk_upcalls = { mmtk_is_mutator, mmtk_scan_roots, mmtk_scan_objects, + mmtk_process_weak_refs, }; } // namespace third_party_heap } // namespace internal diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index 3ddae55..c5235a6 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -19,7 +19,7 @@ namespace internal { namespace third_party_heap { -class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor, public WeakObjectRetainer { +class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { public: explicit MMTkEdgeVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { NewBuffer buf = process_edges(NULL, 0, 0); @@ -69,12 +69,6 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor, public WeakObje ProcessEdge(host.map_slot()); } - virtual Object RetainAs(Object object) override final { - HeapObject heap_object = HeapObject::cast(object); - AddEdge(heap_object); - return object; - } - private: V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { ProcessEdge(p); @@ -197,6 +191,20 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { }; +class MMTkWeakObjectRetainer: public WeakObjectRetainer { + public: + virtual Object RetainAs(Object object) override final { + if (object == Object()) return object; + HeapObject heap_object = HeapObject::cast(object); + if (third_party_heap::Heap::IsValidHeapObject(heap_object)) { + return object; + } else { + return Object(); + } + } +}; + + } From 4d0c6de61f9f4109f425bf435554ce05ff5c62f3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 17 May 2021 16:10:51 +1000 Subject: [PATCH 14/65] Heap::Capacity --- v8/third_party/heap/mmtk/mmtk.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index dfd5b35..367b912 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -61,6 +61,13 @@ MMTk_Heap GetMMTkHeap(Address object_pointer) { static std::atomic_bool IsolateCreated { false }; +#define GB (1ull << 30) +#define FIXED_HEAP_SIZE (1ull * GB) + +size_t Heap::Capacity() { + return FIXED_HEAP_SIZE; +} + std::unique_ptr Heap::New(v8::internal::Isolate* isolate) { DCHECK(!v8_heap); v8_heap = isolate->heap(); @@ -68,8 +75,7 @@ std::unique_ptr Heap::New(v8::internal::Isolate* isolate) { auto isolate_created = IsolateCreated.exchange(true); DCHECK_WITH_MSG(!isolate_created, "Multiple isolates are not supported."); fprintf(stderr, "New Isolate: %lx\n", (unsigned long) isolate); - const size_t GB = 1u << 30; - MMTk_Heap new_heap = v8_new_heap(&mmtk_upcalls, GB); + MMTk_Heap new_heap = v8_new_heap(&mmtk_upcalls, FIXED_HEAP_SIZE); tph_mutator_ = reinterpret_cast(bind_mutator(new_heap, &tph_mutator_)); // FIXME code_range_ = base::AddressRegion(0x60000000, (0xb0000000- 0x60000000)); // isolate->AddCodeRange(code_range_.begin(), code_range_.size()); From 1778beef6c5764c722631a58118a646ba62ed686 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 May 2021 15:55:35 +1000 Subject: [PATCH 15/65] Fix invalid descriptor-array cache --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 16 ++++++++++++++-- v8/third_party/heap/mmtk/mmtkVisitors.h | 5 ++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 9d34a51..b1aba16 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -4,8 +4,10 @@ #include #include "src/objects/slots-inl.h" #include "src/heap/safepoint.h" +#include "src/heap/array-buffer-sweeper.h" #include "src/deoptimizer/deoptimizer.h" #include "src/execution/frames-inl.h" +#include "src/regexp/regexp.h" #include "mmtkVisitors.h" namespace v8 { @@ -25,6 +27,15 @@ static void mmtk_stop_all_mutators(void *tls) { fprintf(stderr, "mmtk_stop_all_mutators: heap verify start\n"); MMTkHeapVerifier::Verify(v8_heap); fprintf(stderr, "mmtk_stop_all_mutators: heap verify end\n"); + + v8_heap->isolate()->descriptor_lookup_cache()->Clear(); + RegExpResultsCache::Clear(v8_heap->string_split_cache()); + RegExpResultsCache::Clear(v8_heap->regexp_multiple_cache()); + // v8_heap->FlushNumberStringCache(); + int len = v8_heap->number_string_cache().length(); + for (int i = 0; i < len; i++) { + v8_heap->number_string_cache().set_undefined(i); + } } static void mmtk_process_weak_refs() { @@ -37,6 +48,7 @@ static void mmtk_resume_mutators(void *tls) { MMTkHeapVerifier::Verify(v8_heap); fprintf(stderr, "mmtk_resume_mutators: heap verify end\n"); fprintf(stderr, "mmtk_resume_mutators\n"); + // v8_heap->array_buffer_sweeper()->RequestSweepFull(); v8_heap->isolate()->inner_pointer_to_code_cache()->Flush(); // The stub caches are not traversed during GC; clear them to force @@ -113,12 +125,12 @@ static size_t mmtk_get_object_size(void* object) { } static void mmtk_scan_roots(ProcessEdgesFn process_edges) { - MMTkEdgeVisitor root_visitor(process_edges); + MMTkEdgeVisitor root_visitor(v8_heap, process_edges); v8_heap->IterateRoots(&root_visitor, {}); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { - MMTkEdgeVisitor visitor(process_edges); + MMTkEdgeVisitor visitor(v8_heap, process_edges); for (size_t i = 0; i < count; i++) { auto ptr = *(objects + i); DCHECK_EQ(((Address) ptr) & 1, 0); diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index c5235a6..fa3501a 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -10,6 +10,7 @@ #include "src/codegen/reloc-info.h" #include "src/objects/code.h" #include "src/objects/code-inl.h" +#include "src/objects/map-inl.h" #include "src/codegen/assembler-inl.h" #include @@ -21,7 +22,8 @@ namespace third_party_heap { class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { public: - explicit MMTkEdgeVisitor(ProcessEdgesFn process_edges): process_edges_(process_edges) { + explicit MMTkEdgeVisitor(v8::internal::Heap* heap, ProcessEdgesFn process_edges): heap_(heap), process_edges_(process_edges) { + USE(heap_); NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; cap_ = buf.cap; @@ -100,6 +102,7 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { } } + v8::internal::Heap* heap_; ProcessEdgesFn process_edges_; void** buffer_ = nullptr; size_t cap_ = 0; From 8d741fe8bba12b2e8c1579d94973ba556c0f317c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 20 May 2021 14:39:18 +1000 Subject: [PATCH 16/65] Fix is_mutator check --- v8/third_party/heap/mmtk/mmtk.cc | 5 ++++- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 367b912..77ef6b8 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -36,6 +36,8 @@ TPHData* get_tph_data(Heap* tph) { std::vector* all_mutators = new std::vector(); +thread_local bool is_mutator = false; + inline void CheckMutator(Heap* tph) { TPHData* tph_data_ = get_tph_data(tph); if (tph_mutator_ == nullptr) { @@ -44,6 +46,7 @@ inline void CheckMutator(Heap* tph) { tph_mutator_->tph_data = tph_data_; all_mutators->push_back(tph_mutator_); } + is_mutator = true; } MMTk_Heap GetMMTkHeap(Address object_pointer) { @@ -62,7 +65,7 @@ MMTk_Heap GetMMTkHeap(Address object_pointer) { static std::atomic_bool IsolateCreated { false }; #define GB (1ull << 30) -#define FIXED_HEAP_SIZE (1ull * GB) +#define FIXED_HEAP_SIZE (4ull * GB) size_t Heap::Capacity() { return FIXED_HEAP_SIZE; diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index b1aba16..bf6cab0 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -83,8 +83,10 @@ static void* mmtk_get_mmtk_mutator(void* tls) { UNIMPLEMENTED(); } +extern thread_local bool is_mutator; + static bool mmtk_is_mutator(void* tls) { - return false; + return is_mutator; } extern std::vector* all_mutators; From a837f142ba47b9c1330242d1e8469d37783f545a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 21 May 2021 12:28:08 +1000 Subject: [PATCH 17/65] Update --- mmtk/Cargo.toml | 1 + v8/third_party/heap/mmtk/mmtkUpcalls.cc | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 3e4ff9d..66460eb 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,6 +10,7 @@ crate-type = ["cdylib"] [profile.dev] panic = "abort" +# opt-level = 3 [profile.release] panic = "abort" diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index bf6cab0..07b8cf5 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -19,11 +19,10 @@ extern v8::internal::Heap* v8_heap; std::mutex m; std::condition_variable* cv = new std::condition_variable(); bool gcInProgress = false; -SafepointScope* scope; static void mmtk_stop_all_mutators(void *tls) { fprintf(stderr, "mmtk_stop_all_mutators\n"); - scope = new SafepointScope(v8_heap); + v8_heap->safepoint()->EnterSafepointScope(ThreadKind::kBackground); fprintf(stderr, "mmtk_stop_all_mutators: heap verify start\n"); MMTkHeapVerifier::Verify(v8_heap); fprintf(stderr, "mmtk_stop_all_mutators: heap verify end\n"); @@ -60,8 +59,7 @@ static void mmtk_resume_mutators(void *tls) { // Some code objects were marked for deoptimization during the GC. // Deoptimizer::DeoptimizeMarkedCode(v8_heap->isolate()); - delete scope; - scope = nullptr; + v8_heap->safepoint()->LeaveSafepointScope(); std::unique_lock lock(m); gcInProgress = false; cv->notify_all(); From 9fa12ef84839074ea3f7dde3f233b3d121aa863c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 24 May 2021 12:23:03 +1000 Subject: [PATCH 18/65] forward roots on root-scanning --- mmtk/src/api.rs | 9 ++++ mmtk/src/lib.rs | 3 +- mmtk/src/scanning.rs | 58 +++++++++++++++++++--- v8/third_party/heap/mmtk/mmtk.h | 5 +- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 4 +- v8/third_party/heap/mmtk/mmtkVisitors.h | 66 ++++++++++++++++--------- 6 files changed, 111 insertions(+), 34 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 3e85732..385457d 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -19,6 +19,15 @@ pub unsafe extern "C" fn release_buffer(ptr: *mut Address, length: usize, capaci let _vec = Vec::
::from_raw_parts(ptr, length, capacity); } +#[no_mangle] +pub unsafe extern "C" fn mmtk_is_movable(object: ObjectReference) -> i32 { + let object = unsafe { + let untagged_word = object.to_address().as_usize() & !0b11usize; + Address::from_usize(untagged_word).to_object_reference() + }; + if object.is_movable() { 1 } else { 0 } +} + #[no_mangle] pub extern "C" fn v8_new_heap(calls: *const V8_Upcalls, heap_size: usize) -> *mut c_void { unsafe { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index aec988f..2a5c849 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -33,6 +33,7 @@ pub struct NewBuffer { } type ProcessEdgesFn = *const extern "C" fn(buf: *mut Address, size: usize, cap: usize) -> NewBuffer; +type TraceRootFn = *const extern "C" fn(slot: Address, ctx: &'static mut GCWorker) -> Address; #[repr(C)] pub struct V8_Upcalls { @@ -50,7 +51,7 @@ pub struct V8_Upcalls { pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut Mutator, pub is_mutator: extern "C" fn(tls: VMThread) -> bool, - pub scan_roots: extern "C" fn(process_edges: ProcessEdgesFn), + pub scan_roots: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void), pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn), pub process_weak_refs: extern "C" fn(), } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 866ccba..2d167cf 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -53,8 +53,8 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots>() { mmtk::memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Prepare, - ScanRoots::::new(), + WorkBucketStage::Closure, + ScanAndForwardRoots::::new(), ); } @@ -63,20 +63,64 @@ impl Scanning for VMScanning { } } -pub struct ScanRoots>(PhantomData); +pub struct ScanAndForwardRoots>(PhantomData); -impl> ScanRoots { +impl> ScanAndForwardRoots { pub fn new() -> Self { Self(PhantomData) } } -impl> GCWork for ScanRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { +impl> GCWork for ScanAndForwardRoots { + fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + unsafe { + debug_assert!(ROOT_OBJECTS.is_empty()); + ((*UPCALLS).scan_roots)(trace_root:: as _, worker as *mut _ as _); + if !ROOT_OBJECTS.is_empty() { + flush_roots::(worker); + } + debug_assert!(ROOT_OBJECTS.is_empty()); + } + } +} + +static mut ROOT_OBJECTS: Vec = Vec::new(); + +fn flush_roots>(worker: &mut GCWorker) { + let mut buf = vec![]; + unsafe { std::mem::swap(&mut buf, &mut ROOT_OBJECTS); } + let scan_objects_work = mmtk::scheduler::gc_work::ScanObjects::::new(buf, false); + mmtk::memory_manager::add_work_packet( + &SINGLETON, + WorkBucketStage::Closure, + scan_objects_work, + ); +} + +pub(crate) extern "C" fn trace_root>(slot: Address, worker: &'static mut GCWorker) -> ObjectReference { + let obj = unsafe { slot.load() }; + let mut w = W::new(vec![], false, &SINGLETON); + w.set_worker(worker); + let new_obj = w.trace_object(obj); + if W::OVERWRITE_REFERENCE { unsafe { - ((*UPCALLS).scan_roots)(create_process_edges_work:: as _); + slot.store((new_obj.to_address().as_usize() & !0b11) | 1); + } + } + unsafe { + if ROOT_OBJECTS.is_empty() { + ROOT_OBJECTS.reserve(W::CAPACITY); + } + } + for o in &w.nodes { + unsafe { ROOT_OBJECTS.push(*o); } + } + unsafe { + if ROOT_OBJECTS.len() > W::CAPACITY { + flush_roots::(worker); } } + new_obj } pub(crate) extern "C" fn create_process_edges_work>( diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 0602039..dfe954b 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -116,6 +116,7 @@ typedef struct { } NewBuffer; typedef NewBuffer (*ProcessEdgesFn)(void** buf, size_t len, size_t cap); +typedef void* (*TraceRootFn)(void* slot, void* ctx); typedef struct { void (*stop_all_mutators) (void *tls); @@ -132,7 +133,7 @@ typedef struct { size_t (*get_object_size) (void* object); void* (*get_mmtk_mutator) (void* tls); bool (*is_mutator) (void* tls); - void (*scan_roots) (ProcessEdgesFn process_edges); + void (*scan_roots) (TraceRootFn process_edges, void* context); void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges); void (*process_weak_refs) (); } V8_Upcalls; @@ -153,6 +154,8 @@ extern void add_phantom_candidate(void* ref, void* referent); extern void harness_begin(void* ref, void *tls); extern void harness_end(void* ref); +extern int mmtk_is_movable(v8::internal::Object o); + #ifdef __cplusplus } #endif diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 07b8cf5..fcd9f48 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -124,8 +124,8 @@ static size_t mmtk_get_object_size(void* object) { UNIMPLEMENTED(); } -static void mmtk_scan_roots(ProcessEdgesFn process_edges) { - MMTkEdgeVisitor root_visitor(v8_heap, process_edges); +static void mmtk_scan_roots(TraceRootFn trace_root, void* context) { + MMTkRootVisitor root_visitor(v8_heap, trace_root, context); v8_heap->IterateRoots(&root_visitor, {}); } diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index fa3501a..f972abc 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -19,21 +19,10 @@ namespace v8 { namespace internal { namespace third_party_heap { - -class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { +class MMTkRootVisitor: public RootVisitor { public: - explicit MMTkEdgeVisitor(v8::internal::Heap* heap, ProcessEdgesFn process_edges): heap_(heap), process_edges_(process_edges) { + explicit MMTkRootVisitor(v8::internal::Heap* heap, TraceRootFn trace_root, void* context): heap_(heap), trace_root_(trace_root), context_(context) { USE(heap_); - NewBuffer buf = process_edges(NULL, 0, 0); - buffer_ = buf.buf; - cap_ = buf.cap; - } - - virtual ~MMTkEdgeVisitor() { - if (cursor_ > 0) flush(); - if (buffer_ != NULL) { - release_buffer(buffer_, cursor_, cap_); - } } virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { @@ -48,6 +37,35 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { for (OffHeapObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); } + private: + V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot slot) { + HeapObject object; + if ((*slot).GetHeapObject(&object)) { + trace_root_((void*) slot.address(), context_); + } + } + + v8::internal::Heap* heap_; + TraceRootFn trace_root_; + void* context_; +}; + +class MMTkEdgeVisitor: public ObjectVisitor { + public: + explicit MMTkEdgeVisitor(v8::internal::Heap* heap, ProcessEdgesFn process_edges): heap_(heap), process_edges_(process_edges) { + USE(heap_); + NewBuffer buf = process_edges(NULL, 0, 0); + buffer_ = buf.buf; + cap_ = buf.cap; + } + + virtual ~MMTkEdgeVisitor() { + if (cursor_ > 0) flush(); + if (buffer_ != NULL) { + release_buffer(buffer_, cursor_, cap_); + } + } + virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { for (ObjectSlot p = start; p < end; ++p) ProcessEdge(p); } @@ -57,14 +75,12 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { } virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { - // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); - AddEdge(target); + AddNonMovingEdge(target); } virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { - // We need to pass the location of the root pointer (i.e. the slot) to MMTk. Fake it for now. - AddEdge(rinfo->target_object()); + AddNonMovingEdge(rinfo->target_object()); } virtual void VisitMapPointer(HeapObject host) override final { @@ -80,17 +96,21 @@ class MMTkEdgeVisitor: public RootVisitor, public ObjectVisitor { V8_INLINE void ProcessEdge(T p) { HeapObject object; if ((*p).GetHeapObject(&object)) { - AddEdge(object); + PushEdge((void*) p.address()); } } - V8_INLINE void AddEdge(HeapObject o) { + V8_INLINE void AddNonMovingEdge(HeapObject o) { + DCHECK(!mmtk_is_movable(o)); + // TODO: Don't malloc HeapObject* edge = new HeapObject(); *edge = o; - buffer_[cursor_++] = (void*) edge; - if (cursor_ >= cap_) { - flush(); - } + PushEdge((void*) edge); + } + + V8_INLINE void PushEdge(void* edge) { + buffer_[cursor_++] = edge; + if (cursor_ >= cap_) flush(); } void flush() { From 618692a16f37fdccdb9716b4f61fc26637596d84 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 25 May 2021 11:17:29 +1000 Subject: [PATCH 19/65] WIP: Refactor --- mmtk/src/api.rs | 10 ++++++++++ mmtk/src/object_model.rs | 19 +++++++++++++++---- mmtk/src/scanning.rs | 11 ++++++++--- v8/third_party/heap/mmtk/mmtk.h | 1 + v8/third_party/heap/mmtk/mmtkUpcalls.cc | 5 ++++- v8/third_party/heap/mmtk/mmtkVisitors.h | 8 ++++++++ 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 385457d..12bb5ae 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -28,6 +28,16 @@ pub unsafe extern "C" fn mmtk_is_movable(object: ObjectReference) -> i32 { if object.is_movable() { 1 } else { 0 } } +#[no_mangle] +pub unsafe extern "C" fn mmtk_get_forwarded_object(object: ObjectReference) -> *mut c_void { + let tag = object.to_address().as_usize() & 0b11usize; + let object = unsafe { + let untagged_word = object.to_address().as_usize() & !0b11usize; + Address::from_usize(untagged_word).to_object_reference() + }; + object.get_forwarded_object().map(|x| (x.to_address().as_usize() | tag) as *mut c_void).unwrap_or(0 as _) +} + #[no_mangle] pub extern "C" fn v8_new_heap(calls: *const V8_Upcalls, heap_size: usize) -> *mut c_void { unsafe { diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index a3a3f8c..fe6c3da 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -10,12 +10,23 @@ pub struct VMObjectModel {} impl ObjectModel for VMObjectModel { const GC_BYTE_OFFSET: isize = 7; + #[inline(always)] fn copy( - _from: ObjectReference, - _allocator: AllocationSemantics, - _copy_context: &mut impl CopyContext, + from: ObjectReference, + allocator: AllocationSemantics, + copy_context: &mut impl CopyContext, ) -> ObjectReference { - unimplemented!() + let bytes = unsafe { ((*UPCALLS).get_object_size)(from) }; + let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, allocator); + // Copy + // println!("Copy {:?} -> {:?}", from, dst); + let src = from.to_address(); + for i in 0..bytes { + unsafe { (dst + i).store((src + i).load::()) }; + } + let to_obj = unsafe { dst.to_object_reference() }; + copy_context.post_copy(to_obj, unsafe { Address::zero() }, bytes, allocator); + to_obj } fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 2d167cf..d994f0b 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -98,13 +98,18 @@ fn flush_roots>(worker: &mut GCWorker) { } pub(crate) extern "C" fn trace_root>(slot: Address, worker: &'static mut GCWorker) -> ObjectReference { - let obj = unsafe { slot.load() }; + let obj: ObjectReference = unsafe { slot.load() }; + let tag = obj.to_address().as_usize() & 0b11usize; let mut w = W::new(vec![], false, &SINGLETON); w.set_worker(worker); - let new_obj = w.trace_object(obj); + let object_untagged = unsafe { + Address::from_usize(obj.to_address().as_usize() & !0b11usize).to_object_reference() + }; + let new_obj = w.trace_object(object_untagged); + // println!("Root {:?} {:?} -> {:?}", slot, obj, new_obj); if W::OVERWRITE_REFERENCE { unsafe { - slot.store((new_obj.to_address().as_usize() & !0b11) | 1); + slot.store((new_obj.to_address().as_usize() & !0b11) | tag); } } unsafe { diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index dfe954b..f9419fb 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -155,6 +155,7 @@ extern void harness_begin(void* ref, void *tls); extern void harness_end(void* ref); extern int mmtk_is_movable(v8::internal::Object o); +extern void* mmtk_get_forwarded_object(v8::internal::Object o); #ifdef __cplusplus } diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index fcd9f48..52ff8ae 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -38,6 +38,7 @@ static void mmtk_stop_all_mutators(void *tls) { } static void mmtk_process_weak_refs() { + // fprintf(stderr, "mmtk_process_weak_refs\n"); MMTkWeakObjectRetainer retainer; v8_heap->ProcessAllWeakReferences(&retainer); } @@ -121,7 +122,9 @@ static void mmtk_dump_object(void* object) { } static size_t mmtk_get_object_size(void* object) { - UNIMPLEMENTED(); + auto o = HeapObject::FromAddress((Address) object); + auto m = o.map(); + return o.SizeFromMap(m); } static void mmtk_scan_roots(TraceRootFn trace_root, void* context) { diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index f972abc..f5c67d1 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -217,11 +217,19 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { class MMTkWeakObjectRetainer: public WeakObjectRetainer { public: virtual Object RetainAs(Object object) override final { + // fprintf(stderr, "RetainAs %p\n", (void*) object.ptr()); if (object == Object()) return object; HeapObject heap_object = HeapObject::cast(object); if (third_party_heap::Heap::IsValidHeapObject(heap_object)) { + auto f = mmtk_get_forwarded_object(heap_object); + if (f != nullptr) { + // fprintf(stderr, "%p -> %p\n", (void*) object.ptr(), (void*) f); + return Object((Address) f); + } + // fprintf(stderr, "%p is dead 1 \n", (void*) object.ptr()); return object; } else { + // fprintf(stderr, "%p is dead 2 \n", (void*) object.ptr()); return Object(); } } From d07f808fc8d1537f3941eba9824f0e99901f3fe2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 25 May 2021 12:43:09 +1000 Subject: [PATCH 20/65] Simplify object archive --- mmtk/src/api.rs | 4 +- mmtk/src/collection.rs | 15 +-- mmtk/src/object_archive.rs | 187 +++++++++---------------------- mmtk/src/scanning.rs | 4 +- v8/third_party/heap/mmtk/mmtk.cc | 38 +------ v8/third_party/heap/mmtk/mmtk.h | 1 - 6 files changed, 65 insertions(+), 184 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 12bb5ae..8c45ca6 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -21,7 +21,7 @@ pub unsafe extern "C" fn release_buffer(ptr: *mut Address, length: usize, capaci #[no_mangle] pub unsafe extern "C" fn mmtk_is_movable(object: ObjectReference) -> i32 { - let object = unsafe { + let object = { let untagged_word = object.to_address().as_usize() & !0b11usize; Address::from_usize(untagged_word).to_object_reference() }; @@ -31,7 +31,7 @@ pub unsafe extern "C" fn mmtk_is_movable(object: ObjectReference) -> i32 { #[no_mangle] pub unsafe extern "C" fn mmtk_get_forwarded_object(object: ObjectReference) -> *mut c_void { let tag = object.to_address().as_usize() & 0b11usize; - let object = unsafe { + let object = { let untagged_word = object.to_address().as_usize() & !0b11usize; Address::from_usize(untagged_word).to_object_reference() }; diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 065cf66..bc4d1ae 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,13 +1,14 @@ use mmtk::scheduler::GCWorker; use mmtk::util::*; use mmtk::scheduler::ProcessEdgesWork; -use mmtk::util::opaque_pointer::*; use mmtk::vm::Collection; use mmtk::{MutatorContext, MMTK}; use UPCALLS; use V8; +use crate::object_archive::global_object_archive; + pub struct VMCollection {} impl Collection for VMCollection { @@ -29,7 +30,7 @@ impl Collection for VMCollection { } } - fn spawn_worker_thread(tls: VMThread, ctx: Option<&GCWorker>) { + fn spawn_worker_thread(_tls: VMThread, ctx: Option<&GCWorker>) { let ctx_ptr = if let Some(r) = ctx { r as *const GCWorker as *mut GCWorker } else { @@ -53,10 +54,8 @@ impl Collection for VMCollection { unimplemented!() } - fn sweep(addr: Address) { - unsafe { - mmtk_delete_object(addr) - } + fn update_object_archive() { + global_object_archive().update(); } fn process_weak_refs() { @@ -65,7 +64,3 @@ impl Collection for VMCollection { } } } - -extern { - fn mmtk_delete_object(addr: Address); -} diff --git a/mmtk/src/object_archive.rs b/mmtk/src/object_archive.rs index b9fe13a..dae5cc5 100644 --- a/mmtk/src/object_archive.rs +++ b/mmtk/src/object_archive.rs @@ -1,13 +1,20 @@ use libc::c_void; +use mmtk::util::ObjectReference; use mmtk::util::address::Address; -use std::sync::RwLock; +use std::collections::HashMap; -const INITIAL_ARCHIVE_SIZE: usize = 10000; -const ADDITIONAL_ARCHIVE_SIZE: usize = 1000; + +lazy_static! { + pub static ref OBJECT_ARCHIVE: ObjectArchive = ObjectArchive::new(); +} + +pub fn global_object_archive() -> &'static mut ObjectArchive { + unsafe { &mut *(&OBJECT_ARCHIVE as &ObjectArchive as *const ObjectArchive as *mut ObjectArchive) } +} #[no_mangle] pub extern "C" fn tph_archive_new() -> *const c_void { - Box::into_raw(Box::new(ObjectArchive::new())) as *const c_void + &OBJECT_ARCHIVE as &ObjectArchive as *const ObjectArchive as *const c_void } #[no_mangle] @@ -43,17 +50,6 @@ pub extern "C" fn tph_archive_inner_to_obj( res.to_mut_ptr() } -#[no_mangle] -pub extern "C" fn tph_archive_obj_to_isolate( - arch: *mut c_void, - obj_ptr: *mut c_void, -) -> *mut c_void { - let arch = unsafe { Box::from_raw(arch as *mut ObjectArchive) }; - let res = arch.object_to_isolate(Address::from_mut_ptr(obj_ptr)); - Box::into_raw(arch); - res.to_mut_ptr() -} - #[no_mangle] pub extern "C" fn tph_archive_obj_to_space(arch: *mut c_void, obj_ptr: *mut c_void) -> u8 { let arch = unsafe { Box::from_raw(arch as *mut ObjectArchive) }; @@ -84,162 +80,87 @@ pub extern "C" fn tph_archive_remove(arch: *mut c_void, obj_ptr: *mut c_void) { Box::into_raw(arch); } +#[derive(Default)] pub struct ObjectArchive { - sorted_addr_list: RwLock>, - isolate_list: RwLock>, - space_list: Vec, + untagged_objects: Vec, + space_map: HashMap, iter_pos: usize, iter_len: usize, } impl ObjectArchive { pub fn new() -> ObjectArchive { - ObjectArchive { - sorted_addr_list: RwLock::new(Vec::with_capacity(INITIAL_ARCHIVE_SIZE)), - isolate_list: RwLock::new(Vec::with_capacity(INITIAL_ARCHIVE_SIZE)), - space_list: Vec::with_capacity(INITIAL_ARCHIVE_SIZE), - iter_pos: 0usize, - iter_len: 0usize, - } + Default::default() } - pub fn insert_object(&mut self, obj_addr: Address, isolate: Address, space: u8) { - let mut lst = match self.sorted_addr_list.write() { - Ok(res) => res, - Err(err) => { - panic!("OA.insert: LST LOCK ACQ failed with err: {:#?}", err); - } - }; - let mut iso_lst = match self.isolate_list.write() { - Ok(res) => res, - Err(err) => { - panic!("OA.insert: ISO LOCK ACQ failed with err: {:#?}", err); - } + pub fn insert_object(&mut self, addr: Address, _isolate: Address, space: u8) { + let untagged_object = unsafe { + Address::from_usize(addr.as_usize() & !0b11).to_object_reference() }; - - assert_eq!(lst.len(), iso_lst.len()); - - if lst.capacity() == lst.len() { - lst.reserve(ADDITIONAL_ARCHIVE_SIZE); - iso_lst.reserve(ADDITIONAL_ARCHIVE_SIZE); - self.space_list.reserve(ADDITIONAL_ARCHIVE_SIZE); - } - match lst.binary_search(&obj_addr.as_usize()) { - Ok(_) => { - debug!("OA.insert: Object {:?} already archived", obj_addr); - } + self.space_map.insert(untagged_object, space); + match self.untagged_objects.binary_search_by(|o| o.to_address().cmp(&untagged_object.to_address())) { + Ok(_) => unreachable!(), Err(idx) => { - lst.insert(idx, obj_addr.as_usize()); - iso_lst.insert(idx, isolate.as_usize()); - self.space_list.insert(idx, space); + self.untagged_objects.insert(idx, untagged_object); } - }; + } } - pub fn remove_object(&mut self, obj_addr: Address) { - let mut lst = match self.sorted_addr_list.write() { - Ok(res) => res, - Err(err) => { - panic!("OA.remove: LST LOCK ACQ failed with err: {:#?}", err); - } - }; - let mut iso_lst = match self.isolate_list.write() { - Ok(res) => res, - Err(err) => { - panic!("OA.remove: ISO LOCK ACQ failed with err: {:#?}", err); - } - }; - assert_eq!(lst.len(), iso_lst.len()); - - let idx = match lst.binary_search(&obj_addr.as_usize()) { - Ok(idx) => idx, - Err(_) => { - panic!("OA.remove: Object {:?} not archived!", obj_addr); - } - }; - lst.remove(idx); - iso_lst.remove(idx); - self.space_list.remove(idx); + pub fn remove_object(&mut self, addr: Address) { + let untagged_object = unsafe { Address::from_usize(addr.as_usize() & !0b11).to_object_reference() }; + self.space_map.remove(&untagged_object); + let index = self.untagged_objects.iter().position(|x| *x == untagged_object).unwrap(); + self.untagged_objects.remove(index); } pub fn inner_addr_to_object(&self, inner_addr: Address) -> Address { - let lst = match self.sorted_addr_list.read() { - Ok(res) => res, - Err(err) => { - panic!("OA.inner_addr_to_object: LOCK ACQ failed: {:#?}", err); - } - }; - let idx = match lst.binary_search(&inner_addr.as_usize()) { + let idx = match self.untagged_objects.binary_search_by(|o| o.to_address().cmp(&inner_addr)) { Ok(idx) => idx, Err(idx) => idx - 1, }; - unsafe { Address::from_usize(lst[idx]) } + self.untagged_objects[idx].to_address() } - pub fn object_to_isolate(&self, obj_addr: Address) -> Address { - let lst = match self.sorted_addr_list.read() { - Ok(res) => res, - Err(err) => { - panic!("OA.object_to_isolate: LST LOCK ACQ failed: {:#?}", err); - } - }; - let iso_lst = match self.isolate_list.read() { - Ok(res) => res, - Err(err) => { - panic!("OA.object_to_isolate: ISO LOCK ACQ failed: {:#?}", err); - } - }; - assert_eq!(lst.len(), iso_lst.len()); - let idx = match lst.binary_search(&obj_addr.as_usize()) { - Ok(idx) => idx, - Err(idx) => idx - 1, - }; - unsafe { Address::from_usize(iso_lst[idx]) } + pub fn object_to_space(&self, mut obj_addr: Address) -> u8 { + obj_addr = unsafe { Address::from_usize(obj_addr.as_usize() & !0b11) }; + debug_assert_eq!(obj_addr.as_usize() & 0b11, 0); + unsafe { self.space_map[&obj_addr.to_object_reference()] } } - pub fn object_to_space(&self, obj_addr: Address) -> u8 { - let lst = match self.sorted_addr_list.read() { - Ok(res) => res, - Err(err) => { - panic!("OA.object_to_isolate: LST LOCK ACQ failed: {:#?}", err); + pub fn update(&mut self) { + let mut new_objects = vec![]; + let mut new_space_map = HashMap::new(); + for object in &self.untagged_objects { + debug_assert_eq!(object.to_address().as_usize() & 0b11, 0); + if object.is_live() { + let new_object = object.get_forwarded_object().unwrap_or(*object); + debug_assert_eq!(new_object.to_address().as_usize() & 0b11, 0); + new_objects.push(new_object); + new_space_map.insert(new_object, self.space_map[&object]); } - }; - let idx = match lst.binary_search(&obj_addr.as_usize()) { - Ok(idx) => idx, - Err(idx) => idx - 1, - }; - self.space_list[idx] + } + new_objects.sort_by(|a, b| a.to_address().cmp(&b.to_address())); + self.untagged_objects = new_objects; + self.space_map = new_space_map; } pub fn reset_iterator(&mut self) { - let lst = match self.sorted_addr_list.read() { - Ok(res) => res, - Err(err) => { - panic!("OA.reset_iterator: LOCK ACQ failed: {:#?}", err); - } - }; self.iter_pos = 0; - self.iter_len = lst.len(); + self.iter_len = self.untagged_objects.len(); } pub fn next_object(&mut self) -> Address { - let lst = match self.sorted_addr_list.read() { - Ok(res) => res, - Err(err) => { - panic!("OA.inner_addr_to_object: LOCK ACQ failed: {:#?}", err); - } - }; - if self.iter_len != lst.len() { + if self.iter_len != self.untagged_objects.len() { warn!( "ObjectArchive changed from {} to {}.", self.iter_len, - lst.len() + self.untagged_objects.len() ); - self.iter_len = lst.len(); + self.iter_len = self.untagged_objects.len(); } self.iter_pos += 1; - if self.iter_pos < lst.len() { - unsafe { Address::from_usize(lst[self.iter_pos]) } + if self.iter_pos < self.untagged_objects.len() { + self.untagged_objects[self.iter_pos].to_address() } else { unsafe { Address::zero() } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index d994f0b..ea607c1 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -30,7 +30,7 @@ impl Scanning for VMScanning { fn scan_objects>( objects: &[ObjectReference], - worker: &mut GCWorker, + _worker: &mut GCWorker, ) { unsafe { let buf = objects.as_ptr(); @@ -86,7 +86,7 @@ impl> GCWork for ScanAndForwardRoots { static mut ROOT_OBJECTS: Vec = Vec::new(); -fn flush_roots>(worker: &mut GCWorker) { +fn flush_roots>(_worker: &mut GCWorker) { let mut buf = vec![]; unsafe { std::mem::swap(&mut buf, &mut ROOT_OBJECTS); } let scan_objects_work = mmtk::scheduler::gc_work::ScanObjects::::new(buf, false); diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 77ef6b8..174f633 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. #include "mmtk.h" +#include "src/heap/heap-inl.h" namespace v8 { namespace internal { @@ -49,19 +50,6 @@ inline void CheckMutator(Heap* tph) { is_mutator = true; } -MMTk_Heap GetMMTkHeap(Address object_pointer) { - for (size_t i = 0; i < tph_data_list->size(); i++) - { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - void* ptr = tph_archive_obj_to_isolate( - tph_data_->archive(), reinterpret_cast(object_pointer)); - if (ptr != nullptr) { - return tph_data_->mmtk_heap(); - } - } - UNREACHABLE(); -} - static std::atomic_bool IsolateCreated { false }; #define GB (1ull << 30) @@ -90,16 +78,7 @@ std::unique_ptr Heap::New(v8::internal::Isolate* isolate) { } v8::internal::Isolate* Heap::GetIsolate(Address object_pointer) { - for (size_t i = 0; i < tph_data_list->size(); i++) - { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - void* ptr = tph_archive_obj_to_isolate( - tph_data_->archive(), reinterpret_cast(object_pointer)); - if (ptr != nullptr) { - return reinterpret_cast(ptr); - } - } - UNREACHABLE(); + return v8_heap->isolate(); } @@ -187,19 +166,6 @@ HeapObject Heap::NextObject() { return HeapObject(); } } -extern "C" { - void mmtk_delete_object(Address addr) { - for (auto tph_data : *tph_data_list) { - auto space = AllocationSpace(tph_archive_obj_to_space(tph_data->archive(), reinterpret_cast(addr))); - if (space == kNoSpace) continue; - tph_archive_remove(tph_data->archive(), (void*) addr); - return; - } - printf("Unknown object %p\n", (void*) addr); - UNREACHABLE(); - } - -} } } // namespace internal diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index f9419fb..210890e 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -104,7 +104,6 @@ extern void tph_archive_remove(void* arch, void* object); extern void tph_archive_iter_reset(void* arch); extern void* tph_archive_iter_next(void* arch); extern void* tph_archive_inner_to_obj(void* arch, void* inner_ptr); -extern void* tph_archive_obj_to_isolate(void* arch, void* obj_ptr); extern uint8_t tph_archive_obj_to_space(void* arch, void* obj_ptr); extern int mmtk_in_space(void* mmtk, void* object, size_t space); From 07ffa95c050791135b5a29cb08b9f1bdad75fef7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 27 May 2021 15:57:05 +1000 Subject: [PATCH 21/65] Fix map and immovable object allocation --- mmtk/src/lib.rs | 4 ++- mmtk/src/scanning.rs | 43 ++++++++++++++++++++++--- v8/third_party/heap/mmtk/mmtk.cc | 11 ++++++- v8/third_party/heap/mmtk/mmtk.h | 3 +- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 4 +-- v8/third_party/heap/mmtk/mmtkVisitors.h | 10 ++++-- 6 files changed, 64 insertions(+), 11 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2a5c849..74b6c19 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -1,4 +1,5 @@ #![feature(vec_into_raw_parts)] +#![feature(thread_local)] extern crate libc; extern crate mmtk; @@ -34,6 +35,7 @@ pub struct NewBuffer { type ProcessEdgesFn = *const extern "C" fn(buf: *mut Address, size: usize, cap: usize) -> NewBuffer; type TraceRootFn = *const extern "C" fn(slot: Address, ctx: &'static mut GCWorker) -> Address; +type TraceFieldFn = *const extern "C" fn(slot: Address, ctx: &'static mut GCWorker) -> Address; #[repr(C)] pub struct V8_Upcalls { @@ -52,7 +54,7 @@ pub struct V8_Upcalls { pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut Mutator, pub is_mutator: extern "C" fn(tls: VMThread) -> bool, pub scan_roots: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void), - pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn), + pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn, trace_field: TraceFieldFn, context: *mut c_void), pub process_weak_refs: extern "C" fn(), } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index ea607c1..9ba0bc9 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -30,12 +30,18 @@ impl Scanning for VMScanning { fn scan_objects>( objects: &[ObjectReference], - _worker: &mut GCWorker, + worker: &mut GCWorker, ) { unsafe { - let buf = objects.as_ptr(); - let len = objects.len(); - ((*UPCALLS).scan_objects)(buf, len, create_process_edges_work:: as _); + debug_assert!(OBJECTS_TO_SCAN.is_empty()); + OBJECTS_TO_SCAN = objects.to_vec(); + while !OBJECTS_TO_SCAN.is_empty() { + let objects = OBJECTS_TO_SCAN.clone(); + OBJECTS_TO_SCAN = vec![]; + let buf = objects.as_ptr(); + let len = objects.len(); + ((*UPCALLS).scan_objects)(buf, len, create_process_edges_work:: as _, trace_slot:: as _, worker as *mut _ as _); + } } } @@ -84,6 +90,7 @@ impl> GCWork for ScanAndForwardRoots { } } +#[thread_local] static mut ROOT_OBJECTS: Vec = Vec::new(); fn flush_roots>(_worker: &mut GCWorker) { @@ -128,6 +135,34 @@ pub(crate) extern "C" fn trace_root>(slot: Address, new_obj } +#[thread_local] +static mut OBJECTS_TO_SCAN: Vec = Vec::new(); + +pub(crate) extern "C" fn trace_slot>(slot: Address, worker: &'static mut GCWorker) -> ObjectReference { + let obj: ObjectReference = unsafe { slot.load() }; + let tag = obj.to_address().as_usize() & 0b11usize; + let mut w = W::new(vec![], false, &SINGLETON); + w.set_worker(worker); + let object_untagged = unsafe { + Address::from_usize(obj.to_address().as_usize() & !0b11usize).to_object_reference() + }; + let new_obj = w.trace_object(object_untagged); + if W::OVERWRITE_REFERENCE { + unsafe { + slot.store((new_obj.to_address().as_usize() & !0b11) | tag); + } + } + unsafe { + if OBJECTS_TO_SCAN.is_empty() { + OBJECTS_TO_SCAN.reserve(W::CAPACITY); + } + } + for o in &w.nodes { + unsafe { OBJECTS_TO_SCAN.push(*o); } + } + new_obj +} + pub(crate) extern "C" fn create_process_edges_work>( ptr: *mut Address, length: usize, diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 174f633..6d3b4f4 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -85,10 +85,13 @@ v8::internal::Isolate* Heap::GetIsolate(Address object_pointer) { // Address space in Rust is statically from 0x60000000 - 0xb0000000 AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, AllocationAlignment align) { CheckMutator(this); + if (!initialization_finished_ && type == AllocationType::kOld) { + type = AllocationType::kMap; + } TPHData* tph_data_ = get_tph_data(this); bool large_object = size_in_bytes > kMaxRegularHeapObjectSize; size_t align_bytes = (type == AllocationType::kCode) ? kCodeAlignment : (align == kWordAligned) ? kSystemPointerSize : (align == kDoubleAligned) ? kDoubleSize : kSystemPointerSize; - int space = (type == AllocationType::kCode) ? 3 : (type == AllocationType::kReadOnly) ? 4 : (large_object) ? 2 : 0; + int space = (type == AllocationType::kCode) ? 3 : (type == AllocationType::kReadOnly) ? 4 : (large_object || type == AllocationType::kMap) ? 2 : 0; Address result = reinterpret_cast
(alloc(tph_mutator_, size_in_bytes, align_bytes, 0, space)); AllocationSpace allocation_space; @@ -96,6 +99,8 @@ AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, Alloc allocation_space = large_object ? CODE_LO_SPACE : CODE_SPACE; } else if (type == AllocationType::kReadOnly) { allocation_space = RO_SPACE; + } else if (type == AllocationType::kMap) { + allocation_space = MAP_SPACE; } else { allocation_space = large_object ? LO_SPACE : OLD_SPACE; } @@ -131,6 +136,10 @@ bool Heap::InSpace(Address address, AllocationSpace allocation_space) { UNREACHABLE(); } +bool Heap::IsImmovable(HeapObject object) { + return mmtk_is_movable(object) == 0; +} + bool Heap::InOldSpace(Address address) { return InSpace(address, OLD_SPACE); } diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 210890e..d3da45c 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -116,6 +116,7 @@ typedef struct { typedef NewBuffer (*ProcessEdgesFn)(void** buf, size_t len, size_t cap); typedef void* (*TraceRootFn)(void* slot, void* ctx); +typedef void* (*TraceFieldFn)(void* slot, void* ctx); typedef struct { void (*stop_all_mutators) (void *tls); @@ -133,7 +134,7 @@ typedef struct { void* (*get_mmtk_mutator) (void* tls); bool (*is_mutator) (void* tls); void (*scan_roots) (TraceRootFn process_edges, void* context); - void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges); + void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context); void (*process_weak_refs) (); } V8_Upcalls; diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 52ff8ae..c17697e 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -132,8 +132,8 @@ static void mmtk_scan_roots(TraceRootFn trace_root, void* context) { v8_heap->IterateRoots(&root_visitor, {}); } -static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges) { - MMTkEdgeVisitor visitor(v8_heap, process_edges); +static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context) { + MMTkEdgeVisitor visitor(v8_heap, process_edges, trace_field, context); for (size_t i = 0; i < count; i++) { auto ptr = *(objects + i); DCHECK_EQ(((Address) ptr) & 1, 0); diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index f5c67d1..e98627f 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -52,7 +52,7 @@ class MMTkRootVisitor: public RootVisitor { class MMTkEdgeVisitor: public ObjectVisitor { public: - explicit MMTkEdgeVisitor(v8::internal::Heap* heap, ProcessEdgesFn process_edges): heap_(heap), process_edges_(process_edges) { + explicit MMTkEdgeVisitor(v8::internal::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context): heap_(heap), process_edges_(process_edges), trace_field_(trace_field), context_(context) { USE(heap_); NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; @@ -80,7 +80,11 @@ class MMTkEdgeVisitor: public ObjectVisitor { } virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { - AddNonMovingEdge(rinfo->target_object()); + auto o = rinfo->target_object(); + trace_field_((void*) &o, context_); + if (o != rinfo->target_object()) { + rinfo->set_target_object(heap_, o); + } } virtual void VisitMapPointer(HeapObject host) override final { @@ -124,6 +128,8 @@ class MMTkEdgeVisitor: public ObjectVisitor { v8::internal::Heap* heap_; ProcessEdgesFn process_edges_; + TraceFieldFn trace_field_; + void* context_; void** buffer_ = nullptr; size_t cap_ = 0; size_t cursor_ = 0; From 167584a348008fcc90f4df282fbc916fa27af6a3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 7 Jun 2021 13:31:52 +1000 Subject: [PATCH 22/65] WIP --- mmtk/Cargo.toml | 2 +- mmtk/src/object_archive.rs | 5 +++-- v8/third_party/heap/mmtk/mmtk.cc | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index cdef8ac..7e67c8a 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [profile.dev] panic = "abort" -# opt-level = 3 +opt-level = 3 [profile.release] panic = "abort" diff --git a/mmtk/src/object_archive.rs b/mmtk/src/object_archive.rs index dae5cc5..54f7f24 100644 --- a/mmtk/src/object_archive.rs +++ b/mmtk/src/object_archive.rs @@ -158,9 +158,10 @@ impl ObjectArchive { ); self.iter_len = self.untagged_objects.len(); } - self.iter_pos += 1; if self.iter_pos < self.untagged_objects.len() { - self.untagged_objects[self.iter_pos].to_address() + let o = self.untagged_objects[self.iter_pos].to_address(); + self.iter_pos += 1; + o } else { unsafe { Address::zero() } } diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 6d3b4f4..56465d1 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -67,12 +67,10 @@ std::unique_ptr Heap::New(v8::internal::Isolate* isolate) { DCHECK_WITH_MSG(!isolate_created, "Multiple isolates are not supported."); fprintf(stderr, "New Isolate: %lx\n", (unsigned long) isolate); MMTk_Heap new_heap = v8_new_heap(&mmtk_upcalls, FIXED_HEAP_SIZE); - tph_mutator_ = reinterpret_cast(bind_mutator(new_heap, &tph_mutator_)); // FIXME code_range_ = base::AddressRegion(0x60000000, (0xb0000000- 0x60000000)); // isolate->AddCodeRange(code_range_.begin(), code_range_.size()); auto v8_tph = std::make_unique(); TPHData* tph_data = new TPHData(v8_tph.get(), new_heap, isolate, tph_archive_new()); - tph_mutator_->tph_data = tph_data; tph_data_list->push_back(tph_data); return v8_tph; } @@ -121,6 +119,7 @@ const v8::base::AddressRegion& Heap::GetCodeRange() { } bool Heap::CollectGarbage() { + v8_heap->increase_gc_count(); handle_user_collection_request(get_tph_data(this)->mmtk_heap(), (void*) 0); return true; } From 29bebe2a336cc95c39b63c77750289cc3771257e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 7 Jun 2021 17:27:00 +1000 Subject: [PATCH 23/65] Main thread tasks --- mmtk/src/scanning.rs | 1 - v8/third_party/heap/mmtk/log.h | 11 ++ v8/third_party/heap/mmtk/main-thread-sync.h | 96 ++++++++++++++++++ v8/third_party/heap/mmtk/mmtk.cc | 3 +- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 106 ++++++++++---------- v8/third_party/heap/mmtk/mmtkVisitors.h | 10 +- 6 files changed, 166 insertions(+), 61 deletions(-) create mode 100644 v8/third_party/heap/mmtk/log.h create mode 100644 v8/third_party/heap/mmtk/main-thread-sync.h diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 9ba0bc9..37dfb58 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -90,7 +90,6 @@ impl> GCWork for ScanAndForwardRoots { } } -#[thread_local] static mut ROOT_OBJECTS: Vec = Vec::new(); fn flush_roots>(_worker: &mut GCWorker) { diff --git a/v8/third_party/heap/mmtk/log.h b/v8/third_party/heap/mmtk/log.h new file mode 100644 index 0000000..84a0382 --- /dev/null +++ b/v8/third_party/heap/mmtk/log.h @@ -0,0 +1,11 @@ +#ifndef MMTK_LOG_H +#define MMTK_LOG_H + +#define ENABLE_LOGGING 1 + +#define MMTK_LOG(...) \ + if (ENABLE_LOGGING) { \ + fprintf(stderr, __VA_ARGS__); \ + } + +#endif // MMTK_LOG_H \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/main-thread-sync.h b/v8/third_party/heap/mmtk/main-thread-sync.h new file mode 100644 index 0000000..f27fc1b --- /dev/null +++ b/v8/third_party/heap/mmtk/main-thread-sync.h @@ -0,0 +1,96 @@ +#ifndef MAIN_THREAD_SYNC_H +#define MAIN_THREAD_SYNC_H + +#include +#include +#include +#include +#include "src/heap/safepoint.h" +#include "log.h" + +namespace mmtk { + +class MainThreadSynchronizer; + +class TaskAwaiter { + std::mutex m_ {}; + std::condition_variable cv_ {}; + bool task_completed_ = false; + + void Complete() { + std::unique_lock lock(m_); + task_completed_ = true; + cv_.notify_all(); + } + + friend class MainThreadSynchronizer; + + public: + void Wait() { + std::unique_lock lock(m_); + while (!task_completed_) cv_.wait(lock); + } +}; + +class MainThreadSynchronizer { + std::mutex m_ {}; + std::condition_variable cv_ {}; + bool gc_in_progress_ = false; + std::deque> main_thread_tasks_ {}; + std::unique_ptr safepoint_scope_ = nullptr; + + void DrainTasks() { + while (!main_thread_tasks_.empty()) { + MMTK_LOG("[main-thread] Run One Task\n"); + auto task = main_thread_tasks_.front(); + main_thread_tasks_.pop_front(); + task(); + } + } + + public: + void Block() { + MMTK_LOG("[main-thread] Blocked\n"); + gc_in_progress_ = true; + std::unique_lock lock(m_); + while (gc_in_progress_) { + DrainTasks(); + MMTK_LOG("[main-thread] Sleep\n"); + cv_.wait(lock); + MMTK_LOG("[main-thread] Wake\n"); + } + DrainTasks(); + MMTK_LOG("[main-thread] Resumed\n"); + } + + void WakeUp() { + std::unique_lock lock(m_); + gc_in_progress_ = false; + cv_.notify_all(); + } + + void RunMainThreadTask(std::function task) { + auto awaiter = std::make_unique(); + { + std::unique_lock lock(m_); + main_thread_tasks_.push_back([task, &awaiter]() { + task(); + awaiter->Complete(); + }); + cv_.notify_all(); + } + awaiter->Wait(); + } + + void EnterSafepoint(v8::internal::Heap* heap) { + safepoint_scope_.reset(new v8::internal::SafepointScope(heap)); + } + + void ExitSafepoint() { + safepoint_scope_ = nullptr; + } +}; + +} + +#endif // MAIN_THREAD_SYNC_H \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 56465d1..5c84609 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -4,6 +4,7 @@ #include "mmtk.h" #include "src/heap/heap-inl.h" +#include "log.h" namespace v8 { namespace internal { @@ -65,7 +66,7 @@ std::unique_ptr Heap::New(v8::internal::Isolate* isolate) { // MMTK current default maximum heap size is 1GB. auto isolate_created = IsolateCreated.exchange(true); DCHECK_WITH_MSG(!isolate_created, "Multiple isolates are not supported."); - fprintf(stderr, "New Isolate: %lx\n", (unsigned long) isolate); + MMTK_LOG("New Isolate: %lx\n", (unsigned long) isolate); MMTk_Heap new_heap = v8_new_heap(&mmtk_upcalls, FIXED_HEAP_SIZE); // FIXME code_range_ = base::AddressRegion(0x60000000, (0xb0000000- 0x60000000)); // isolate->AddCodeRange(code_range_.begin(), code_range_.size()); diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index c17697e..9b47a8a 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -1,7 +1,4 @@ -#include "src/base/logging.h" #include "mmtkUpcalls.h" -#include -#include #include "src/objects/slots-inl.h" #include "src/heap/safepoint.h" #include "src/heap/array-buffer-sweeper.h" @@ -9,6 +6,8 @@ #include "src/execution/frames-inl.h" #include "src/regexp/regexp.h" #include "mmtkVisitors.h" +#include "main-thread-sync.h" +#include "log.h" namespace v8 { namespace internal { @@ -16,76 +15,73 @@ namespace third_party_heap { extern v8::internal::Heap* v8_heap; -std::mutex m; -std::condition_variable* cv = new std::condition_variable(); -bool gcInProgress = false; +mmtk::MainThreadSynchronizer* main_thread_synchronizer = new mmtk::MainThreadSynchronizer(); static void mmtk_stop_all_mutators(void *tls) { - fprintf(stderr, "mmtk_stop_all_mutators\n"); - v8_heap->safepoint()->EnterSafepointScope(ThreadKind::kBackground); - fprintf(stderr, "mmtk_stop_all_mutators: heap verify start\n"); - MMTkHeapVerifier::Verify(v8_heap); - fprintf(stderr, "mmtk_stop_all_mutators: heap verify end\n"); - - v8_heap->isolate()->descriptor_lookup_cache()->Clear(); - RegExpResultsCache::Clear(v8_heap->string_split_cache()); - RegExpResultsCache::Clear(v8_heap->regexp_multiple_cache()); - // v8_heap->FlushNumberStringCache(); - int len = v8_heap->number_string_cache().length(); - for (int i = 0; i < len; i++) { - v8_heap->number_string_cache().set_undefined(i); - } + MMTK_LOG("[mmtk_stop_all_mutators] START\n"); + main_thread_synchronizer->RunMainThreadTask([=]() { + main_thread_synchronizer->EnterSafepoint(v8_heap); + MMTK_LOG("[mmtk_stop_all_mutators] Verify heap\n"); + MMTkHeapVerifier::Verify(v8_heap); + MMTK_LOG("[mmtk_stop_all_mutators] Flush cache\n"); + v8_heap->isolate()->descriptor_lookup_cache()->Clear(); + RegExpResultsCache::Clear(v8_heap->string_split_cache()); + RegExpResultsCache::Clear(v8_heap->regexp_multiple_cache()); + // v8_heap->FlushNumberStringCache(); + int len = v8_heap->number_string_cache().length(); + for (int i = 0; i < len; i++) { + v8_heap->number_string_cache().set_undefined(i); + } + }); + MMTK_LOG("[mmtk_stop_all_mutators] END\n"); } static void mmtk_process_weak_refs() { - // fprintf(stderr, "mmtk_process_weak_refs\n"); - MMTkWeakObjectRetainer retainer; - v8_heap->ProcessAllWeakReferences(&retainer); + main_thread_synchronizer->RunMainThreadTask([=]() { + MMTK_LOG("[mmtk_process_weak_refs]\n"); + MMTkWeakObjectRetainer retainer; + v8_heap->ProcessAllWeakReferences(&retainer); + }); } static void mmtk_resume_mutators(void *tls) { - fprintf(stderr, "mmtk_resume_mutators: heap verify start\n"); - MMTkHeapVerifier::Verify(v8_heap); - fprintf(stderr, "mmtk_resume_mutators: heap verify end\n"); - fprintf(stderr, "mmtk_resume_mutators\n"); - // v8_heap->array_buffer_sweeper()->RequestSweepFull(); - - v8_heap->isolate()->inner_pointer_to_code_cache()->Flush(); - // The stub caches are not traversed during GC; clear them to force - // their lazy re-initialization. This must be done after the - // GC, because it relies on the new address of certain old space - // objects (empty string, illegal builtin). - v8_heap->isolate()->load_stub_cache()->Clear(); - v8_heap->isolate()->store_stub_cache()->Clear(); - // Some code objects were marked for deoptimization during the GC. - // Deoptimizer::DeoptimizeMarkedCode(v8_heap->isolate()); - - v8_heap->safepoint()->LeaveSafepointScope(); - std::unique_lock lock(m); - gcInProgress = false; - cv->notify_all(); + MMTK_LOG("[mmtk_resume_mutators] START\n"); + main_thread_synchronizer->RunMainThreadTask([=]() { + MMTK_LOG("[mmtk_resume_mutators] Verify heap\n"); + MMTkHeapVerifier::Verify(v8_heap); + MMTK_LOG("[mmtk_resume_mutators] Flush cache\n"); + v8_heap->isolate()->inner_pointer_to_code_cache()->Flush(); + // The stub caches are not traversed during GC; clear them to force + // their lazy re-initialization. This must be done after the + // GC, because it relies on the new address of certain old space + // objects (empty string, illegal builtin). + v8_heap->isolate()->load_stub_cache()->Clear(); + v8_heap->isolate()->store_stub_cache()->Clear(); + // v8_heap->array_buffer_sweeper()->RequestSweepFull(); + // Some code objects were marked for deoptimization during the GC. + // Deoptimizer::DeoptimizeMarkedCode(v8_heap->isolate()); + main_thread_synchronizer->ExitSafepoint(); + }); + main_thread_synchronizer->WakeUp(); + MMTK_LOG("[mmtk_resume_mutators] END\n"); } static void mmtk_spawn_collector_thread(void* tls, void* ctx) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } - static void mmtk_block_for_gc() { - gcInProgress = true; - std::unique_lock lock(m); - cv->wait(lock, []{ return !gcInProgress; }); - fprintf(stderr, "mmtk_block_for_gc end\n"); + main_thread_synchronizer->Block(); } static void* mmtk_get_mmtk_mutator(void* tls) { - UNIMPLEMENTED(); + UNIMPLEMENTED(); } extern thread_local bool is_mutator; static bool mmtk_is_mutator(void* tls) { - return is_mutator; + return is_mutator; } extern std::vector* all_mutators; @@ -97,7 +93,7 @@ static void* mmtk_get_next_mutator() { } static void mmtk_reset_mutator_iterator() { - index = 0; + index = 0; } @@ -128,8 +124,10 @@ static size_t mmtk_get_object_size(void* object) { } static void mmtk_scan_roots(TraceRootFn trace_root, void* context) { - MMTkRootVisitor root_visitor(v8_heap, trace_root, context); - v8_heap->IterateRoots(&root_visitor, {}); + main_thread_synchronizer->RunMainThreadTask([=]() { + MMTkRootVisitor root_visitor(v8_heap, trace_root, context); + v8_heap->IterateRoots(&root_visitor, {}); + }); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context) { diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index e98627f..9dc117d 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -1,8 +1,8 @@ #ifndef MMTK_VISITORS_H #define MMTK_VISITORS_H -#include "src/base/logging.h" #include "mmtkUpcalls.h" +#include "log.h" #include #include #include "src/objects/slots-inl.h" @@ -223,19 +223,19 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { class MMTkWeakObjectRetainer: public WeakObjectRetainer { public: virtual Object RetainAs(Object object) override final { - // fprintf(stderr, "RetainAs %p\n", (void*) object.ptr()); + // LOG("RetainAs %p\n", (void*) object.ptr()); if (object == Object()) return object; HeapObject heap_object = HeapObject::cast(object); if (third_party_heap::Heap::IsValidHeapObject(heap_object)) { auto f = mmtk_get_forwarded_object(heap_object); if (f != nullptr) { - // fprintf(stderr, "%p -> %p\n", (void*) object.ptr(), (void*) f); + // LOG("%p -> %p\n", (void*) object.ptr(), (void*) f); return Object((Address) f); } - // fprintf(stderr, "%p is dead 1 \n", (void*) object.ptr()); + // LOG("%p is dead 1 \n", (void*) object.ptr()); return object; } else { - // fprintf(stderr, "%p is dead 2 \n", (void*) object.ptr()); + // LOG("%p is dead 2 \n", (void*) object.ptr()); return Object(); } } From 5ceadf6c47654a7a6faf7dfb577e669ca2b329b8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 9 Jun 2021 14:55:30 +1000 Subject: [PATCH 24/65] Fix `OnMoveEvent` --- mmtk/src/lib.rs | 1 + mmtk/src/object_model.rs | 1 + v8/third_party/heap/mmtk/mmtk.h | 1 + v8/third_party/heap/mmtk/mmtkUpcalls.cc | 7 +++++++ 4 files changed, 10 insertions(+) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 74b6c19..880b37a 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -56,6 +56,7 @@ pub struct V8_Upcalls { pub scan_roots: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void), pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn, trace_field: TraceFieldFn, context: *mut c_void), pub process_weak_refs: extern "C" fn(), + pub on_move_event: extern "C" fn(from: ObjectReference, to: ObjectReference, size: usize), } pub static mut UPCALLS: *const V8_Upcalls = null_mut(); diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index fe6c3da..3617b22 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -26,6 +26,7 @@ impl ObjectModel for VMObjectModel { } let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, unsafe { Address::zero() }, bytes, allocator); + unsafe { ((*UPCALLS).on_move_event)(from, to_obj, bytes) }; to_obj } diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index d3da45c..67e9d4c 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -136,6 +136,7 @@ typedef struct { void (*scan_roots) (TraceRootFn process_edges, void* context); void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context); void (*process_weak_refs) (); + void (*on_move_event) (void* from, void* to, size_t size); } V8_Upcalls; /** diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 9b47a8a..6a7ad0d 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -123,6 +123,12 @@ static size_t mmtk_get_object_size(void* object) { return o.SizeFromMap(m); } +static void mmtk_on_move_event(void* from_address, void* to_address, size_t size) { + auto from = HeapObject::FromAddress((Address) from_address); + auto to = HeapObject::FromAddress((Address) to_address); + v8_heap->OnMoveEvent(to, from, (int) size); +} + static void mmtk_scan_roots(TraceRootFn trace_root, void* context) { main_thread_synchronizer->RunMainThreadTask([=]() { MMTkRootVisitor root_visitor(v8_heap, trace_root, context); @@ -158,6 +164,7 @@ V8_Upcalls mmtk_upcalls = { mmtk_scan_roots, mmtk_scan_objects, mmtk_process_weak_refs, + mmtk_on_move_event, }; } // namespace third_party_heap } // namespace internal From 6f4d0c90635f946f68ca42638fe75bf5132ea2ef Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 9 Jun 2021 17:03:34 +1000 Subject: [PATCH 25/65] Minor fix --- v8/third_party/heap/mmtk/mmtkVisitors.h | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtkVisitors.h index 9dc117d..1b70d89 100644 --- a/v8/third_party/heap/mmtk/mmtkVisitors.h +++ b/v8/third_party/heap/mmtk/mmtkVisitors.h @@ -76,7 +76,9 @@ class MMTkEdgeVisitor: public ObjectVisitor { virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); - AddNonMovingEdge(target); + DCHECK(!mmtk_is_movable(target)); + trace_field_((void*) &target, context_); + DCHECK_EQ(target, Code::GetCodeFromTargetAddress(rinfo->target_address())); } virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { @@ -104,14 +106,6 @@ class MMTkEdgeVisitor: public ObjectVisitor { } } - V8_INLINE void AddNonMovingEdge(HeapObject o) { - DCHECK(!mmtk_is_movable(o)); - // TODO: Don't malloc - HeapObject* edge = new HeapObject(); - *edge = o; - PushEdge((void*) edge); - } - V8_INLINE void PushEdge(void* edge) { buffer_[cursor_++] = edge; if (cursor_ >= cap_) flush(); From 3194334e87c1f3a8158fbd83b84a2835976f2e01 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 10 Jun 2021 12:33:54 +1000 Subject: [PATCH 26/65] Rename --- v8/third_party/heap/mmtk/{mmtkVisitors.h => mmtk-visitors.h} | 0 v8/third_party/heap/mmtk/mmtkUpcalls.cc | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename v8/third_party/heap/mmtk/{mmtkVisitors.h => mmtk-visitors.h} (100%) diff --git a/v8/third_party/heap/mmtk/mmtkVisitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h similarity index 100% rename from v8/third_party/heap/mmtk/mmtkVisitors.h rename to v8/third_party/heap/mmtk/mmtk-visitors.h diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 6a7ad0d..4142bf4 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -5,7 +5,7 @@ #include "src/deoptimizer/deoptimizer.h" #include "src/execution/frames-inl.h" #include "src/regexp/regexp.h" -#include "mmtkVisitors.h" +#include "mmtk-visitors.h" #include "main-thread-sync.h" #include "log.h" From c1368bed79115376aeaa83e5e0721a2d2821d762 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 10 Jun 2021 14:59:00 +1000 Subject: [PATCH 27/65] StringTable processing --- v8/third_party/heap/mmtk/mmtk.cc | 2 + v8/third_party/heap/mmtk/mmtkUpcalls.cc | 4 +- v8/third_party/heap/mmtk/weak-refs.h | 118 ++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 v8/third_party/heap/mmtk/weak-refs.h diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 4f02ed0..9a73d43 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -125,7 +125,9 @@ const v8::base::AddressRegion& Heap::GetCodeRange() { bool Heap::CollectGarbage() { v8_heap->increase_gc_count(); + v8_heap->SetGCState(v8::internal::Heap::MARK_COMPACT); handle_user_collection_request(get_tph_data(this)->mmtk_heap(), (void*) 0); + v8_heap->SetGCState(v8::internal::Heap::NOT_IN_GC); return true; } diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 4142bf4..be7a838 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -8,6 +8,7 @@ #include "mmtk-visitors.h" #include "main-thread-sync.h" #include "log.h" +#include "weak-refs.h" namespace v8 { namespace internal { @@ -39,8 +40,7 @@ static void mmtk_stop_all_mutators(void *tls) { static void mmtk_process_weak_refs() { main_thread_synchronizer->RunMainThreadTask([=]() { MMTK_LOG("[mmtk_process_weak_refs]\n"); - MMTkWeakObjectRetainer retainer; - v8_heap->ProcessAllWeakReferences(&retainer); + mmtk::global_weakref_processor->ClearNonLiveReferences(); }); } diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h new file mode 100644 index 0000000..c3a8377 --- /dev/null +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -0,0 +1,118 @@ +#ifndef MMTK_WEAK_REFS_H +#define MMTK_WEAK_REFS_H + +#include "src/heap/heap.h" +#include "src/objects/string-table.h" +#include "src/objects/visitors.h" +#include "mmtk-visitors.h" + +namespace v8 { +namespace internal { +namespace third_party_heap { + +extern v8::internal::Heap* v8_heap; + +} +} +} + +namespace mmtk { + +namespace i = v8::internal; + +class InternalizedStringTableCleaner : public i::RootVisitor { + public: + explicit InternalizedStringTableCleaner(i::Heap* heap) + : heap_(heap), pointers_removed_(0) {} + + void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override { + UNREACHABLE(); + } + + void VisitRootPointers(i::Root root, const char* description, i::OffHeapObjectSlot start, i::OffHeapObjectSlot end) override { + DCHECK_EQ(root, i::Root::kStringTable); + // Visit all HeapObject pointers in [start, end). + auto isolate = heap_->isolate(); + for (auto p = start; p < end; ++p) { + i::Object o = p.load(isolate); + if (o.IsHeapObject()) { + auto heap_object = i::HeapObject::cast(o); + DCHECK(!i::Heap::InYoungGeneration(heap_object)); + if (!is_live_object(reinterpret_cast(heap_object.address()))) { + pointers_removed_++; + // Set the entry to the_hole_value (as deleted). + p.store(i::StringTable::deleted_element()); + } + } + } + } + + int PointersRemoved() { return pointers_removed_; } + + private: + i::Heap* heap_; + int pointers_removed_; +}; + +class ExternalStringTableCleaner : public i::RootVisitor { + public: + explicit ExternalStringTableCleaner(i::Heap* heap) : heap_(heap) {} + + void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override { + // Visit all HeapObject pointers in [start, end). + auto the_hole = i::ReadOnlyRoots(heap_).the_hole_value(); + for (auto p = start; p < end; ++p) { + i::Object o = *p; + if (o.IsHeapObject()) { + auto heap_object = i::HeapObject::cast(o); + if (!is_live_object(reinterpret_cast(heap_object.address()))) { + if (o.IsExternalString()) { + heap_->FinalizeExternalString(i::String::cast(o)); + } else { + // The original external string may have been internalized. + DCHECK(o.IsThinString()); + } + // Set the entry to the_hole_value (as deleted). + p.store(the_hole); + } + } + } + } + + private: + i::Heap* heap_; +}; + + +class WeakRefs { + public: + static v8::internal::Heap* heap() { + return v8::internal::third_party_heap::v8_heap; + } + + void ClearNonLiveReferences() { + { + // TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR_STRING_TABLE); + + // Prune the string table removing all strings only pointed to by the + // string table. Cannot use string_table() here because the string + // table is marked. + v8::internal::StringTable* string_table = heap()->isolate()->string_table(); + InternalizedStringTableCleaner internalized_visitor(heap()); + string_table->DropOldData(); + string_table->IterateElements(&internalized_visitor); + string_table->NotifyElementsRemoved(internalized_visitor.PointersRemoved()); + + ExternalStringTableCleaner external_visitor(heap()); + heap()->UpdateExternalStringTable(&external_visitor); + } + v8::internal::third_party_heap::MMTkWeakObjectRetainer retainer; + heap()->ProcessAllWeakReferences(&retainer); + } +}; + +WeakRefs* global_weakref_processor = new WeakRefs(); + +} + +#endif // MMTK_WEAK_REFS_H \ No newline at end of file From 2bb3ccb2ef16603ad5621b94906b601ad0a84c03 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 10 Jun 2021 16:49:32 +1000 Subject: [PATCH 28/65] ClearNonLiveReferences support --- v8/third_party/heap/mmtk/mmtk-visitors.h | 98 +++-- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 3 + v8/third_party/heap/mmtk/weak-refs.h | 456 ++++++++++++++++++++++- 3 files changed, 526 insertions(+), 31 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 1b70d89..d9b6154 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -12,6 +12,8 @@ #include "src/objects/code-inl.h" #include "src/objects/map-inl.h" #include "src/codegen/assembler-inl.h" +#include "src/heap/objects-visiting-inl.h" +#include "weak-refs.h" #include @@ -19,6 +21,8 @@ namespace v8 { namespace internal { namespace third_party_heap { +namespace i = v8::internal; + class MMTkRootVisitor: public RootVisitor { public: explicit MMTkRootVisitor(v8::internal::Heap* heap, TraceRootFn trace_root, void* context): heap_(heap), trace_root_(trace_root), context_(context) { @@ -50,7 +54,7 @@ class MMTkRootVisitor: public RootVisitor { void* context_; }; -class MMTkEdgeVisitor: public ObjectVisitor { +class MMTkEdgeVisitor: public HeapVisitor { public: explicit MMTkEdgeVisitor(v8::internal::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context): heap_(heap), process_edges_(process_edges), trace_field_(trace_field), context_(context) { USE(heap_); @@ -64,6 +68,71 @@ class MMTkEdgeVisitor: public ObjectVisitor { if (buffer_ != NULL) { release_buffer(buffer_, cursor_, cap_); } + weak_objects_->transition_arrays.FlushToGlobal(task_id_); + weak_objects_->ephemeron_hash_tables.FlushToGlobal(task_id_); + weak_objects_->current_ephemerons.FlushToGlobal(task_id_); + weak_objects_->next_ephemerons.FlushToGlobal(task_id_); + weak_objects_->discovered_ephemerons.FlushToGlobal(task_id_); + weak_objects_->weak_references.FlushToGlobal(task_id_); + weak_objects_->js_weak_refs.FlushToGlobal(task_id_); + weak_objects_->weak_cells.FlushToGlobal(task_id_); + weak_objects_->weak_objects_in_code.FlushToGlobal(task_id_); + weak_objects_->bytecode_flushing_candidates.FlushToGlobal(task_id_); + weak_objects_->flushed_js_functions.FlushToGlobal(task_id_); + } + + V8_INLINE void VisitSharedFunctionInfo(Map map, SharedFunctionInfo shared_info) { + // If the SharedFunctionInfo has old bytecode, mark it as flushable, + // otherwise visit the function data field strongly. + if (shared_info.ShouldFlushBytecode(v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()))) { + weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); + } + } + + V8_INLINE void VisitJSFunction(Map map, JSFunction object) { + if (v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()) != BytecodeFlushMode::kDoNotFlushBytecode && object.NeedsResetDueToFlushedBytecode()) { + weak_objects_->flushed_js_functions.Push(task_id_, object); + } + } + + V8_INLINE void VisitTransitionArray(Map map, TransitionArray array) { + weak_objects_->transition_arrays.Push(task_id_, array); + } + + void VisitEphemeronHashTable(Map map, EphemeronHashTable table) { + weak_objects_->ephemeron_hash_tables.Push(task_id_, table); + for (InternalIndex i : table.IterateEntries()) { + // ObjectSlot key_slot = + // table.RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i)); + HeapObject key = HeapObject::cast(table.KeyAt(i)); + + // concrete_visitor()->SynchronizePageAccess(key); + // concrete_visitor()->RecordSlot(table, key_slot, key); + + // ObjectSlot value_slot = table.RawFieldOfElementAt(EphemeronHashTable::EntryToValueIndex(i)); + + // if (concrete_visitor()->marking_state()->IsBlackOrGrey(key)) { + // VisitPointer(table, value_slot); + // } else { + Object value_obj = table.ValueAt(i); + if (value_obj.IsHeapObject()) { + HeapObject value = HeapObject::cast(value_obj); + // concrete_visitor()->SynchronizePageAccess(value); + // concrete_visitor()->RecordSlot(table, value_slot, value); + // Revisit ephemerons with both key and value unreachable at end + // of concurrent marking cycle. + if (!mmtk::WeakRefs::is_live(value)) { + weak_objects_->discovered_ephemerons.Push(task_id_, Ephemeron{key, value}); + } + } + // } + } + } + + void VisitJSWeakRef(Map map, JSWeakRef weak_ref) { + if (weak_ref.target().IsHeapObject()) { + weak_objects_->js_weak_refs.Push(task_id_, weak_ref); + } } virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { @@ -127,6 +196,8 @@ class MMTkEdgeVisitor: public ObjectVisitor { void** buffer_ = nullptr; size_t cap_ = 0; size_t cursor_ = 0; + i::WeakObjects* weak_objects_ = mmtk::global_weakref_processor->weak_objects(); + int task_id_ = 1; }; @@ -213,31 +284,6 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { std::vector mark_stack_; }; - -class MMTkWeakObjectRetainer: public WeakObjectRetainer { - public: - virtual Object RetainAs(Object object) override final { - // LOG("RetainAs %p\n", (void*) object.ptr()); - if (object == Object()) return object; - HeapObject heap_object = HeapObject::cast(object); - if (third_party_heap::Heap::IsValidHeapObject(heap_object)) { - auto f = mmtk_get_forwarded_object(heap_object); - if (f != nullptr) { - // LOG("%p -> %p\n", (void*) object.ptr(), (void*) f); - return Object((Address) f); - } - // LOG("%p is dead 1 \n", (void*) object.ptr()); - return object; - } else { - // LOG("%p is dead 2 \n", (void*) object.ptr()); - return Object(); - } - } -}; - - - - } } } diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index be7a838..8dfccd1 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -3,6 +3,8 @@ #include "src/heap/safepoint.h" #include "src/heap/array-buffer-sweeper.h" #include "src/deoptimizer/deoptimizer.h" +#include "src/objects/embedder-data-array-inl.h" +#include "src/objects/js-collection-inl.h" #include "src/execution/frames-inl.h" #include "src/regexp/regexp.h" #include "mmtk-visitors.h" @@ -143,6 +145,7 @@ static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn proce DCHECK_EQ(((Address) ptr) & 1, 0); auto obj = HeapObject::FromAddress(((Address) ptr)); obj.Iterate(&visitor); + visitor.Visit(obj); } } diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index c3a8377..021a8ea 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -4,7 +4,7 @@ #include "src/heap/heap.h" #include "src/objects/string-table.h" #include "src/objects/visitors.h" -#include "mmtk-visitors.h" +#include "src/objects/transitions-inl.h" namespace v8 { namespace internal { @@ -19,6 +19,29 @@ extern v8::internal::Heap* v8_heap; namespace mmtk { namespace i = v8::internal; +namespace tph = v8::internal::third_party_heap; + + +class MMTkWeakObjectRetainer: public i::WeakObjectRetainer { + public: + virtual i::Object RetainAs(i::Object object) override final { + // LOG("RetainAs %p\n", (void*) object.ptr()); + if (object == i::Object()) return object; + auto heap_object = i::HeapObject::cast(object); + if (tph::Heap::IsValidHeapObject(heap_object)) { + auto f = mmtk_get_forwarded_object(heap_object); + if (f != nullptr) { + // LOG("%p -> %p\n", (void*) object.ptr(), (void*) f); + return i::Object((i::Address) f); + } + // LOG("%p is dead 1 \n", (void*) object.ptr()); + return object; + } else { + // LOG("%p is dead 2 \n", (void*) object.ptr()); + return i::Object(); + } + } +}; class InternalizedStringTableCleaner : public i::RootVisitor { public: @@ -85,12 +108,411 @@ class ExternalStringTableCleaner : public i::RootVisitor { class WeakRefs { + static constexpr int kMainThreadTask = 0; + i::WeakObjects weak_objects_; + + void FlushBytecodeFromSFI(i::SharedFunctionInfo shared_info) { + DCHECK(shared_info.HasBytecodeArray()); + // Retain objects required for uncompiled data. + auto inferred_name = shared_info.inferred_name(); + int start_position = shared_info.StartPosition(); + int end_position = shared_info.EndPosition(); + + shared_info.DiscardCompiledMetadata(isolate(), [](i::HeapObject object, i::ObjectSlot slot, i::HeapObject target) { + // RecordSlot(object, slot, target); + }); + + // The size of the bytecode array should always be larger than an + // UncompiledData object. + STATIC_ASSERT(i::BytecodeArray::SizeFor(0) >= i::UncompiledDataWithoutPreparseData::kSize); + + // Replace bytecode array with an uncompiled data array. + auto compiled_data = shared_info.GetBytecodeArray(isolate()); + // auto compiled_data_start = compiled_data.address(); + int compiled_data_size = compiled_data.Size(); + // auto chunk = MemoryChunk::FromAddress(compiled_data_start); + + // Clear any recorded slots for the compiled data as being invalid. + // DCHECK_NULL(chunk->sweeping_slot_set()); + // RememberedSet::RemoveRange( + // chunk, compiled_data_start, compiled_data_start + compiled_data_size, + // SlotSet::FREE_EMPTY_BUCKETS); + // RememberedSet::RemoveRange( + // chunk, compiled_data_start, compiled_data_start + compiled_data_size, + // SlotSet::FREE_EMPTY_BUCKETS); + + // Swap the map, using set_map_after_allocation to avoid verify heap checks + // which are not necessary since we are doing this during the GC atomic pause. + compiled_data.set_map_after_allocation( + i::ReadOnlyRoots(heap()).uncompiled_data_without_preparse_data_map(), + i::SKIP_WRITE_BARRIER); + + // Create a filler object for any left over space in the bytecode array. + if (!heap()->IsLargeObject(compiled_data)) { + heap()->CreateFillerObjectAt( + compiled_data.address() + i::UncompiledDataWithoutPreparseData::kSize, + compiled_data_size - i::UncompiledDataWithoutPreparseData::kSize, + i::ClearRecordedSlots::kNo); + } + + // Initialize the uncompiled data. + auto uncompiled_data = i::UncompiledData::cast(compiled_data); + uncompiled_data.InitAfterBytecodeFlush( + inferred_name, start_position, end_position, + [](i::HeapObject object, i::ObjectSlot slot, i::HeapObject target) { + // RecordSlot(object, slot, target); + }); + + // Mark the uncompiled data as black, and ensure all fields have already been + // marked. + DCHECK(is_live(inferred_name)); + DCHECK(is_live(uncompiled_data)); + + // Use the raw function data setter to avoid validity checks, since we're + // performing the unusual task of decompiling. + shared_info.set_function_data(uncompiled_data, v8::kReleaseStore); + DCHECK(!shared_info.is_compiled()); + } + + void ClearOldBytecodeCandidates() { + DCHECK(i::FLAG_flush_bytecode || weak_objects_.bytecode_flushing_candidates.IsEmpty()); + i::SharedFunctionInfo flushing_candidate; + while (weak_objects_.bytecode_flushing_candidates.Pop(kMainThreadTask, &flushing_candidate)) { + // If the BytecodeArray is dead, flush it, which will replace the field with + // an uncompiled data object. + if (!is_live_object(reinterpret_cast(flushing_candidate.GetBytecodeArray(heap()->isolate()).address()))) { + FlushBytecodeFromSFI(flushing_candidate); + } + // Now record the slot, which has either been updated to an uncompiled data, + // or is the BytecodeArray which is still alive. + // auto slot = + // flushing_candidate.RawField(SharedFunctionInfo::kFunctionDataOffset); + // RecordSlot(flushing_candidate, slot, HeapObject::cast(*slot)); + } + } + + void ClearFlushedJsFunctions() { + DCHECK(i::FLAG_flush_bytecode || weak_objects_.flushed_js_functions.IsEmpty()); + i::JSFunction flushed_js_function; + while (weak_objects_.flushed_js_functions.Pop(kMainThreadTask, &flushed_js_function)) { + auto gc_notify_updated_slot = [](i::HeapObject object, i::ObjectSlot slot, i::Object target) { + // RecordSlot(object, slot, HeapObject::cast(target)); + }; + flushed_js_function.ResetIfBytecodeFlushed(gc_notify_updated_slot); + UNREACHABLE(); + } + } + + void ClearFullMapTransitions() { + i::TransitionArray array; + while (weak_objects_.transition_arrays.Pop(kMainThreadTask, &array)) { + int num_transitions = array.number_of_entries(); + if (num_transitions > 0) { + i::Map map; + // The array might contain "undefined" elements because it's not yet + // filled. Allow it. + if (array.GetTargetIfExists(0, isolate(), &map)) { + DCHECK(!map.is_null()); // Weak pointers aren't cleared yet. + auto constructor_or_back_pointer = map.constructor_or_back_pointer(); + if (constructor_or_back_pointer.IsSmi()) { + DCHECK(isolate()->has_active_deserializer()); + DCHECK_EQ(constructor_or_back_pointer, i::Deserializer::uninitialized_field_value()); + continue; + } + auto parent = i::Map::cast(map.constructor_or_back_pointer()); + bool parent_is_alive = is_live(parent); + auto descriptors = parent_is_alive ? parent.instance_descriptors(isolate()) : i::DescriptorArray(); + bool descriptors_owner_died = CompactTransitionArray(parent, array, descriptors); + if (descriptors_owner_died) { + TrimDescriptorArray(parent, descriptors); + } + } + } + } + } + + bool TransitionArrayNeedsCompaction(i::TransitionArray transitions, int num_transitions) { + for (int i = 0; i < num_transitions; ++i) { + i::MaybeObject raw_target = transitions.GetRawTarget(i); + if (raw_target.IsSmi()) { + // This target is still being deserialized, + DCHECK(isolate()->has_active_deserializer()); + DCHECK_EQ(raw_target.ToSmi(), i::Deserializer::uninitialized_field_value()); + return false; + } else if (!is_live(i::TransitionsAccessor::GetTargetFromRaw(raw_target))) { + return true; + } + } + return false; + } + + bool CompactTransitionArray(i::Map map, i::TransitionArray transitions, i::DescriptorArray descriptors) { + DCHECK(!map.is_prototype_map()); + int num_transitions = transitions.number_of_entries(); + if (!TransitionArrayNeedsCompaction(transitions, num_transitions)) { + return false; + } + bool descriptors_owner_died = false; + int transition_index = 0; + // Compact all live transitions to the left. + for (int i = 0; i < num_transitions; ++i) { + i::Map target = transitions.GetTarget(i); + DCHECK_EQ(target.constructor_or_back_pointer(), map); + if (!is_live(target)) { + if (!descriptors.is_null() && + target.instance_descriptors(isolate()) == descriptors) { + DCHECK(!target.is_prototype_map()); + descriptors_owner_died = true; + } + } else { + if (i != transition_index) { + i::Name key = transitions.GetKey(i); + transitions.SetKey(transition_index, key); + // i::HeapObjectSlot key_slot = transitions.GetKeySlot(transition_index); + // RecordSlot(transitions, key_slot, key); + i::MaybeObject raw_target = transitions.GetRawTarget(i); + transitions.SetRawTarget(transition_index, raw_target); + // i::HeapObjectSlot target_slot = + // transitions.GetTargetSlot(transition_index); + // RecordSlot(transitions, target_slot, raw_target->GetHeapObject()); + } + transition_index++; + } + } + // If there are no transitions to be cleared, return. + if (transition_index == num_transitions) { + DCHECK(!descriptors_owner_died); + return false; + } + // Note that we never eliminate a transition array, though we might right-trim + // such that number_of_transitions() == 0. If this assumption changes, + // TransitionArray::Insert() will need to deal with the case that a transition + // array disappeared during GC. + int trim = transitions.Capacity() - transition_index; + if (trim > 0) { + heap()->RightTrimWeakFixedArray(transitions, trim * i::TransitionArray::kEntrySize); + transitions.SetNumberOfTransitions(transition_index); + } + return descriptors_owner_died; + } + + void RightTrimDescriptorArray(i::DescriptorArray array, int descriptors_to_trim) { + int old_nof_all_descriptors = array.number_of_all_descriptors(); + int new_nof_all_descriptors = old_nof_all_descriptors - descriptors_to_trim; + DCHECK_LT(0, descriptors_to_trim); + DCHECK_LE(0, new_nof_all_descriptors); + auto start = array.GetDescriptorSlot(new_nof_all_descriptors).address(); + auto end = array.GetDescriptorSlot(old_nof_all_descriptors).address(); + // MemoryChunk* chunk = MemoryChunk::FromHeapObject(array); + // DCHECK_NULL(chunk->sweeping_slot_set()); + // RememberedSet::RemoveRange(chunk, start, end, + // SlotSet::FREE_EMPTY_BUCKETS); + // RememberedSet::RemoveRange(chunk, start, end, + // SlotSet::FREE_EMPTY_BUCKETS); + heap()->CreateFillerObjectAt(start, static_cast(end - start), i::ClearRecordedSlots::kNo); + array.set_number_of_all_descriptors(new_nof_all_descriptors); + } + + void TrimDescriptorArray(i::Map map, i::DescriptorArray descriptors) { + int number_of_own_descriptors = map.NumberOfOwnDescriptors(); + if (number_of_own_descriptors == 0) { + DCHECK(descriptors == i::ReadOnlyRoots(heap()).empty_descriptor_array()); + return; + } + int to_trim = descriptors.number_of_all_descriptors() - number_of_own_descriptors; + if (to_trim > 0) { + descriptors.set_number_of_descriptors(number_of_own_descriptors); + RightTrimDescriptorArray(descriptors, to_trim); + TrimEnumCache(map, descriptors); + descriptors.Sort(); + } + DCHECK(descriptors.number_of_descriptors() == number_of_own_descriptors); + map.set_owns_descriptors(true); + } + + void TrimEnumCache(i::Map map, i::DescriptorArray descriptors) { + int live_enum = map.EnumLength(); + if (live_enum == i::kInvalidEnumCacheSentinel) { + live_enum = map.NumberOfEnumerableProperties(); + } + if (live_enum == 0) return descriptors.ClearEnumCache(); + auto enum_cache = descriptors.enum_cache(); + + auto keys = enum_cache.keys(); + int to_trim = keys.length() - live_enum; + if (to_trim <= 0) return; + heap()->RightTrimFixedArray(keys, to_trim); + + auto indices = enum_cache.indices(); + to_trim = indices.length() - live_enum; + if (to_trim <= 0) return; + heap()->RightTrimFixedArray(indices, to_trim); + } + + void ClearWeakCollections() { + i::EphemeronHashTable table; + while (weak_objects_.ephemeron_hash_tables.Pop(kMainThreadTask, &table)) { + for (i::InternalIndex i : table.IterateEntries()) { + auto key = i::HeapObject::cast(table.KeyAt(i)); + if (!is_live(key)) { + table.RemoveEntry(i); + } + } + } + for (auto it = heap()->ephemeron_remembered_set_.begin(); it != heap()->ephemeron_remembered_set_.end();) { + if (!is_live(it->first)) { + it = heap()->ephemeron_remembered_set_.erase(it); + } else { + ++it; + } + } + } + + void ClearWeakReferences() { + std::pair slot; + auto cleared_weak_ref = i::HeapObjectReference::ClearedValue(isolate()); + while (weak_objects_.weak_references.Pop(kMainThreadTask, &slot)) { + i::HeapObject value; + // The slot could have been overwritten, so we have to treat it + // as MaybeObjectSlot. + i::MaybeObjectSlot location(slot.second); + if ((*location)->GetHeapObjectIfWeak(&value)) { + DCHECK(!value.IsCell()); + if (is_live(value)) { + // The value of the weak reference is alive. + // RecordSlot(slot.first, HeapObjectSlot(location), value); + } else { + if (value.IsMap()) { + // The map is non-live. + ClearPotentialSimpleMapTransition(i::Map::cast(value)); + } + location.store(cleared_weak_ref); + } + } + } + } + + void ClearPotentialSimpleMapTransition(i::Map dead_target) { + DCHECK(!is_live(dead_target)); + auto potential_parent = dead_target.constructor_or_back_pointer(); + if (potential_parent.IsMap()) { + auto parent = i::Map::cast(potential_parent); + i::DisallowGarbageCollection no_gc_obviously; + if (is_live(parent) && i::TransitionsAccessor(isolate(), parent, &no_gc_obviously).HasSimpleTransitionTo(dead_target)) { + ClearPotentialSimpleMapTransition(parent, dead_target); + } + } + } + + void ClearPotentialSimpleMapTransition(i::Map map, i::Map dead_target) { + DCHECK(!map.is_prototype_map()); + DCHECK(!dead_target.is_prototype_map()); + DCHECK_EQ(map.raw_transitions(), i::HeapObjectReference::Weak(dead_target)); + // Take ownership of the descriptor array. + int number_of_own_descriptors = map.NumberOfOwnDescriptors(); + auto descriptors = map.instance_descriptors(isolate()); + if (descriptors == dead_target.instance_descriptors(isolate()) && number_of_own_descriptors > 0) { + TrimDescriptorArray(map, descriptors); + DCHECK(descriptors.number_of_descriptors() == number_of_own_descriptors); + } + } + + void ClearJSWeakRefs() { + i::JSWeakRef weak_ref; + while (weak_objects_.js_weak_refs.Pop(kMainThreadTask, &weak_ref)) { + auto target = i::HeapObject::cast(weak_ref.target()); + if (!is_live(target)) { + weak_ref.set_target(i::ReadOnlyRoots(isolate()).undefined_value()); + } else { + // The value of the JSWeakRef is alive. + // i::ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); + // RecordSlot(weak_ref, slot, target); + } + } + i::WeakCell weak_cell; + while (weak_objects_.weak_cells.Pop(kMainThreadTask, &weak_cell)) { + auto gc_notify_updated_slot = [](i::HeapObject object, i::ObjectSlot slot, i::Object target) { + // if (target.IsHeapObject()) { + // RecordSlot(object, slot, HeapObject::cast(target)); + // } + }; + auto target = i::HeapObject::cast(weak_cell.target()); + if (!is_live(target)) { + DCHECK(!target.IsUndefined()); + // The value of the WeakCell is dead. + auto finalization_registry = i::JSFinalizationRegistry::cast(weak_cell.finalization_registry()); + if (!finalization_registry.scheduled_for_cleanup()) { + heap()->EnqueueDirtyJSFinalizationRegistry(finalization_registry, gc_notify_updated_slot); + } + // We're modifying the pointers in WeakCell and JSFinalizationRegistry + // during GC; thus we need to record the slots it writes. The normal write + // barrier is not enough, since it's disabled before GC. + weak_cell.Nullify(isolate(), gc_notify_updated_slot); + DCHECK(finalization_registry.NeedsCleanup()); + DCHECK(finalization_registry.scheduled_for_cleanup()); + } else { + // The value of the WeakCell is alive. + // i::ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); + // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); + } + + auto unregister_token = weak_cell.unregister_token(); + if (!is_live(unregister_token)) { + // The unregister token is dead. Remove any corresponding entries in the + // key map. Multiple WeakCell with the same token will have all their + // unregister_token field set to undefined when processing the first + // WeakCell. Like above, we're modifying pointers during GC, so record the + // slots. + auto undefined = i::ReadOnlyRoots(isolate()).undefined_value(); + auto finalization_registry = i::JSFinalizationRegistry::cast(weak_cell.finalization_registry()); + finalization_registry.RemoveUnregisterToken( + i::JSReceiver::cast(unregister_token), isolate(), + [undefined](i::WeakCell matched_cell) { + matched_cell.set_unregister_token(undefined); + }, + gc_notify_updated_slot); + } else { + // The unregister_token is alive. + // ObjectSlot slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); + // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); + } + } + heap()->PostFinalizationRegistryCleanupTaskIfNeeded(); + } + + void MarkDependentCodeForDeoptimization() { + std::pair weak_object_in_code; + while (weak_objects_.weak_objects_in_code.Pop(kMainThreadTask, &weak_object_in_code)) { + auto object = weak_object_in_code.first; + auto code = weak_object_in_code.second; + if (!is_live(object) && !code.embedded_objects_cleared()) { + if (!code.marked_for_deoptimization()) { + code.SetMarkedForDeoptimization("weak objects"); + have_code_to_deoptimize_ = true; + } + code.ClearEmbeddedObjects(heap()); + DCHECK(code.embedded_objects_cleared()); + } + } + } + + bool have_code_to_deoptimize_ = false; public: - static v8::internal::Heap* heap() { - return v8::internal::third_party_heap::v8_heap; + static bool is_live(i::HeapObject o) { + return is_live_object(reinterpret_cast(o.address())); + } + static i::Isolate* isolate() { + return heap()->isolate(); + } + static i::Heap* heap() { + return tph::v8_heap; + } + i::WeakObjects* weak_objects() { + return &weak_objects_; } void ClearNonLiveReferences() { + have_code_to_deoptimize_ = false; { // TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR_STRING_TABLE); @@ -106,8 +528,32 @@ class WeakRefs { ExternalStringTableCleaner external_visitor(heap()); heap()->UpdateExternalStringTable(&external_visitor); } - v8::internal::third_party_heap::MMTkWeakObjectRetainer retainer; - heap()->ProcessAllWeakReferences(&retainer); + ClearOldBytecodeCandidates(); + ClearFlushedJsFunctions(); + { + MMTkWeakObjectRetainer retainer; + heap()->ProcessAllWeakReferences(&retainer); + } + ClearFullMapTransitions(); + ClearWeakReferences(); + ClearWeakCollections(); + ClearJSWeakRefs(); + + MarkDependentCodeForDeoptimization(); + + DCHECK(weak_objects_.transition_arrays.IsEmpty()); + DCHECK(weak_objects_.weak_references.IsEmpty()); + DCHECK(weak_objects_.weak_objects_in_code.IsEmpty()); + DCHECK(weak_objects_.js_weak_refs.IsEmpty()); + DCHECK(weak_objects_.weak_cells.IsEmpty()); + DCHECK(weak_objects_.bytecode_flushing_candidates.IsEmpty()); + DCHECK(weak_objects_.flushed_js_functions.IsEmpty()); + + if (have_code_to_deoptimize_) { + // Some code objects were marked for deoptimization during the GC. + i::Deoptimizer::DeoptimizeMarkedCode(isolate()); + have_code_to_deoptimize_ = false; + } } }; From 5901e89152814b1273cf6f059e2cec3e7384d2aa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 10 Jun 2021 16:56:41 +1000 Subject: [PATCH 29/65] refactor namespace --- v8/third_party/heap/mmtk/mmtk-visitors.h | 128 +++++++++++------------ v8/third_party/heap/mmtk/mmtkUpcalls.cc | 8 +- 2 files changed, 66 insertions(+), 70 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index d9b6154..ae1775d 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -17,33 +17,32 @@ #include -namespace v8 { -namespace internal { -namespace third_party_heap { +namespace mmtk { namespace i = v8::internal; +namespace tph = v8::internal::third_party_heap; -class MMTkRootVisitor: public RootVisitor { +class MMTkRootVisitor: public i::RootVisitor { public: - explicit MMTkRootVisitor(v8::internal::Heap* heap, TraceRootFn trace_root, void* context): heap_(heap), trace_root_(trace_root), context_(context) { + explicit MMTkRootVisitor(i::Heap* heap, TraceRootFn trace_root, void* context): heap_(heap), trace_root_(trace_root), context_(context) { USE(heap_); } - virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { + virtual void VisitRootPointer(i::Root root, const char* description, i::FullObjectSlot p) override final { ProcessRootEdge(root, p); } - virtual void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override final { - for (FullObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); + virtual void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override final { + for (auto p = start; p < end; ++p) ProcessRootEdge(root, p); } - virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) override final { - for (OffHeapObjectSlot p = start; p < end; ++p) ProcessRootEdge(root, p); + virtual void VisitRootPointers(i::Root root, const char* description, i::OffHeapObjectSlot start, i::OffHeapObjectSlot end) override final { + for (auto p = start; p < end; ++p) ProcessRootEdge(root, p); } private: - V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot slot) { - HeapObject object; + V8_INLINE void ProcessRootEdge(i::Root root, i::FullObjectSlot slot) { + i::HeapObject object; if ((*slot).GetHeapObject(&object)) { trace_root_((void*) slot.address(), context_); } @@ -54,9 +53,9 @@ class MMTkRootVisitor: public RootVisitor { void* context_; }; -class MMTkEdgeVisitor: public HeapVisitor { +class MMTkEdgeVisitor: public i::HeapVisitor { public: - explicit MMTkEdgeVisitor(v8::internal::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context): heap_(heap), process_edges_(process_edges), trace_field_(trace_field), context_(context) { + explicit MMTkEdgeVisitor(i::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context): heap_(heap), process_edges_(process_edges), trace_field_(trace_field), context_(context) { USE(heap_); NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; @@ -81,30 +80,30 @@ class MMTkEdgeVisitor: public HeapVisitor { weak_objects_->flushed_js_functions.FlushToGlobal(task_id_); } - V8_INLINE void VisitSharedFunctionInfo(Map map, SharedFunctionInfo shared_info) { + V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { // If the SharedFunctionInfo has old bytecode, mark it as flushable, // otherwise visit the function data field strongly. - if (shared_info.ShouldFlushBytecode(v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()))) { + if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); } } - V8_INLINE void VisitJSFunction(Map map, JSFunction object) { - if (v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()) != BytecodeFlushMode::kDoNotFlushBytecode && object.NeedsResetDueToFlushedBytecode()) { + V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { + if (v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()) != i::BytecodeFlushMode::kDoNotFlushBytecode && object.NeedsResetDueToFlushedBytecode()) { weak_objects_->flushed_js_functions.Push(task_id_, object); } } - V8_INLINE void VisitTransitionArray(Map map, TransitionArray array) { + V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { weak_objects_->transition_arrays.Push(task_id_, array); } - void VisitEphemeronHashTable(Map map, EphemeronHashTable table) { + void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { weak_objects_->ephemeron_hash_tables.Push(task_id_, table); - for (InternalIndex i : table.IterateEntries()) { + for (i::InternalIndex i : table.IterateEntries()) { // ObjectSlot key_slot = // table.RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i)); - HeapObject key = HeapObject::cast(table.KeyAt(i)); + auto key = i::HeapObject::cast(table.KeyAt(i)); // concrete_visitor()->SynchronizePageAccess(key); // concrete_visitor()->RecordSlot(table, key_slot, key); @@ -114,43 +113,43 @@ class MMTkEdgeVisitor: public HeapVisitor { // if (concrete_visitor()->marking_state()->IsBlackOrGrey(key)) { // VisitPointer(table, value_slot); // } else { - Object value_obj = table.ValueAt(i); + auto value_obj = table.ValueAt(i); if (value_obj.IsHeapObject()) { - HeapObject value = HeapObject::cast(value_obj); + auto value = i::HeapObject::cast(value_obj); // concrete_visitor()->SynchronizePageAccess(value); // concrete_visitor()->RecordSlot(table, value_slot, value); // Revisit ephemerons with both key and value unreachable at end // of concurrent marking cycle. - if (!mmtk::WeakRefs::is_live(value)) { - weak_objects_->discovered_ephemerons.Push(task_id_, Ephemeron{key, value}); + if (!WeakRefs::is_live(value)) { + weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); } } // } } } - void VisitJSWeakRef(Map map, JSWeakRef weak_ref) { + void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { if (weak_ref.target().IsHeapObject()) { weak_objects_->js_weak_refs.Push(task_id_, weak_ref); } } - virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { - for (ObjectSlot p = start; p < end; ++p) ProcessEdge(p); + virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { + for (auto p = start; p < end; ++p) ProcessEdge(p); } - virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override final { - for (MaybeObjectSlot p = start; p < end; ++p) ProcessEdge(p); + virtual void VisitPointers(i::HeapObject host, i::MaybeObjectSlot start, i::MaybeObjectSlot end) override final { + for (auto p = start; p < end; ++p) ProcessEdge(p); } - virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { - Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); + virtual void VisitCodeTarget(i::Code host, i::RelocInfo* rinfo) override final { + auto target = i::Code::GetCodeFromTargetAddress(rinfo->target_address()); DCHECK(!mmtk_is_movable(target)); trace_field_((void*) &target, context_); - DCHECK_EQ(target, Code::GetCodeFromTargetAddress(rinfo->target_address())); + DCHECK_EQ(target, i::Code::GetCodeFromTargetAddress(rinfo->target_address())); } - virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { + virtual void VisitEmbeddedPointer(i::Code host, i::RelocInfo* rinfo) override final { auto o = rinfo->target_object(); trace_field_((void*) &o, context_); if (o != rinfo->target_object()) { @@ -158,18 +157,18 @@ class MMTkEdgeVisitor: public HeapVisitor { } } - virtual void VisitMapPointer(HeapObject host) override final { + virtual void VisitMapPointer(i::HeapObject host) override final { ProcessEdge(host.map_slot()); } private: - V8_INLINE void ProcessRootEdge(Root root, FullObjectSlot p) { + V8_INLINE void ProcessRootEdge(i::Root root, i::FullObjectSlot p) { ProcessEdge(p); } template V8_INLINE void ProcessEdge(T p) { - HeapObject object; + i::HeapObject object; if ((*p).GetHeapObject(&object)) { PushEdge((void*) p.address()); } @@ -202,43 +201,43 @@ class MMTkEdgeVisitor: public HeapVisitor { -class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { +class MMTkHeapVerifier: public i::RootVisitor, public i::ObjectVisitor { public: explicit MMTkHeapVerifier() { } virtual ~MMTkHeapVerifier() {} - virtual void VisitRootPointer(Root root, const char* description, FullObjectSlot p) override final { + virtual void VisitRootPointer(i::Root root, const char* description, i::FullObjectSlot p) override final { VerifyRootEdge(root, p); } - virtual void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override final { - for (FullObjectSlot p = start; p < end; ++p) VerifyRootEdge(root, p); + virtual void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override final { + for (auto p = start; p < end; ++p) VerifyRootEdge(root, p); } - virtual void VisitRootPointers(Root root, const char* description, OffHeapObjectSlot start, OffHeapObjectSlot end) override final { - for (OffHeapObjectSlot p = start; p < end; ++p) VerifyRootEdge(root, p); + virtual void VisitRootPointers(i::Root root, const char* description, i::OffHeapObjectSlot start, i::OffHeapObjectSlot end) override final { + for (auto p = start; p < end; ++p) VerifyRootEdge(root, p); } - virtual void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override final { - for (ObjectSlot p = start; p < end; ++p) VerifyEdge(host, p); + virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { + for (auto p = start; p < end; ++p) VerifyEdge(host, p); } - virtual void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override final { - for (MaybeObjectSlot p = start; p < end; ++p) VerifyEdge(host, p); + virtual void VisitPointers(i::HeapObject host, i::MaybeObjectSlot start, i::MaybeObjectSlot end) override final { + for (auto p = start; p < end; ++p) VerifyEdge(host, p); } - virtual void VisitCodeTarget(Code host, RelocInfo* rinfo) override final { - Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); + virtual void VisitCodeTarget(i::Code host, i::RelocInfo* rinfo) override final { + auto target = i::Code::GetCodeFromTargetAddress(rinfo->target_address()); VerifyHeapObject(host, 0, target); } - virtual void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override final { + virtual void VisitEmbeddedPointer(i::Code host, i::RelocInfo* rinfo) override final { VerifyHeapObject(host, 0, rinfo->target_object()); } - virtual void VisitMapPointer(HeapObject host) override final { + virtual void VisitMapPointer(i::HeapObject host) override final { VerifyEdge(host, host.map_slot()); } @@ -250,43 +249,40 @@ class MMTkHeapVerifier: public RootVisitor, public ObjectVisitor { } } - static void Verify(v8::internal::Heap* heap) { + static void Verify(i::Heap* heap) { MMTkHeapVerifier visitor; heap->IterateRoots(&visitor, {}); visitor.TransitiveClosure(); } private: - V8_INLINE void VerifyRootEdge(Root root, FullObjectSlot p) { - VerifyEdge(HeapObject(), p); + V8_INLINE void VerifyRootEdge(i::Root root, i::FullObjectSlot p) { + VerifyEdge(i::HeapObject(), p); } template - V8_INLINE void VerifyEdge(HeapObject host, T p) { - HeapObject object; + V8_INLINE void VerifyEdge(i::HeapObject host, T p) { + i::HeapObject object; if ((*p).GetHeapObject(&object)) { VerifyHeapObject(host, p.address(), object); } } - V8_INLINE void VerifyHeapObject(HeapObject host, Address edge, HeapObject o) { + V8_INLINE void VerifyHeapObject(i::HeapObject host, i::Address edge, i::HeapObject o) { if (marked_objects_.find(o.ptr()) == marked_objects_.end()) { marked_objects_.insert(o.ptr()); - if (!third_party_heap::Heap::IsValidHeapObject(o)) { + if (!tph::Heap::IsValidHeapObject(o)) { printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } - CHECK(third_party_heap::Heap::IsValidHeapObject(o)); + CHECK(tph::Heap::IsValidHeapObject(o)); mark_stack_.push_back(o); } } - std::unordered_set
marked_objects_; - std::vector mark_stack_; + std::unordered_set marked_objects_; + std::vector mark_stack_; }; -} -} -} - +} // namespace mmtk #endif // MMTK_VISITORS_H \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 8dfccd1..aeac4fe 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -25,7 +25,7 @@ static void mmtk_stop_all_mutators(void *tls) { main_thread_synchronizer->RunMainThreadTask([=]() { main_thread_synchronizer->EnterSafepoint(v8_heap); MMTK_LOG("[mmtk_stop_all_mutators] Verify heap\n"); - MMTkHeapVerifier::Verify(v8_heap); + mmtk::MMTkHeapVerifier::Verify(v8_heap); MMTK_LOG("[mmtk_stop_all_mutators] Flush cache\n"); v8_heap->isolate()->descriptor_lookup_cache()->Clear(); RegExpResultsCache::Clear(v8_heap->string_split_cache()); @@ -50,7 +50,7 @@ static void mmtk_resume_mutators(void *tls) { MMTK_LOG("[mmtk_resume_mutators] START\n"); main_thread_synchronizer->RunMainThreadTask([=]() { MMTK_LOG("[mmtk_resume_mutators] Verify heap\n"); - MMTkHeapVerifier::Verify(v8_heap); + mmtk::MMTkHeapVerifier::Verify(v8_heap); MMTK_LOG("[mmtk_resume_mutators] Flush cache\n"); v8_heap->isolate()->inner_pointer_to_code_cache()->Flush(); // The stub caches are not traversed during GC; clear them to force @@ -133,13 +133,13 @@ static void mmtk_on_move_event(void* from_address, void* to_address, size_t size static void mmtk_scan_roots(TraceRootFn trace_root, void* context) { main_thread_synchronizer->RunMainThreadTask([=]() { - MMTkRootVisitor root_visitor(v8_heap, trace_root, context); + mmtk::MMTkRootVisitor root_visitor(v8_heap, trace_root, context); v8_heap->IterateRoots(&root_visitor, {}); }); } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context) { - MMTkEdgeVisitor visitor(v8_heap, process_edges, trace_field, context); + mmtk::MMTkEdgeVisitor visitor(v8_heap, process_edges, trace_field, context); for (size_t i = 0; i < count; i++) { auto ptr = *(objects + i); DCHECK_EQ(((Address) ptr) & 1, 0); From f8219265fe26aa6aaf7224f93821797ea317441d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 10 Jun 2021 20:47:49 +1000 Subject: [PATCH 30/65] Process weak fields --- v8/third_party/heap/mmtk/mmtk-visitors.h | 18 +++++-- v8/third_party/heap/mmtk/weak-refs.h | 60 +++++++++++++++++++++--- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index ae1775d..d9a66b6 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -120,7 +120,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // concrete_visitor()->RecordSlot(table, value_slot, value); // Revisit ephemerons with both key and value unreachable at end // of concurrent marking cycle. - if (!WeakRefs::is_live(value)) { + if (!mmtk::is_live(value)) { weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); } } @@ -135,11 +135,11 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { - for (auto p = start; p < end; ++p) ProcessEdge(p); + for (auto p = start; p < end; ++p) ProcessEdge2(host, p); } virtual void VisitPointers(i::HeapObject host, i::MaybeObjectSlot start, i::MaybeObjectSlot end) override final { - for (auto p = start; p < end; ++p) ProcessEdge(p); + for (auto p = start; p < end; ++p) ProcessEdge2(host, p); } virtual void VisitCodeTarget(i::Code host, i::RelocInfo* rinfo) override final { @@ -174,6 +174,18 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } } + template + V8_INLINE void ProcessEdge2(i::HeapObject host, TSlot slot) { + i::HeapObject object; + if ((*slot).GetHeapObjectIfStrong(&object)) { + PushEdge((void*) slot.address()); + } else if (TSlot::kCanBeWeak && (*slot).GetHeapObjectIfWeak(&object)) { + // ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); + i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); + weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); + } + } + V8_INLINE void PushEdge(void* edge) { buffer_[cursor_++] = edge; if (cursor_ >= cap_) flush(); diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 021a8ea..b4b1591 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -19,8 +19,28 @@ extern v8::internal::Heap* v8_heap; namespace mmtk { namespace i = v8::internal; +namespace base = v8::base; namespace tph = v8::internal::third_party_heap; +bool is_live(i::HeapObject o) { + return is_live_object(reinterpret_cast(o.address())); +} + +i::MaybeObject to_weakref(i::HeapObject o) { + DCHECK(o.IsStrong()); + return i::MaybeObject::MakeWeak(i::MaybeObject::FromObject(o)); +} + +base::Optional get_forwarded_ref(i::HeapObject o) { + DCHECK(o.IsStrong()); + auto f = mmtk_get_forwarded_object(o); + if (f != nullptr) { + auto x = i::HeapObject::cast(i::Object((i::Address) f)); + DCHECK(o.IsStrong()); + return x; + } + return base::nullopt; +} class MMTkWeakObjectRetainer: public i::WeakObjectRetainer { public: @@ -61,10 +81,13 @@ class InternalizedStringTableCleaner : public i::RootVisitor { if (o.IsHeapObject()) { auto heap_object = i::HeapObject::cast(o); DCHECK(!i::Heap::InYoungGeneration(heap_object)); - if (!is_live_object(reinterpret_cast(heap_object.address()))) { + if (!is_live(heap_object)) { pointers_removed_++; // Set the entry to the_hole_value (as deleted). p.store(i::StringTable::deleted_element()); + } else { + auto forwarded = get_forwarded_ref(heap_object); + if (forwarded) p.store(*forwarded); } } } @@ -88,7 +111,7 @@ class ExternalStringTableCleaner : public i::RootVisitor { i::Object o = *p; if (o.IsHeapObject()) { auto heap_object = i::HeapObject::cast(o); - if (!is_live_object(reinterpret_cast(heap_object.address()))) { + if (!is_live(heap_object)) { if (o.IsExternalString()) { heap_->FinalizeExternalString(i::String::cast(o)); } else { @@ -97,6 +120,9 @@ class ExternalStringTableCleaner : public i::RootVisitor { } // Set the entry to the_hole_value (as deleted). p.store(the_hole); + } else { + auto forwarded = get_forwarded_ref(heap_object); + if (forwarded) p.store(*forwarded); } } } @@ -168,6 +194,9 @@ class WeakRefs { DCHECK(is_live(inferred_name)); DCHECK(is_live(uncompiled_data)); + auto forwarded = get_forwarded_ref(uncompiled_data); + if (forwarded) uncompiled_data = i::UncompiledData::cast(*forwarded); + // Use the raw function data setter to avoid validity checks, since we're // performing the unusual task of decompiling. shared_info.set_function_data(uncompiled_data, v8::kReleaseStore); @@ -180,8 +209,10 @@ class WeakRefs { while (weak_objects_.bytecode_flushing_candidates.Pop(kMainThreadTask, &flushing_candidate)) { // If the BytecodeArray is dead, flush it, which will replace the field with // an uncompiled data object. - if (!is_live_object(reinterpret_cast(flushing_candidate.GetBytecodeArray(heap()->isolate()).address()))) { + if (!is_live(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { FlushBytecodeFromSFI(flushing_candidate); + } else { + DCHECK(!get_forwarded_ref(flushing_candidate.GetBytecodeArray(heap()->isolate()))); } // Now record the slot, which has either been updated to an uncompiled data, // or is the BytecodeArray which is still alive. @@ -221,6 +252,9 @@ class WeakRefs { } auto parent = i::Map::cast(map.constructor_or_back_pointer()); bool parent_is_alive = is_live(parent); + if (parent_is_alive) { + DCHECK(!get_forwarded_ref(parent)); + } auto descriptors = parent_is_alive ? parent.instance_descriptors(isolate()) : i::DescriptorArray(); bool descriptors_owner_died = CompactTransitionArray(parent, array, descriptors); if (descriptors_owner_died) { @@ -241,6 +275,8 @@ class WeakRefs { return false; } else if (!is_live(i::TransitionsAccessor::GetTargetFromRaw(raw_target))) { return true; + } else { + DCHECK(!get_forwarded_ref(i::TransitionsAccessor::GetTargetFromRaw(raw_target))); } } return false; @@ -265,6 +301,7 @@ class WeakRefs { descriptors_owner_died = true; } } else { + DCHECK(!get_forwarded_ref(target)); if (i != transition_index) { i::Name key = transitions.GetKey(i); transitions.SetKey(transition_index, key); @@ -356,6 +393,8 @@ class WeakRefs { auto key = i::HeapObject::cast(table.KeyAt(i)); if (!is_live(key)) { table.RemoveEntry(i); + } else { + DCHECK(!get_forwarded_ref(key)); } } } @@ -363,6 +402,7 @@ class WeakRefs { if (!is_live(it->first)) { it = heap()->ephemeron_remembered_set_.erase(it); } else { + DCHECK(!get_forwarded_ref(it->first)); ++it; } } @@ -381,6 +421,8 @@ class WeakRefs { if (is_live(value)) { // The value of the weak reference is alive. // RecordSlot(slot.first, HeapObjectSlot(location), value); + auto forwarded = get_forwarded_ref(value); + if (forwarded) location.store(to_weakref(*forwarded)); } else { if (value.IsMap()) { // The map is non-live. @@ -398,6 +440,9 @@ class WeakRefs { if (potential_parent.IsMap()) { auto parent = i::Map::cast(potential_parent); i::DisallowGarbageCollection no_gc_obviously; + if (is_live(parent)) { + DCHECK(!get_forwarded_ref(parent)); + } if (is_live(parent) && i::TransitionsAccessor(isolate(), parent, &no_gc_obviously).HasSimpleTransitionTo(dead_target)) { ClearPotentialSimpleMapTransition(parent, dead_target); } @@ -424,6 +469,8 @@ class WeakRefs { if (!is_live(target)) { weak_ref.set_target(i::ReadOnlyRoots(isolate()).undefined_value()); } else { + auto forwarded = get_forwarded_ref(weak_ref.target()); + if (forwarded) weak_ref.set_target(*forwarded); // The value of the JSWeakRef is alive. // i::ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); // RecordSlot(weak_ref, slot, target); @@ -451,6 +498,7 @@ class WeakRefs { DCHECK(finalization_registry.NeedsCleanup()); DCHECK(finalization_registry.scheduled_for_cleanup()); } else { + DCHECK(!get_forwarded_ref(target)); // The value of the WeakCell is alive. // i::ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); @@ -472,6 +520,7 @@ class WeakRefs { }, gc_notify_updated_slot); } else { + DCHECK(!get_forwarded_ref(unregister_token)); // The unregister_token is alive. // ObjectSlot slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); @@ -492,15 +541,14 @@ class WeakRefs { } code.ClearEmbeddedObjects(heap()); DCHECK(code.embedded_objects_cleared()); + } else if (is_live(object)) { + DCHECK(!get_forwarded_ref(object)); } } } bool have_code_to_deoptimize_ = false; public: - static bool is_live(i::HeapObject o) { - return is_live_object(reinterpret_cast(o.address())); - } static i::Isolate* isolate() { return heap()->isolate(); } From 485a70f5b437bf8ecbbb30269211f8a5d0f5b55b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 11 Jun 2021 10:35:12 +1000 Subject: [PATCH 31/65] Refactor --- mmtk/src/lib.rs | 12 ++++++++++-- mmtk/src/scanning.rs | 4 ++-- v8/third_party/heap/mmtk/mmtk-visitors.h | 23 +++++++++-------------- v8/third_party/heap/mmtk/mmtk.h | 4 ++-- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 8 ++++---- v8/third_party/heap/mmtk/weak-refs.h | 18 ++++++++++++++++++ 6 files changed, 45 insertions(+), 24 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 880b37a..795d0df 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -9,6 +9,7 @@ extern crate lazy_static; #[macro_use] extern crate log; +use std::env; use std::ptr::null_mut; use libc::c_void; @@ -53,8 +54,8 @@ pub struct V8_Upcalls { pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut Mutator, pub is_mutator: extern "C" fn(tls: VMThread) -> bool, - pub scan_roots: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void), - pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn, trace_field: TraceFieldFn, context: *mut c_void), + pub scan_roots: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void, task_id: usize), + pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn, trace_field: TraceFieldFn, context: *mut c_void, task_id: usize), pub process_weak_refs: extern "C" fn(), pub on_move_event: extern "C" fn(from: ObjectReference, to: ObjectReference, size: usize), } @@ -76,6 +77,13 @@ impl VMBinding for V8 { lazy_static! { pub static ref SINGLETON: MMTK = { + if let Ok(threads) = env::var("MMTK_THREADS").map(|x| x.parse::().unwrap()) { + if threads > 8 { + env::set_var("MMTK_THREADS", "8"); + } + } else { + env::set_var("MMTK_THREADS", "8"); + } MMTK::new() }; } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 37dfb58..997e585 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -40,7 +40,7 @@ impl Scanning for VMScanning { OBJECTS_TO_SCAN = vec![]; let buf = objects.as_ptr(); let len = objects.len(); - ((*UPCALLS).scan_objects)(buf, len, create_process_edges_work:: as _, trace_slot:: as _, worker as *mut _ as _); + ((*UPCALLS).scan_objects)(buf, len, create_process_edges_work:: as _, trace_slot:: as _, worker as *mut _ as _, worker.ordinal); } } } @@ -81,7 +81,7 @@ impl> GCWork for ScanAndForwardRoots { fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { unsafe { debug_assert!(ROOT_OBJECTS.is_empty()); - ((*UPCALLS).scan_roots)(trace_root:: as _, worker as *mut _ as _); + ((*UPCALLS).scan_roots)(trace_root:: as _, worker as *mut _ as _, worker.ordinal); if !ROOT_OBJECTS.is_empty() { flush_roots::(worker); } diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index d9a66b6..5db05da 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -24,8 +24,11 @@ namespace tph = v8::internal::third_party_heap; class MMTkRootVisitor: public i::RootVisitor { public: - explicit MMTkRootVisitor(i::Heap* heap, TraceRootFn trace_root, void* context): heap_(heap), trace_root_(trace_root), context_(context) { + explicit MMTkRootVisitor(i::Heap* heap, TraceRootFn trace_root, void* context, int task_id) + : heap_(heap), trace_root_(trace_root), context_(context), task_id_(task_id) { USE(heap_); + USE(task_id_); + DCHECK(task_id <= 8); } virtual void VisitRootPointer(i::Root root, const char* description, i::FullObjectSlot p) override final { @@ -51,12 +54,15 @@ class MMTkRootVisitor: public i::RootVisitor { v8::internal::Heap* heap_; TraceRootFn trace_root_; void* context_; + int task_id_; }; class MMTkEdgeVisitor: public i::HeapVisitor { public: - explicit MMTkEdgeVisitor(i::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context): heap_(heap), process_edges_(process_edges), trace_field_(trace_field), context_(context) { + explicit MMTkEdgeVisitor(i::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id) + : heap_(heap), process_edges_(process_edges), trace_field_(trace_field), context_(context), task_id_(task_id) { USE(heap_); + DCHECK(task_id <= 8); NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; cap_ = buf.cap; @@ -67,17 +73,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { if (buffer_ != NULL) { release_buffer(buffer_, cursor_, cap_); } - weak_objects_->transition_arrays.FlushToGlobal(task_id_); - weak_objects_->ephemeron_hash_tables.FlushToGlobal(task_id_); - weak_objects_->current_ephemerons.FlushToGlobal(task_id_); - weak_objects_->next_ephemerons.FlushToGlobal(task_id_); - weak_objects_->discovered_ephemerons.FlushToGlobal(task_id_); - weak_objects_->weak_references.FlushToGlobal(task_id_); - weak_objects_->js_weak_refs.FlushToGlobal(task_id_); - weak_objects_->weak_cells.FlushToGlobal(task_id_); - weak_objects_->weak_objects_in_code.FlushToGlobal(task_id_); - weak_objects_->bytecode_flushing_candidates.FlushToGlobal(task_id_); - weak_objects_->flushed_js_functions.FlushToGlobal(task_id_); } V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { @@ -208,7 +203,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { size_t cap_ = 0; size_t cursor_ = 0; i::WeakObjects* weak_objects_ = mmtk::global_weakref_processor->weak_objects(); - int task_id_ = 1; + int task_id_; }; diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 67e9d4c..17c767e 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -133,8 +133,8 @@ typedef struct { size_t (*get_object_size) (void* object); void* (*get_mmtk_mutator) (void* tls); bool (*is_mutator) (void* tls); - void (*scan_roots) (TraceRootFn process_edges, void* context); - void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context); + void (*scan_roots) (TraceRootFn process_edges, void* context, int task_id); + void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id); void (*process_weak_refs) (); void (*on_move_event) (void* from, void* to, size_t size); } V8_Upcalls; diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index aeac4fe..1295e8c 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -131,15 +131,15 @@ static void mmtk_on_move_event(void* from_address, void* to_address, size_t size v8_heap->OnMoveEvent(to, from, (int) size); } -static void mmtk_scan_roots(TraceRootFn trace_root, void* context) { +static void mmtk_scan_roots(TraceRootFn trace_root, void* context, int task_id) { main_thread_synchronizer->RunMainThreadTask([=]() { - mmtk::MMTkRootVisitor root_visitor(v8_heap, trace_root, context); + mmtk::MMTkRootVisitor root_visitor(v8_heap, trace_root, context, task_id); v8_heap->IterateRoots(&root_visitor, {}); }); } -static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context) { - mmtk::MMTkEdgeVisitor visitor(v8_heap, process_edges, trace_field, context); +static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id) { + mmtk::MMTkEdgeVisitor visitor(v8_heap, process_edges, trace_field, context, task_id); for (size_t i = 0; i < count; i++) { auto ptr = *(objects + i); DCHECK_EQ(((Address) ptr) & 1, 0); diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index b4b1591..823b9e5 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -548,6 +548,23 @@ class WeakRefs { } bool have_code_to_deoptimize_ = false; + + void Flush() { + for (int i = 0; i < 8; i++) { + weak_objects_.transition_arrays.FlushToGlobal(i); + weak_objects_.ephemeron_hash_tables.FlushToGlobal(i); + weak_objects_.current_ephemerons.FlushToGlobal(i); + weak_objects_.next_ephemerons.FlushToGlobal(i); + weak_objects_.discovered_ephemerons.FlushToGlobal(i); + weak_objects_.weak_references.FlushToGlobal(i); + weak_objects_.js_weak_refs.FlushToGlobal(i); + weak_objects_.weak_cells.FlushToGlobal(i); + weak_objects_.weak_objects_in_code.FlushToGlobal(i); + weak_objects_.bytecode_flushing_candidates.FlushToGlobal(i); + weak_objects_.flushed_js_functions.FlushToGlobal(i); + } + } + public: static i::Isolate* isolate() { return heap()->isolate(); @@ -560,6 +577,7 @@ class WeakRefs { } void ClearNonLiveReferences() { + Flush(); have_code_to_deoptimize_ = false; { // TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR_STRING_TABLE); From 5fb598b2fb4c0d5cedc25702af5a9a4ec621d5f8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 11 Jun 2021 10:58:27 +1000 Subject: [PATCH 32/65] weak_cell processing --- v8/third_party/heap/mmtk/mmtk-visitors.h | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 5db05da..326be03 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -45,6 +45,7 @@ class MMTkRootVisitor: public i::RootVisitor { private: V8_INLINE void ProcessRootEdge(i::Root root, i::FullObjectSlot slot) { + DCHECK(!i::HasWeakHeapObjectTag(*slot)); i::HeapObject object; if ((*slot).GetHeapObject(&object)) { trace_root_((void*) slot.address(), context_); @@ -123,6 +124,17 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } } + void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { + auto target = weak_cell.relaxed_target(); + auto unregister_token = weak_cell.relaxed_unregister_token(); + if (!mmtk::is_live(target) || !mmtk::is_live(unregister_token)) { + // WeakCell points to a potentially dead object or a dead unregister + // token. We have to process them when we know the liveness of the whole + // transitive closure. + weak_objects_->weak_cells.Push(task_id_, weak_cell); + } + } + void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { if (weak_ref.target().IsHeapObject()) { weak_objects_->js_weak_refs.Push(task_id_, weak_ref); @@ -130,11 +142,11 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { - for (auto p = start; p < end; ++p) ProcessEdge2(host, p); + for (auto p = start; p < end; ++p) ProcessEdge(host, p); } virtual void VisitPointers(i::HeapObject host, i::MaybeObjectSlot start, i::MaybeObjectSlot end) override final { - for (auto p = start; p < end; ++p) ProcessEdge2(host, p); + for (auto p = start; p < end; ++p) ProcessEdge(host, p); } virtual void VisitCodeTarget(i::Code host, i::RelocInfo* rinfo) override final { @@ -153,24 +165,12 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } virtual void VisitMapPointer(i::HeapObject host) override final { - ProcessEdge(host.map_slot()); + ProcessEdge(host, host.map_slot()); } private: - V8_INLINE void ProcessRootEdge(i::Root root, i::FullObjectSlot p) { - ProcessEdge(p); - } - - template - V8_INLINE void ProcessEdge(T p) { - i::HeapObject object; - if ((*p).GetHeapObject(&object)) { - PushEdge((void*) p.address()); - } - } - template - V8_INLINE void ProcessEdge2(i::HeapObject host, TSlot slot) { + V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { i::HeapObject object; if ((*slot).GetHeapObjectIfStrong(&object)) { PushEdge((void*) slot.address()); From 46b3d86d6ec7587f05b347ec180cf4d2b8baa2bc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 14 Jun 2021 19:33:46 +1000 Subject: [PATCH 33/65] Fix weak embedded pointer --- v8/third_party/heap/mmtk/mmtk-visitors.h | 54 ++++++++++++++++++++++-- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 12 ++++++ v8/third_party/heap/mmtk/weak-refs.h | 15 +++++-- 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 326be03..83dfa25 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -58,6 +58,48 @@ class MMTkRootVisitor: public i::RootVisitor { int task_id_; }; +class MMTkCustomRootBodyVisitor final : public i::ObjectVisitor { + public: + explicit MMTkCustomRootBodyVisitor(i::Heap* heap, TraceRootFn trace_root, void* context, int task_id) + : heap_(heap), trace_root_(trace_root), context_(context), task_id_(task_id) { + USE(heap_); + USE(task_id_); + DCHECK(task_id <= 8); + } + + void VisitPointer(i::HeapObject host, i::ObjectSlot p) final {} + + void VisitMapPointer(i::HeapObject host) final {} + + void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) final {} + + void VisitPointers(i::HeapObject host, i::MaybeObjectSlot start, + i::MaybeObjectSlot end) final { + // At the moment, custom roots cannot contain weak pointers. + UNREACHABLE(); + } + + // VisitEmbedderPointer is defined by ObjectVisitor to call VisitPointers. + void VisitCodeTarget(i::Code host, i::RelocInfo* rinfo) override { + auto target = i::Code::GetCodeFromTargetAddress(rinfo->target_address()); + DCHECK(!mmtk_is_movable(target)); + trace_root_((void*) &target, context_); + DCHECK_EQ(target, i::Code::GetCodeFromTargetAddress(rinfo->target_address())); + } + + void VisitEmbeddedPointer(i::Code host, i::RelocInfo* rinfo) override { + auto o = rinfo->target_object(); + trace_root_((void*) &o, context_); + if (o != rinfo->target_object()) rinfo->set_target_object(heap_, o); + } + + private: + v8::internal::Heap* heap_; + TraceRootFn trace_root_; + void* context_; + int task_id_; +}; + class MMTkEdgeVisitor: public i::HeapVisitor { public: explicit MMTkEdgeVisitor(i::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id) @@ -158,9 +200,15 @@ class MMTkEdgeVisitor: public i::HeapVisitor { virtual void VisitEmbeddedPointer(i::Code host, i::RelocInfo* rinfo) override final { auto o = rinfo->target_object(); - trace_field_((void*) &o, context_); - if (o != rinfo->target_object()) { - rinfo->set_target_object(heap_, o); + auto f = mmtk::get_forwarded_ref(o); + if (f) { + rinfo->set_target_object(heap_, *f); + } else if (host.IsWeakObject(o)) { + DCHECK(i::RelocInfo::IsCodeTarget(rinfo->rmode()) || i::RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); + weak_objects_->weak_objects_in_code.Push(task_id_, std::make_pair(o, host)); + } else { + trace_field_((void*) &o, context_); + if (o != rinfo->target_object()) rinfo->set_target_object(heap_, o); } } diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 1295e8c..6f6fb34 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -134,7 +134,19 @@ static void mmtk_on_move_event(void* from_address, void* to_address, size_t size static void mmtk_scan_roots(TraceRootFn trace_root, void* context, int task_id) { main_thread_synchronizer->RunMainThreadTask([=]() { mmtk::MMTkRootVisitor root_visitor(v8_heap, trace_root, context, task_id); + mmtk::MMTkCustomRootBodyVisitor custom_root_body_visitor(v8_heap, trace_root, context, task_id); v8_heap->IterateRoots(&root_visitor, {}); + for (i::StackFrameIterator it(v8_heap->isolate(), v8_heap->isolate()->thread_local_top()); !it.done(); it.Advance()) { + if (it.frame()->is_unoptimized()) break; + if (it.frame()->type() == StackFrame::OPTIMIZED) { + auto code = it.frame()->LookupCode(); + if (!code.CanDeoptAt(v8_heap->isolate(), it.frame()->pc())) { + trace_root((void*) &code, context); + v8::internal::Code::BodyDescriptor::IterateBody(code.map(), code, &custom_root_body_visitor); + } + break; + } + } }); } diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 823b9e5..2119a2f 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -534,15 +534,24 @@ class WeakRefs { while (weak_objects_.weak_objects_in_code.Pop(kMainThreadTask, &weak_object_in_code)) { auto object = weak_object_in_code.first; auto code = weak_object_in_code.second; - if (!is_live(object) && !code.embedded_objects_cleared()) { + auto object_is_live = is_live(object); + if (!object_is_live && !code.embedded_objects_cleared()) { if (!code.marked_for_deoptimization()) { code.SetMarkedForDeoptimization("weak objects"); have_code_to_deoptimize_ = true; } code.ClearEmbeddedObjects(heap()); DCHECK(code.embedded_objects_cleared()); - } else if (is_live(object)) { - DCHECK(!get_forwarded_ref(object)); + } else if (object_is_live) { + auto f = mmtk::get_forwarded_ref(object); + if (f) { + int mode_mask = i::RelocInfo::EmbeddedObjectModeMask() | (1 << i::RelocInfo::CODE_TARGET); + for (i::RelocIterator it(code, mode_mask); !it.done(); it.next()) { + DCHECK(i::RelocInfo::IsEmbeddedObjectMode(it.rinfo()->rmode())); + if (it.rinfo()->target_object() == object) + it.rinfo()->set_target_object(heap(), *f, i::SKIP_WRITE_BARRIER); + } + } } } } From b6d0cba10eca631fc090f46c1e106dc2955755d9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 16 Jun 2021 11:11:11 +1000 Subject: [PATCH 34/65] WIP --- v8/third_party/heap/mmtk/mmtk-visitors.h | 19 +++++++++++++++++++ v8/third_party/heap/mmtk/weak-refs.h | 20 +++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 83dfa25..8840869 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -123,6 +123,8 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // otherwise visit the function data field strongly. if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); + } else { + VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); } } @@ -216,6 +218,23 @@ class MMTkEdgeVisitor: public i::HeapVisitor { ProcessEdge(host, host.map_slot()); } + // Custom weak pointers must be ignored by the GC but not other + // visitors. They're used for e.g., lists that are recreated after GC. The + // default implementation treats them as strong pointers. Visitors who want to + // ignore them must override this function with empty. + // virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, + // i::ObjectSlot end) override final { + // // VisitPointers(host, start, end); + // if (!(host.IsWeakCell() || host.IsJSWeakRef())) { + // VisitPointers(host, start, end); + // return; + // } + // // DCHECK(host.IsWeakCell() || host.IsJSWeakRef()); + // // for (auto p = start; p < end; ++p) { + // // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, i::HeapObjectSlot(p.address()))); + // // } + // } + private: template V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 2119a2f..14e5363 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -210,6 +210,7 @@ class WeakRefs { // If the BytecodeArray is dead, flush it, which will replace the field with // an uncompiled data object. if (!is_live(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { + UNREACHABLE(); FlushBytecodeFromSFI(flushing_candidate); } else { DCHECK(!get_forwarded_ref(flushing_candidate.GetBytecodeArray(heap()->isolate()))); @@ -430,6 +431,20 @@ class WeakRefs { } location.store(cleared_weak_ref); } + } else if ((*location)->GetHeapObjectIfStrong(&value)) { + DCHECK(!value.IsCell()); + if (is_live(value)) { + // The value of the weak reference is alive. + // RecordSlot(slot.first, HeapObjectSlot(location), value); + auto forwarded = get_forwarded_ref(value); + if (forwarded) location.store(to_weakref(*forwarded)); + } else { + if (value.IsMap()) { + // The map is non-live. + ClearPotentialSimpleMapTransition(i::Map::cast(value)); + } + location.store(cleared_weak_ref); + } } } } @@ -504,7 +519,8 @@ class WeakRefs { // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); } - auto unregister_token = weak_cell.unregister_token(); + i::ObjectSlot slot = weak_cell.RawField(i::WeakCell::kUnregisterTokenOffset); + i::HeapObject unregister_token = i::HeapObject::cast(*slot);//weak_cell.unregister_token(); if (!is_live(unregister_token)) { // The unregister token is dead. Remove any corresponding entries in the // key map. Multiple WeakCell with the same token will have all their @@ -521,6 +537,8 @@ class WeakRefs { gc_notify_updated_slot); } else { DCHECK(!get_forwarded_ref(unregister_token)); + // auto f = get_forwarded_ref(unregister_token); + // if (f) *slot = *f; // The unregister_token is alive. // ObjectSlot slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); From 038365fefd56c5334c9dd612de3219be7599eca1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Jun 2021 13:06:02 +1000 Subject: [PATCH 35/65] WIP: Switch visitor --- v8/third_party/heap/mmtk/mmtk-visitors.h | 334 ++++++++++++++++++----- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 1 - 2 files changed, 268 insertions(+), 67 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 8840869..3d1388e 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -118,72 +118,274 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } } - V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { - // If the SharedFunctionInfo has old bytecode, mark it as flushable, - // otherwise visit the function data field strongly. - if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { - weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); - } else { - VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); - } - } + // V8_INLINE bool AllowDefaultJSObjectVisit() { return false; } + + // V8_INLINE void VisitBytecodeArray(i::Map map, i::BytecodeArray object) { + // if (!ShouldVisit(object)) return; + // int size = i::BytecodeArray::BodyDescriptor::SizeOf(map, object); + // this->VisitMapPointer(object); + // i::BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); + // // if (!is_forced_gc_) { + // object.MakeOlder(); + // // } + // } + // V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { + // if (!ShouldVisit(array)) return; + // VisitMapPointer(array); + // // int size = i::DescriptorArray::BodyDescriptor::SizeOf(map, array); + // VisitPointers(array, array.GetFirstPointerSlot(), array.GetDescriptorSlot(0)); + // VisitDescriptors(array, array.number_of_descriptors()); + // } + // V8_INLINE void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { + // if (!ShouldVisit(table)) return; + // weak_objects_->ephemeron_hash_tables.Push(task_id_, table); + + // for (i::InternalIndex i : table.IterateEntries()) { + // // i::ObjectSlot key_slot = + // // table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); + // // i::HeapObject key = i::HeapObject::cast(table.KeyAt(i)); + + // // SynchronizePageAccess(key); + // // RecordSlot(table, key_slot, key); + + // i::ObjectSlot value_slot = + // table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); + + // // if (marking_state()->IsBlackOrGrey(key)) { + // VisitPointer(table, value_slot); + // // } else { + // // i::Object value_obj = table.ValueAt(i); + + // // if (value_obj.IsHeapObject()) { + // // i::HeapObject value = i::HeapObject::cast(value_obj); + // // // SynchronizePageAccess(value); + // // // RecordSlot(table, value_slot, value); + + // // // Revisit ephemerons with both key and value unreachable at end + // // // of concurrent marking cycle. + // // if (marking_state()->IsWhite(value)) { + // // weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); + // // } + // // } + // // } + // } + // } + // V8_INLINE void VisitFixedArray(i::Map map, i::FixedArray object) { + // // Arrays with the progress bar are not left-trimmable because they reside + // // in the large object space. + // VisitLeftTrimmableArray(map, object); + // } + // V8_INLINE void VisitFixedDoubleArray(i::Map map, i::FixedDoubleArray object) { + // VisitLeftTrimmableArray(map, object); + // } + // V8_INLINE void VisitJSApiObject(i::Map map, i::JSObject object) { + // VisitEmbedderTracingSubclass(map, object); + // } + // V8_INLINE void VisitJSArrayBuffer(i::Map map, i::JSArrayBuffer object) { + // object.MarkExtension(); + // VisitEmbedderTracingSubclass(map, object); + // } + // V8_INLINE void VisitJSDataView(i::Map map, i::JSDataView object) { + // VisitEmbedderTracingSubclass(map, object); + // } + // V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { + // VisitJSObjectSubclass(map, object); + // // Check if the JSFunction needs reset due to bytecode being flushed. + // if (/*bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode &&*/ + // object.NeedsResetDueToFlushedBytecode()) { + // weak_objects_->flushed_js_functions.Push(task_id_, object); + // } + // } + // V8_INLINE void VisitJSTypedArray(i::Map map, i::JSTypedArray object) { + // VisitEmbedderTracingSubclass(map, object); + // } + // V8_INLINE void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { + // VisitJSObjectSubclass(map, weak_ref); + // // if (size == 0) return 0; + // if (weak_ref.target().IsHeapObject()) { + // // i::HeapObject target = i::HeapObject::cast(weak_ref.target()); + // // SynchronizePageAccess(target); + // // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target)) { + // // // Record the slot inside the JSWeakRef, since the + // // // VisitJSObjectSubclass above didn't visit it. + // // ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); + // // concrete_visitor()->RecordSlot(weak_ref, slot, target); + // // } else { + // // JSWeakRef points to a potentially dead object. We have to process + // // them when we know the liveness of the whole transitive closure. + // weak_objects_->js_weak_refs.Push(task_id_, weak_ref); + // // } + // } + // } + // V8_INLINE void VisitMap(i::Map meta_map, i::Map map) { + // if (!ShouldVisit(map)) return; + // i::Map::BodyDescriptor::SizeOf(meta_map, map); + // // VisitDescriptorsForMap(map); + + // // Mark the pointer fields of the Map. If there is a transitions array, it has + // // been marked already, so it is fine that one of these fields contains a + // // pointer to it. + // i::Map::BodyDescriptor::IterateBody(meta_map, map, map.Size(), this); + // } + // V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { + // if (!ShouldVisit(shared_info)) return; + + // int size = i::SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); + // VisitMapPointer(shared_info); + // i::SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); + + // // If the SharedFunctionInfo has old bytecode, mark it as flushable, + // // otherwise visit the function data field strongly. + // if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { + // weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); + // } else { + // VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); + // } + // } + // V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { + // if (!ShouldVisit(array)) return; + // VisitMapPointer(array); + // int size = i::TransitionArray::BodyDescriptor::SizeOf(map, array); + // i::TransitionArray::BodyDescriptor::IterateBody(map, array, size, this); + // weak_objects_->transition_arrays.Push(task_id_, array); + // } + // V8_INLINE void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { + // if (!ShouldVisit(weak_cell)) return; + + // int size = i::WeakCell::BodyDescriptor::SizeOf(map, weak_cell); + // this->VisitMapPointer(weak_cell); + // i::WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this); + // // i::HeapObject target = weak_cell.relaxed_target(); + // // i::HeapObject unregister_token = weak_cell.relaxed_unregister_token(); + // // concrete_visitor()->SynchronizePageAccess(target); + // // concrete_visitor()->SynchronizePageAccess(unregister_token); + // // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target) && + // // concrete_visitor()->marking_state()->IsBlackOrGrey(unregister_token)) { + // // // Record the slots inside the WeakCell, since the IterateBody above + // // // didn't visit it. + // // ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); + // // concrete_visitor()->RecordSlot(weak_cell, slot, target); + // // slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); + // // concrete_visitor()->RecordSlot(weak_cell, slot, unregister_token); + // // } else { + // // WeakCell points to a potentially dead object or a dead unregister + // // token. We have to process them when we know the liveness of the whole + // // transitive closure. + // weak_objects_->weak_cells.Push(task_id_, weak_cell); + // // } + // } - V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { - if (v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()) != i::BytecodeFlushMode::kDoNotFlushBytecode && object.NeedsResetDueToFlushedBytecode()) { - weak_objects_->flushed_js_functions.Push(task_id_, object); - } - } + // void VisitDescriptors(i::DescriptorArray descriptor_array, int number_of_own_descriptors) { + // // int16_t new_marked = static_cast(number_of_own_descriptors); + // // int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors( + // // mark_compact_epoch_, new_marked); + // // if (old_marked < new_marked) { + // // VisitPointers(descriptor_array, + // // MaybeObjectSlot(descriptor_array.GetDescriptorSlot(old_marked)), + // // MaybeObjectSlot(descriptor_array.GetDescriptorSlot(new_marked))); + // // } + // } - V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { - weak_objects_->transition_arrays.Push(task_id_, array); - } - - void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { - weak_objects_->ephemeron_hash_tables.Push(task_id_, table); - for (i::InternalIndex i : table.IterateEntries()) { - // ObjectSlot key_slot = - // table.RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i)); - auto key = i::HeapObject::cast(table.KeyAt(i)); - - // concrete_visitor()->SynchronizePageAccess(key); - // concrete_visitor()->RecordSlot(table, key_slot, key); - - // ObjectSlot value_slot = table.RawFieldOfElementAt(EphemeronHashTable::EntryToValueIndex(i)); - - // if (concrete_visitor()->marking_state()->IsBlackOrGrey(key)) { - // VisitPointer(table, value_slot); - // } else { - auto value_obj = table.ValueAt(i); - if (value_obj.IsHeapObject()) { - auto value = i::HeapObject::cast(value_obj); - // concrete_visitor()->SynchronizePageAccess(value); - // concrete_visitor()->RecordSlot(table, value_slot, value); - // Revisit ephemerons with both key and value unreachable at end - // of concurrent marking cycle. - if (!mmtk::is_live(value)) { - weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); - } - } - // } - } - } + // template + // void VisitLeftTrimmableArray(i::Map map, T object) { + // // The length() function checks that the length is a Smi. + // // This is not necessarily the case if the array is being left-trimmed. + // i::Object length = object.unchecked_length(v8::kAcquireLoad); + // if (!ShouldVisit(object)) return; + // // The cached length must be the actual length as the array is not black. + // // Left trimming marks the array black before over-writing the length. + // DCHECK(length.IsSmi()); + // int size = T::SizeFor(i::Smi::ToInt(length)); + // VisitMapPointer(object); + // T::BodyDescriptor::IterateBody(map, object, size, this); + // } - void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { - auto target = weak_cell.relaxed_target(); - auto unregister_token = weak_cell.relaxed_unregister_token(); - if (!mmtk::is_live(target) || !mmtk::is_live(unregister_token)) { - // WeakCell points to a potentially dead object or a dead unregister - // token. We have to process them when we know the liveness of the whole - // transitive closure. - weak_objects_->weak_cells.Push(task_id_, weak_cell); - } - } + // template + // void VisitEmbedderTracingSubclass(i::Map map, T object) { + // DCHECK(object.IsApiWrapper()); + // VisitJSObjectSubclass(map, object); + // // if (size && is_embedder_tracing_enabled_) { + // // // Success: The object needs to be processed for embedder references on + // // // the main thread. + // // local_marking_worklists_->PushEmbedder(object); + // // } + // } + + // template + // void VisitJSObjectSubclass(i::Map map, T object) { + // if (!ShouldVisit(object)) return; + // VisitMapPointer(object); + // int size = T::BodyDescriptor::SizeOf(map, object); + // T::BodyDescriptor::IterateBody(map, object, size, this); + // } - void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { - if (weak_ref.target().IsHeapObject()) { - weak_objects_->js_weak_refs.Push(task_id_, weak_ref); - } - } + + // V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { + // // If the SharedFunctionInfo has old bytecode, mark it as flushable, + // // otherwise visit the function data field strongly. + // if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { + // weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); + // } else { + // VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); + // } + // } + + // V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { + // if (v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()) != i::BytecodeFlushMode::kDoNotFlushBytecode && object.NeedsResetDueToFlushedBytecode()) { + // weak_objects_->flushed_js_functions.Push(task_id_, object); + // } + // } + + // V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { + // weak_objects_->transition_arrays.Push(task_id_, array); + // } + + // void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { + // weak_objects_->ephemeron_hash_tables.Push(task_id_, table); + // for (i::InternalIndex i : table.IterateEntries()) { + // // ObjectSlot key_slot = + // // table.RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i)); + // auto key = i::HeapObject::cast(table.KeyAt(i)); + + // // concrete_visitor()->SynchronizePageAccess(key); + // // concrete_visitor()->RecordSlot(table, key_slot, key); + + // // ObjectSlot value_slot = table.RawFieldOfElementAt(EphemeronHashTable::EntryToValueIndex(i)); + + // // if (concrete_visitor()->marking_state()->IsBlackOrGrey(key)) { + // // VisitPointer(table, value_slot); + // // } else { + // auto value_obj = table.ValueAt(i); + // if (value_obj.IsHeapObject()) { + // auto value = i::HeapObject::cast(value_obj); + // // concrete_visitor()->SynchronizePageAccess(value); + // // concrete_visitor()->RecordSlot(table, value_slot, value); + // // Revisit ephemerons with both key and value unreachable at end + // // of concurrent marking cycle. + // if (!mmtk::is_live(value)) { + // weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); + // } + // } + // // } + // } + // } + + // void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { + // auto target = weak_cell.relaxed_target(); + // auto unregister_token = weak_cell.relaxed_unregister_token(); + // if (!mmtk::is_live(target) || !mmtk::is_live(unregister_token)) { + // // WeakCell points to a potentially dead object or a dead unregister + // // token. We have to process them when we know the liveness of the whole + // // transitive closure. + // weak_objects_->weak_cells.Push(task_id_, weak_cell); + // } + // } + + // void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { + // if (weak_ref.target().IsHeapObject()) { + // weak_objects_->js_weak_refs.Push(task_id_, weak_ref); + // } + // } virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { for (auto p = start; p < end; ++p) ProcessEdge(host, p); @@ -234,7 +436,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, i::HeapObjectSlot(p.address()))); // // } // } - private: template V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { @@ -242,9 +443,10 @@ class MMTkEdgeVisitor: public i::HeapVisitor { if ((*slot).GetHeapObjectIfStrong(&object)) { PushEdge((void*) slot.address()); } else if (TSlot::kCanBeWeak && (*slot).GetHeapObjectIfWeak(&object)) { - // ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); - i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); - weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); + PushEdge((void*) slot.address()); + // // ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); + // i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); + // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); } } diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 6f6fb34..5cd7d18 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -156,7 +156,6 @@ static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn proce auto ptr = *(objects + i); DCHECK_EQ(((Address) ptr) & 1, 0); auto obj = HeapObject::FromAddress(((Address) ptr)); - obj.Iterate(&visitor); visitor.Visit(obj); } } From b4036d879097a8d1f26e038c1fa1b81bdec0f15a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Jun 2021 15:59:41 +1000 Subject: [PATCH 36/65] WIP: Switch visitor --- v8/third_party/heap/mmtk/mmtk-visitors.h | 29 +++++++++++++++--------- v8/third_party/heap/mmtk/weak-refs.h | 17 +++++++++++--- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 3d1388e..ce82a23 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -242,13 +242,13 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); // } // } - // V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { - // if (!ShouldVisit(array)) return; - // VisitMapPointer(array); - // int size = i::TransitionArray::BodyDescriptor::SizeOf(map, array); - // i::TransitionArray::BodyDescriptor::IterateBody(map, array, size, this); - // weak_objects_->transition_arrays.Push(task_id_, array); - // } + V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { + // if (!ShouldVisit(array)) return; + VisitMapPointer(array); + int size = i::TransitionArray::BodyDescriptor::SizeOf(map, array); + i::TransitionArray::BodyDescriptor::IterateBody(map, array, size, this); + weak_objects_->transition_arrays.Push(task_id_, array); + } // V8_INLINE void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { // if (!ShouldVisit(weak_cell)) return; @@ -439,14 +439,21 @@ class MMTkEdgeVisitor: public i::HeapVisitor { private: template V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { + DCHECK(mmtk::is_live(host)); + DCHECK(!mmtk::get_forwarded_ref(host)); i::HeapObject object; if ((*slot).GetHeapObjectIfStrong(&object)) { PushEdge((void*) slot.address()); } else if (TSlot::kCanBeWeak && (*slot).GetHeapObjectIfWeak(&object)) { - PushEdge((void*) slot.address()); - // // ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); - // i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); - // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); + // if (auto f = mmtk::get_forwarded_ref(object)) { + // i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); + // s.StoreHeapObject(*f); + // } else { + // PushEdge((void*) slot.address()); + // // ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); + i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); + weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); + // } } } diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 14e5363..e0c7952 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -431,27 +431,38 @@ class WeakRefs { } location.store(cleared_weak_ref); } - } else if ((*location)->GetHeapObjectIfStrong(&value)) { + } /*else if ((*location)->GetHeapObjectIfStrong(&value)) { DCHECK(!value.IsCell()); if (is_live(value)) { // The value of the weak reference is alive. // RecordSlot(slot.first, HeapObjectSlot(location), value); auto forwarded = get_forwarded_ref(value); - if (forwarded) location.store(to_weakref(*forwarded)); + if (forwarded) { + printf("[WeakRef] Strong %p -> %p\n", (void*) value.ptr(), (void*) forwarded->ptr()); + location.store(to_weakref(*forwarded)); + } else { + printf("[WeakRef] Strong %p \n", (void*) value.ptr()); + } } else { + printf("[WeakRef] Strong Dead %p\n", (void*) value.ptr()); if (value.IsMap()) { // The map is non-live. ClearPotentialSimpleMapTransition(i::Map::cast(value)); } location.store(cleared_weak_ref); } - } + }*/ } } void ClearPotentialSimpleMapTransition(i::Map dead_target) { DCHECK(!is_live(dead_target)); auto potential_parent = dead_target.constructor_or_back_pointer(); + if (is_live(i::HeapObject::cast(potential_parent))) { + if (auto f = get_forwarded_ref(i::HeapObject::cast(potential_parent))) { + potential_parent = *f; + } + } if (potential_parent.IsMap()) { auto parent = i::Map::cast(potential_parent); i::DisallowGarbageCollection no_gc_obviously; From b925865a251f616aed442e8859dfa9d7b0d77924 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 21 Jun 2021 11:37:15 +1000 Subject: [PATCH 37/65] WIP --- mmtk/src/collection.rs | 12 ++- mmtk/src/lib.rs | 2 +- mmtk/src/object_archive.rs | 15 ++++ mmtk/src/object_model.rs | 1 + mmtk/src/scanning.rs | 4 +- v8/third_party/heap/mmtk/mmtk-visitors.h | 109 ++++++++++++++--------- v8/third_party/heap/mmtk/mmtk.h | 2 +- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 7 +- v8/third_party/heap/mmtk/weak-refs.h | 17 ++-- 9 files changed, 115 insertions(+), 54 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index bc4d1ae..d86f9df 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -8,6 +8,9 @@ use UPCALLS; use V8; use crate::object_archive::global_object_archive; +use crate::scanning::ROOT_OBJECTS; +use crate::scanning::flush_roots; +use crate::scanning::trace_root; pub struct VMCollection {} @@ -58,9 +61,14 @@ impl Collection for VMCollection { global_object_archive().update(); } - fn process_weak_refs() { + fn process_weak_refs>(worker: &mut GCWorker) { unsafe { - ((*UPCALLS).process_weak_refs)(); + debug_assert!(ROOT_OBJECTS.is_empty()); + ((*UPCALLS).process_weak_refs)(trace_root:: as _, worker as *mut _ as _); + if !ROOT_OBJECTS.is_empty() { + flush_roots::(worker); + } + debug_assert!(ROOT_OBJECTS.is_empty()); } } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 795d0df..962639d 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -56,7 +56,7 @@ pub struct V8_Upcalls { pub is_mutator: extern "C" fn(tls: VMThread) -> bool, pub scan_roots: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void, task_id: usize), pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn, trace_field: TraceFieldFn, context: *mut c_void, task_id: usize), - pub process_weak_refs: extern "C" fn(), + pub process_weak_refs: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void), pub on_move_event: extern "C" fn(from: ObjectReference, to: ObjectReference, size: usize), } diff --git a/mmtk/src/object_archive.rs b/mmtk/src/object_archive.rs index 54f7f24..23b56c4 100644 --- a/mmtk/src/object_archive.rs +++ b/mmtk/src/object_archive.rs @@ -2,6 +2,7 @@ use libc::c_void; use mmtk::util::ObjectReference; use mmtk::util::address::Address; use std::collections::HashMap; +use std::sync::Mutex; lazy_static! { @@ -139,11 +140,25 @@ impl ObjectArchive { new_space_map.insert(new_object, self.space_map[&object]); } } + new_objects.dedup(); new_objects.sort_by(|a, b| a.to_address().cmp(&b.to_address())); self.untagged_objects = new_objects; self.space_map = new_space_map; } + pub fn copy(&mut self, from: Address, to: Address) { + lazy_static! { + static ref LOCK: Mutex<()> = Mutex::default(); + } + let _lock = LOCK.lock().unwrap(); + unsafe { + let space = self.space_map[&from.to_object_reference()]; + self.space_map.insert(to.to_object_reference(), space); + // self.untagged_objects.push(to.to_object_reference()); + // self.untagged_objects.sort_by(|a, b| a.to_address().cmp(&b.to_address())); + } + } + pub fn reset_iterator(&mut self) { self.iter_pos = 0; self.iter_len = self.untagged_objects.len(); diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 3617b22..264022a 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -27,6 +27,7 @@ impl ObjectModel for VMObjectModel { let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, unsafe { Address::zero() }, bytes, allocator); unsafe { ((*UPCALLS).on_move_event)(from, to_obj, bytes) }; + super::object_archive::global_object_archive().copy(src, dst); to_obj } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 997e585..b776f15 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -90,9 +90,9 @@ impl> GCWork for ScanAndForwardRoots { } } -static mut ROOT_OBJECTS: Vec = Vec::new(); +pub(crate) static mut ROOT_OBJECTS: Vec = Vec::new(); -fn flush_roots>(_worker: &mut GCWorker) { +pub(crate) fn flush_roots>(_worker: &mut GCWorker) { let mut buf = vec![]; unsafe { std::mem::swap(&mut buf, &mut ROOT_OBJECTS); } let scan_objects_work = mmtk::scheduler::gc_work::ScanObjects::::new(buf, false); diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index ce82a23..68550f9 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -16,6 +16,13 @@ #include "weak-refs.h" #include +// #define WEAKREF_PROCESSING + +#ifdef WEAKREF_PROCESSING +#define WEAKREF_PROCESSING_BOOL true +#else +#define WEAKREF_PROCESSING_BOOL false +#endif namespace mmtk { @@ -120,15 +127,17 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // V8_INLINE bool AllowDefaultJSObjectVisit() { return false; } - // V8_INLINE void VisitBytecodeArray(i::Map map, i::BytecodeArray object) { - // if (!ShouldVisit(object)) return; - // int size = i::BytecodeArray::BodyDescriptor::SizeOf(map, object); - // this->VisitMapPointer(object); - // i::BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); - // // if (!is_forced_gc_) { - // object.MakeOlder(); - // // } - // } +#ifdef WEAKREF_PROCESSING + + V8_INLINE void VisitBytecodeArray(i::Map map, i::BytecodeArray object) { + if (!ShouldVisit(object)) return; + int size = i::BytecodeArray::BodyDescriptor::SizeOf(map, object); + this->VisitMapPointer(object); + i::BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); + // if (!is_forced_gc_) { + object.MakeOlder(); + // } + } // V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { // if (!ShouldVisit(array)) return; // VisitMapPointer(array); @@ -188,14 +197,14 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // V8_INLINE void VisitJSDataView(i::Map map, i::JSDataView object) { // VisitEmbedderTracingSubclass(map, object); // } - // V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { - // VisitJSObjectSubclass(map, object); - // // Check if the JSFunction needs reset due to bytecode being flushed. - // if (/*bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode &&*/ - // object.NeedsResetDueToFlushedBytecode()) { - // weak_objects_->flushed_js_functions.Push(task_id_, object); - // } - // } + V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { + VisitJSObjectSubclass(map, object); + // Check if the JSFunction needs reset due to bytecode being flushed. + if (/*bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode &&*/ + object.NeedsResetDueToFlushedBytecode()) { + weak_objects_->flushed_js_functions.Push(task_id_, object); + } + } // V8_INLINE void VisitJSTypedArray(i::Map map, i::JSTypedArray object) { // VisitEmbedderTracingSubclass(map, object); // } @@ -227,21 +236,30 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // // pointer to it. // i::Map::BodyDescriptor::IterateBody(meta_map, map, map.Size(), this); // } - // V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { - // if (!ShouldVisit(shared_info)) return; + V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { + // if (!ShouldVisit(shared_info)) return; + int size = i::SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); + VisitMapPointer(shared_info); + i::SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); + + auto data = shared_info.function_data(v8::kAcquireLoad); + if (data.IsHeapObject() || data.IsWeak()) { + if (auto f = mmtk::get_forwarded_ref(i::HeapObject::cast(data))) { + shared_info.set_function_data(*f, v8::kReleaseStore); + } + // trace_field_((void*) &data, context_); + // shared_info.set_function_data(data, v8::kReleaseStore); + } - // int size = i::SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); - // VisitMapPointer(shared_info); - // i::SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); + // If the SharedFunctionInfo has old bytecode, mark it as flushable, + // otherwise visit the function data field strongly. + if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { + weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); + } else { + VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); + } + } - // // If the SharedFunctionInfo has old bytecode, mark it as flushable, - // // otherwise visit the function data field strongly. - // if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { - // weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); - // } else { - // VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); - // } - // } V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { // if (!ShouldVisit(array)) return; VisitMapPointer(array); @@ -311,13 +329,13 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // // } // } - // template - // void VisitJSObjectSubclass(i::Map map, T object) { - // if (!ShouldVisit(object)) return; - // VisitMapPointer(object); - // int size = T::BodyDescriptor::SizeOf(map, object); - // T::BodyDescriptor::IterateBody(map, object, size, this); - // } + template + void VisitJSObjectSubclass(i::Map map, T object) { + // if (!ShouldVisit(object)) return; + VisitMapPointer(object); + int size = T::BodyDescriptor::SizeOf(map, object); + T::BodyDescriptor::IterateBody(map, object, size, this); + } // V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { @@ -387,6 +405,8 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // } // } +#endif + virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { for (auto p = start; p < end; ++p) ProcessEdge(host, p); } @@ -407,7 +427,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { auto f = mmtk::get_forwarded_ref(o); if (f) { rinfo->set_target_object(heap_, *f); - } else if (host.IsWeakObject(o)) { + } else if (host.IsWeakObject(o) && WEAKREF_PROCESSING_BOOL) { DCHECK(i::RelocInfo::IsCodeTarget(rinfo->rmode()) || i::RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); weak_objects_->weak_objects_in_code.Push(task_id_, std::make_pair(o, host)); } else { @@ -424,8 +444,8 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // visitors. They're used for e.g., lists that are recreated after GC. The // default implementation treats them as strong pointers. Visitors who want to // ignore them must override this function with empty. - // virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, - // i::ObjectSlot end) override final { +#ifdef WEAKREF_PROCESSING + virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { // // VisitPointers(host, start, end); // if (!(host.IsWeakCell() || host.IsJSWeakRef())) { // VisitPointers(host, start, end); @@ -435,7 +455,8 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // // for (auto p = start; p < end; ++p) { // // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, i::HeapObjectSlot(p.address()))); // // } - // } + } +#endif private: template V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { @@ -445,15 +466,19 @@ class MMTkEdgeVisitor: public i::HeapVisitor { if ((*slot).GetHeapObjectIfStrong(&object)) { PushEdge((void*) slot.address()); } else if (TSlot::kCanBeWeak && (*slot).GetHeapObjectIfWeak(&object)) { + if (!WEAKREF_PROCESSING_BOOL) { + PushEdge((void*) slot.address()); + } else { // if (auto f = mmtk::get_forwarded_ref(object)) { // i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); // s.StoreHeapObject(*f); - // } else { + // } else { // PushEdge((void*) slot.address()); // // ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); // } + } } } diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 17c767e..b52123d 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -135,7 +135,7 @@ typedef struct { bool (*is_mutator) (void* tls); void (*scan_roots) (TraceRootFn process_edges, void* context, int task_id); void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id); - void (*process_weak_refs) (); + void (*process_weak_refs) (TraceRootFn process_edges, void* context); void (*on_move_event) (void* from, void* to, size_t size); } V8_Upcalls; diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 5cd7d18..4b2f09a 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -39,9 +39,12 @@ static void mmtk_stop_all_mutators(void *tls) { MMTK_LOG("[mmtk_stop_all_mutators] END\n"); } -static void mmtk_process_weak_refs() { +static void mmtk_process_weak_refs(TraceRootFn trace_root, void* context) { main_thread_synchronizer->RunMainThreadTask([=]() { MMTK_LOG("[mmtk_process_weak_refs]\n"); + mmtk::global_weakref_processor->trace_ = [=](void* slot) { + trace_root(slot, context); + }; mmtk::global_weakref_processor->ClearNonLiveReferences(); }); } @@ -73,7 +76,9 @@ static void mmtk_spawn_collector_thread(void* tls, void* ctx) { } static void mmtk_block_for_gc() { + v8_heap->SetGCState(v8::internal::Heap::MARK_COMPACT); main_thread_synchronizer->Block(); + v8_heap->SetGCState(v8::internal::Heap::NOT_IN_GC); } static void* mmtk_get_mmtk_mutator(void* tls) { diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index e0c7952..072efac 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -192,10 +192,11 @@ class WeakRefs { // Mark the uncompiled data as black, and ensure all fields have already been // marked. DCHECK(is_live(inferred_name)); - DCHECK(is_live(uncompiled_data)); + // DCHECK(is_live(uncompiled_data)); - auto forwarded = get_forwarded_ref(uncompiled_data); - if (forwarded) uncompiled_data = i::UncompiledData::cast(*forwarded); + trace_((void*) &uncompiled_data); + // auto forwarded = trace_((void*) &uncompiled_data); + // if (forwarded) uncompiled_data = i::UncompiledData::cast(*forwarded); // Use the raw function data setter to avoid validity checks, since we're // performing the unusual task of decompiling. @@ -209,8 +210,13 @@ class WeakRefs { while (weak_objects_.bytecode_flushing_candidates.Pop(kMainThreadTask, &flushing_candidate)) { // If the BytecodeArray is dead, flush it, which will replace the field with // an uncompiled data object. + auto data = flushing_candidate.function_data(v8::kAcquireLoad); + if (data.IsHeapObject() || data.IsWeak()) { + if (auto f = get_forwarded_ref(i::HeapObject::cast(data))) { + flushing_candidate.set_function_data(*f, v8::kReleaseStore); + } + } if (!is_live(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { - UNREACHABLE(); FlushBytecodeFromSFI(flushing_candidate); } else { DCHECK(!get_forwarded_ref(flushing_candidate.GetBytecodeArray(heap()->isolate()))); @@ -231,7 +237,6 @@ class WeakRefs { // RecordSlot(object, slot, HeapObject::cast(target)); }; flushed_js_function.ResetIfBytecodeFlushed(gc_notify_updated_slot); - UNREACHABLE(); } } @@ -604,6 +609,8 @@ class WeakRefs { } public: + std::function trace_ = [](void*) { UNREACHABLE(); }; + static i::Isolate* isolate() { return heap()->isolate(); } From e543fea2c8322c6046f9d655cfe6731e56f29095 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 22 Jun 2021 10:45:13 +1000 Subject: [PATCH 38/65] [weakref] More progress --- v8/third_party/heap/mmtk/mmtk-visitors.h | 341 +++++++---------------- v8/third_party/heap/mmtk/weak-refs.h | 13 +- 2 files changed, 102 insertions(+), 252 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 68550f9..b68e0b7 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -16,7 +16,7 @@ #include "weak-refs.h" #include -// #define WEAKREF_PROCESSING +#define WEAKREF_PROCESSING #ifdef WEAKREF_PROCESSING #define WEAKREF_PROCESSING_BOOL true @@ -128,119 +128,104 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // V8_INLINE bool AllowDefaultJSObjectVisit() { return false; } #ifdef WEAKREF_PROCESSING + V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { + // if (!ShouldVisit(array)) return; + VisitMapPointer(array); + // int size = i::DescriptorArray::BodyDescriptor::SizeOf(map, array); + VisitPointers(array, array.GetFirstPointerSlot(), array.GetDescriptorSlot(0)); + VisitDescriptors(array, array.number_of_descriptors()); + } + + void VisitDescriptors(i::DescriptorArray descriptor_array, int number_of_own_descriptors) { + int16_t new_marked = static_cast(number_of_own_descriptors); + // int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors( + // heap_->gc_count(), new_marked); + // if (old_marked < new_marked) { + VisitPointers(descriptor_array, + i::MaybeObjectSlot(descriptor_array.GetDescriptorSlot(0)), + i::MaybeObjectSlot(descriptor_array.GetDescriptorSlot(new_marked))); + // } + } + + V8_INLINE void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { + // if (!ShouldVisit(weak_cell)) return; + + int size = i::WeakCell::BodyDescriptor::SizeOf(map, weak_cell); + this->VisitMapPointer(weak_cell); + skip_weak_ = true; + i::WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this); + skip_weak_ = false; + // i::HeapObject target = weak_cell.relaxed_target(); + // i::HeapObject unregister_token = weak_cell.relaxed_unregister_token(); + // concrete_visitor()->SynchronizePageAccess(target); + // concrete_visitor()->SynchronizePageAccess(unregister_token); + // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target) && + // concrete_visitor()->marking_state()->IsBlackOrGrey(unregister_token)) { + // // Record the slots inside the WeakCell, since the IterateBody above + // // didn't visit it. + // ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); + // concrete_visitor()->RecordSlot(weak_cell, slot, target); + // slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); + // concrete_visitor()->RecordSlot(weak_cell, slot, unregister_token); + // } else { + // WeakCell points to a potentially dead object or a dead unregister + // token. We have to process them when we know the liveness of the whole + // transitive closure. + weak_objects_->weak_cells.Push(task_id_, weak_cell); + // } + } + + V8_INLINE void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { + skip_weak_ = true; + VisitJSObjectSubclass(map, weak_ref); + skip_weak_ = false; + // if (size == 0) return 0; + if (weak_ref.target().IsHeapObject()) { + // i::HeapObject target = i::HeapObject::cast(weak_ref.target()); + // SynchronizePageAccess(target); + // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target)) { + // // Record the slot inside the JSWeakRef, since the + // // VisitJSObjectSubclass above didn't visit it. + // ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); + // concrete_visitor()->RecordSlot(weak_ref, slot, target); + // } else { + // JSWeakRef points to a potentially dead object. We have to process + // them when we know the liveness of the whole transitive closure. + weak_objects_->js_weak_refs.Push(task_id_, weak_ref); + // } + } + } V8_INLINE void VisitBytecodeArray(i::Map map, i::BytecodeArray object) { if (!ShouldVisit(object)) return; int size = i::BytecodeArray::BodyDescriptor::SizeOf(map, object); this->VisitMapPointer(object); + skip_weak_ = true; i::BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); + skip_weak_ = false; // if (!is_forced_gc_) { object.MakeOlder(); // } } - // V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { - // if (!ShouldVisit(array)) return; - // VisitMapPointer(array); - // // int size = i::DescriptorArray::BodyDescriptor::SizeOf(map, array); - // VisitPointers(array, array.GetFirstPointerSlot(), array.GetDescriptorSlot(0)); - // VisitDescriptors(array, array.number_of_descriptors()); - // } - // V8_INLINE void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { - // if (!ShouldVisit(table)) return; - // weak_objects_->ephemeron_hash_tables.Push(task_id_, table); - - // for (i::InternalIndex i : table.IterateEntries()) { - // // i::ObjectSlot key_slot = - // // table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); - // // i::HeapObject key = i::HeapObject::cast(table.KeyAt(i)); - - // // SynchronizePageAccess(key); - // // RecordSlot(table, key_slot, key); - - // i::ObjectSlot value_slot = - // table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); - - // // if (marking_state()->IsBlackOrGrey(key)) { - // VisitPointer(table, value_slot); - // // } else { - // // i::Object value_obj = table.ValueAt(i); - - // // if (value_obj.IsHeapObject()) { - // // i::HeapObject value = i::HeapObject::cast(value_obj); - // // // SynchronizePageAccess(value); - // // // RecordSlot(table, value_slot, value); - - // // // Revisit ephemerons with both key and value unreachable at end - // // // of concurrent marking cycle. - // // if (marking_state()->IsWhite(value)) { - // // weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); - // // } - // // } - // // } - // } - // } - // V8_INLINE void VisitFixedArray(i::Map map, i::FixedArray object) { - // // Arrays with the progress bar are not left-trimmable because they reside - // // in the large object space. - // VisitLeftTrimmableArray(map, object); - // } - // V8_INLINE void VisitFixedDoubleArray(i::Map map, i::FixedDoubleArray object) { - // VisitLeftTrimmableArray(map, object); - // } - // V8_INLINE void VisitJSApiObject(i::Map map, i::JSObject object) { - // VisitEmbedderTracingSubclass(map, object); - // } - // V8_INLINE void VisitJSArrayBuffer(i::Map map, i::JSArrayBuffer object) { - // object.MarkExtension(); - // VisitEmbedderTracingSubclass(map, object); - // } - // V8_INLINE void VisitJSDataView(i::Map map, i::JSDataView object) { - // VisitEmbedderTracingSubclass(map, object); - // } + V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { + skip_weak_ = true; VisitJSObjectSubclass(map, object); + skip_weak_ = false; // Check if the JSFunction needs reset due to bytecode being flushed. if (/*bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode &&*/ object.NeedsResetDueToFlushedBytecode()) { weak_objects_->flushed_js_functions.Push(task_id_, object); } } - // V8_INLINE void VisitJSTypedArray(i::Map map, i::JSTypedArray object) { - // VisitEmbedderTracingSubclass(map, object); - // } - // V8_INLINE void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { - // VisitJSObjectSubclass(map, weak_ref); - // // if (size == 0) return 0; - // if (weak_ref.target().IsHeapObject()) { - // // i::HeapObject target = i::HeapObject::cast(weak_ref.target()); - // // SynchronizePageAccess(target); - // // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target)) { - // // // Record the slot inside the JSWeakRef, since the - // // // VisitJSObjectSubclass above didn't visit it. - // // ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); - // // concrete_visitor()->RecordSlot(weak_ref, slot, target); - // // } else { - // // JSWeakRef points to a potentially dead object. We have to process - // // them when we know the liveness of the whole transitive closure. - // weak_objects_->js_weak_refs.Push(task_id_, weak_ref); - // // } - // } - // } - // V8_INLINE void VisitMap(i::Map meta_map, i::Map map) { - // if (!ShouldVisit(map)) return; - // i::Map::BodyDescriptor::SizeOf(meta_map, map); - // // VisitDescriptorsForMap(map); - - // // Mark the pointer fields of the Map. If there is a transitions array, it has - // // been marked already, so it is fine that one of these fields contains a - // // pointer to it. - // i::Map::BodyDescriptor::IterateBody(meta_map, map, map.Size(), this); - // } + V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { // if (!ShouldVisit(shared_info)) return; int size = i::SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); VisitMapPointer(shared_info); + skip_weak_ = true; i::SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); + skip_weak_ = false; auto data = shared_info.function_data(v8::kAcquireLoad); if (data.IsHeapObject() || data.IsWeak()) { @@ -260,6 +245,14 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } } + template + void VisitJSObjectSubclass(i::Map map, T object) { + // if (!ShouldVisit(object)) return; + VisitMapPointer(object); + int size = T::BodyDescriptor::SizeOf(map, object); + T::BodyDescriptor::IterateBody(map, object, size, this); + } + V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { // if (!ShouldVisit(array)) return; VisitMapPointer(array); @@ -267,143 +260,16 @@ class MMTkEdgeVisitor: public i::HeapVisitor { i::TransitionArray::BodyDescriptor::IterateBody(map, array, size, this); weak_objects_->transition_arrays.Push(task_id_, array); } - // V8_INLINE void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { - // if (!ShouldVisit(weak_cell)) return; - - // int size = i::WeakCell::BodyDescriptor::SizeOf(map, weak_cell); - // this->VisitMapPointer(weak_cell); - // i::WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this); - // // i::HeapObject target = weak_cell.relaxed_target(); - // // i::HeapObject unregister_token = weak_cell.relaxed_unregister_token(); - // // concrete_visitor()->SynchronizePageAccess(target); - // // concrete_visitor()->SynchronizePageAccess(unregister_token); - // // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target) && - // // concrete_visitor()->marking_state()->IsBlackOrGrey(unregister_token)) { - // // // Record the slots inside the WeakCell, since the IterateBody above - // // // didn't visit it. - // // ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); - // // concrete_visitor()->RecordSlot(weak_cell, slot, target); - // // slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); - // // concrete_visitor()->RecordSlot(weak_cell, slot, unregister_token); - // // } else { - // // WeakCell points to a potentially dead object or a dead unregister - // // token. We have to process them when we know the liveness of the whole - // // transitive closure. - // weak_objects_->weak_cells.Push(task_id_, weak_cell); - // // } - // } - - // void VisitDescriptors(i::DescriptorArray descriptor_array, int number_of_own_descriptors) { - // // int16_t new_marked = static_cast(number_of_own_descriptors); - // // int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors( - // // mark_compact_epoch_, new_marked); - // // if (old_marked < new_marked) { - // // VisitPointers(descriptor_array, - // // MaybeObjectSlot(descriptor_array.GetDescriptorSlot(old_marked)), - // // MaybeObjectSlot(descriptor_array.GetDescriptorSlot(new_marked))); - // // } - // } - - // template - // void VisitLeftTrimmableArray(i::Map map, T object) { - // // The length() function checks that the length is a Smi. - // // This is not necessarily the case if the array is being left-trimmed. - // i::Object length = object.unchecked_length(v8::kAcquireLoad); - // if (!ShouldVisit(object)) return; - // // The cached length must be the actual length as the array is not black. - // // Left trimming marks the array black before over-writing the length. - // DCHECK(length.IsSmi()); - // int size = T::SizeFor(i::Smi::ToInt(length)); - // VisitMapPointer(object); - // T::BodyDescriptor::IterateBody(map, object, size, this); - // } - - // template - // void VisitEmbedderTracingSubclass(i::Map map, T object) { - // DCHECK(object.IsApiWrapper()); - // VisitJSObjectSubclass(map, object); - // // if (size && is_embedder_tracing_enabled_) { - // // // Success: The object needs to be processed for embedder references on - // // // the main thread. - // // local_marking_worklists_->PushEmbedder(object); - // // } - // } - template - void VisitJSObjectSubclass(i::Map map, T object) { - // if (!ShouldVisit(object)) return; - VisitMapPointer(object); - int size = T::BodyDescriptor::SizeOf(map, object); - T::BodyDescriptor::IterateBody(map, object, size, this); + // Custom weak pointers must be ignored by the GC but not other + // visitors. They're used for e.g., lists that are recreated after GC. The + // default implementation treats them as strong pointers. Visitors who want to + // ignore them must override this function with empty. + virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { + if (!skip_weak_) VisitPointers(host, start, end); } - - // V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { - // // If the SharedFunctionInfo has old bytecode, mark it as flushable, - // // otherwise visit the function data field strongly. - // if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { - // weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); - // } else { - // VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); - // } - // } - - // V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { - // if (v8::internal::Heap::GetBytecodeFlushMode(heap_->isolate()) != i::BytecodeFlushMode::kDoNotFlushBytecode && object.NeedsResetDueToFlushedBytecode()) { - // weak_objects_->flushed_js_functions.Push(task_id_, object); - // } - // } - - // V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { - // weak_objects_->transition_arrays.Push(task_id_, array); - // } - - // void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { - // weak_objects_->ephemeron_hash_tables.Push(task_id_, table); - // for (i::InternalIndex i : table.IterateEntries()) { - // // ObjectSlot key_slot = - // // table.RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i)); - // auto key = i::HeapObject::cast(table.KeyAt(i)); - - // // concrete_visitor()->SynchronizePageAccess(key); - // // concrete_visitor()->RecordSlot(table, key_slot, key); - - // // ObjectSlot value_slot = table.RawFieldOfElementAt(EphemeronHashTable::EntryToValueIndex(i)); - - // // if (concrete_visitor()->marking_state()->IsBlackOrGrey(key)) { - // // VisitPointer(table, value_slot); - // // } else { - // auto value_obj = table.ValueAt(i); - // if (value_obj.IsHeapObject()) { - // auto value = i::HeapObject::cast(value_obj); - // // concrete_visitor()->SynchronizePageAccess(value); - // // concrete_visitor()->RecordSlot(table, value_slot, value); - // // Revisit ephemerons with both key and value unreachable at end - // // of concurrent marking cycle. - // if (!mmtk::is_live(value)) { - // weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); - // } - // } - // // } - // } - // } - - // void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { - // auto target = weak_cell.relaxed_target(); - // auto unregister_token = weak_cell.relaxed_unregister_token(); - // if (!mmtk::is_live(target) || !mmtk::is_live(unregister_token)) { - // // WeakCell points to a potentially dead object or a dead unregister - // // token. We have to process them when we know the liveness of the whole - // // transitive closure. - // weak_objects_->weak_cells.Push(task_id_, weak_cell); - // } - // } - - // void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { - // if (weak_ref.target().IsHeapObject()) { - // weak_objects_->js_weak_refs.Push(task_id_, weak_ref); - // } - // } + bool skip_weak_ = false; #endif @@ -440,23 +306,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { ProcessEdge(host, host.map_slot()); } - // Custom weak pointers must be ignored by the GC but not other - // visitors. They're used for e.g., lists that are recreated after GC. The - // default implementation treats them as strong pointers. Visitors who want to - // ignore them must override this function with empty. -#ifdef WEAKREF_PROCESSING - virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { - // // VisitPointers(host, start, end); - // if (!(host.IsWeakCell() || host.IsJSWeakRef())) { - // VisitPointers(host, start, end); - // return; - // } - // // DCHECK(host.IsWeakCell() || host.IsJSWeakRef()); - // // for (auto p = start; p < end; ++p) { - // // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, i::HeapObjectSlot(p.address()))); - // // } - } -#endif private: template V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { @@ -468,16 +317,12 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } else if (TSlot::kCanBeWeak && (*slot).GetHeapObjectIfWeak(&object)) { if (!WEAKREF_PROCESSING_BOOL) { PushEdge((void*) slot.address()); - } else { - // if (auto f = mmtk::get_forwarded_ref(object)) { + // } else if (auto f = mmtk::get_forwarded_ref(object)) { // i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); // s.StoreHeapObject(*f); - // } else { - // PushEdge((void*) slot.address()); - // // ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); + } else { i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); - // } } } } diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 072efac..054c389 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -519,6 +519,8 @@ class WeakRefs { DCHECK(!target.IsUndefined()); // The value of the WeakCell is dead. auto finalization_registry = i::JSFinalizationRegistry::cast(weak_cell.finalization_registry()); + if (auto f = get_forwarded_ref(finalization_registry)) finalization_registry = i::JSFinalizationRegistry::cast(*f); + DCHECK(is_live(finalization_registry)); if (!finalization_registry.scheduled_for_cleanup()) { heap()->EnqueueDirtyJSFinalizationRegistry(finalization_registry, gc_notify_updated_slot); } @@ -529,7 +531,10 @@ class WeakRefs { DCHECK(finalization_registry.NeedsCleanup()); DCHECK(finalization_registry.scheduled_for_cleanup()); } else { - DCHECK(!get_forwarded_ref(target)); + i::ObjectSlot slot = weak_cell.RawField(i::WeakCell::kTargetOffset); + if (auto f = get_forwarded_ref(target)) { + slot.store(*f); + } // The value of the WeakCell is alive. // i::ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); @@ -552,9 +557,9 @@ class WeakRefs { }, gc_notify_updated_slot); } else { - DCHECK(!get_forwarded_ref(unregister_token)); - // auto f = get_forwarded_ref(unregister_token); - // if (f) *slot = *f; + if (auto f = get_forwarded_ref(unregister_token)) { + slot.store(*f); + } // The unregister_token is alive. // ObjectSlot slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); From 76df2d4ed5a9267df39a4925cab1faa33d2630c3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 25 Jun 2021 14:24:23 +1000 Subject: [PATCH 39/65] Ephemeron support --- mmtk/src/lib.rs | 1 + mmtk/src/scanning.rs | 35 +++++++++++++++++++- v8/third_party/heap/mmtk/mmtk-visitors.h | 38 ++++++++++++++++++++++ v8/third_party/heap/mmtk/mmtk.h | 1 + v8/third_party/heap/mmtk/mmtkUpcalls.cc | 10 ++++++ v8/third_party/heap/mmtk/weak-refs.h | 41 ++++++++++++++++++++++-- 6 files changed, 122 insertions(+), 4 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 962639d..24d054e 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -58,6 +58,7 @@ pub struct V8_Upcalls { pub scan_objects: extern "C" fn(objects: *const ObjectReference, count: usize, process_edges: ProcessEdgesFn, trace_field: TraceFieldFn, context: *mut c_void, task_id: usize), pub process_weak_refs: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void), pub on_move_event: extern "C" fn(from: ObjectReference, to: ObjectReference, size: usize), + pub process_ephemerons: extern "C" fn(trace_root: TraceRootFn, context: *mut c_void, task_id: usize), } pub static mut UPCALLS: *const V8_Upcalls = null_mut(); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index b776f15..0a8fbd5 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -56,7 +56,19 @@ impl Scanning for VMScanning { unimplemented!() } - fn scan_vm_specific_roots>() { + fn scan_vm_specific_roots>(worker: &mut GCWorker) { + let x = worker as *mut GCWorker as usize; + SINGLETON.scheduler.on_closure_end(Box::new(move || { + unsafe { + let w = x as *mut GCWorker; + debug_assert!(ROOT_OBJECTS.is_empty()); + ((*UPCALLS).process_ephemerons)(trace_root:: as _, w as _, (*w).ordinal); + if !ROOT_OBJECTS.is_empty() { + flush_roots::(&mut *w); + } + debug_assert!(ROOT_OBJECTS.is_empty()); + } + })); mmtk::memory_manager::add_work_packet( &SINGLETON, WorkBucketStage::Closure, @@ -69,6 +81,27 @@ impl Scanning for VMScanning { } } +pub struct ProcessEphemerons>(PhantomData); + +impl> ProcessEphemerons { + pub fn new() -> Self { + Self(PhantomData) + } +} + +impl> GCWork for ProcessEphemerons { + fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + unsafe { + debug_assert!(ROOT_OBJECTS.is_empty()); + ((*UPCALLS).process_ephemerons)(trace_root:: as _, worker as *mut _ as _, worker.ordinal); + if !ROOT_OBJECTS.is_empty() { + flush_roots::(worker); + } + debug_assert!(ROOT_OBJECTS.is_empty()); + } + } +} + pub struct ScanAndForwardRoots>(PhantomData); impl> ScanAndForwardRoots { diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index b68e0b7..a1d4cf2 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -128,6 +128,44 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // V8_INLINE bool AllowDefaultJSObjectVisit() { return false; } #ifdef WEAKREF_PROCESSING + void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { + // if (!concrete_visitor()->ShouldVisit(table)) return 0; + weak_objects_->ephemeron_hash_tables.Push(task_id_, table); + + for (auto i : table.IterateEntries()) { + auto key_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); + auto key = i::HeapObject::cast(table.KeyAt(i)); + + // VisitPointer(table, key_slot); + // concrete_visitor()->SynchronizePageAccess(key); + // concrete_visitor()->RecordSlot(table, key_slot, key); + + auto value_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); + + if (is_live(key)) { + if (auto f = get_forwarded_ref(key)) { + key_slot.store(*f); + } + VisitPointer(table, value_slot); + } else { + auto value_obj = table.ValueAt(i); + if (value_obj.IsHeapObject()) { + auto value = i::HeapObject::cast(value_obj); + printf("discover ephemeron key=%p value=%p\n", (void*) key.ptr(), (void*) value.ptr()); + // concrete_visitor()->SynchronizePageAccess(value); + // concrete_visitor()->RecordSlot(table, value_slot, value); + + // // Revisit ephemerons with both key and value unreachable at end + // // of concurrent marking cycle. + // if (concrete_visitor()->marking_state()->IsWhite(value)) { + weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); + // } + } + } + } + // return table.SizeFromMap(map); + } + V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { // if (!ShouldVisit(array)) return; VisitMapPointer(array); diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index b52123d..0ee43f2 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -137,6 +137,7 @@ typedef struct { void (*scan_objects) (void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id); void (*process_weak_refs) (TraceRootFn process_edges, void* context); void (*on_move_event) (void* from, void* to, size_t size); + void (*process_ephemerons) (TraceRootFn process_edges, void* context, int task_id); } V8_Upcalls; /** diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 4b2f09a..7a28f18 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -165,6 +165,15 @@ static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn proce } } +static void mmtk_process_ephemerons(TraceRootFn trace_root, void* context, int task_id) { + main_thread_synchronizer->RunMainThreadTask([=]() { + mmtk::global_weakref_processor->trace_ = [=](void* slot) { + trace_root(slot, context); + }; + mmtk::global_weakref_processor->ProcessEphemerons(); + }); +} + V8_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -184,6 +193,7 @@ V8_Upcalls mmtk_upcalls = { mmtk_scan_objects, mmtk_process_weak_refs, mmtk_on_move_event, + mmtk_process_ephemerons, }; } // namespace third_party_heap } // namespace internal diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 054c389..c50406f 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -32,11 +32,9 @@ i::MaybeObject to_weakref(i::HeapObject o) { } base::Optional get_forwarded_ref(i::HeapObject o) { - DCHECK(o.IsStrong()); auto f = mmtk_get_forwarded_object(o); if (f != nullptr) { auto x = i::HeapObject::cast(i::Object((i::Address) f)); - DCHECK(o.IsStrong()); return x; } return base::nullopt; @@ -400,7 +398,15 @@ class WeakRefs { if (!is_live(key)) { table.RemoveEntry(i); } else { - DCHECK(!get_forwarded_ref(key)); + if (auto f = get_forwarded_ref(key)) { + auto key_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); + key_slot.store(*f); + } + auto value = i::HeapObject::cast(table.ValueAt(i)); + if (auto f = get_forwarded_ref(value)) { + auto value_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); + value_slot.store(*f); + } } } } @@ -613,7 +619,34 @@ class WeakRefs { } } + void ProcessEphemeron(i::Ephemeron ephemeron) { + if (is_live(ephemeron.key)) { + if (!is_live(ephemeron.value)) { + trace_((void*) &ephemeron.value); + } + } else { + weak_objects_.next_ephemerons.Push(kMainThreadTask, ephemeron); + } + } + public: + + + void ProcessEphemerons() { + Flush(); + + i::Ephemeron ephemeron; + + DCHECK(weak_objects_.current_ephemerons.IsEmpty()); + weak_objects_.current_ephemerons.Swap(weak_objects_.next_ephemerons); + while (weak_objects_.current_ephemerons.Pop(kMainThreadTask, &ephemeron)) { + ProcessEphemeron(ephemeron); + } + while (weak_objects_.discovered_ephemerons.Pop(kMainThreadTask, &ephemeron)) { + ProcessEphemeron(ephemeron); + } + } + std::function trace_ = [](void*) { UNREACHABLE(); }; static i::Isolate* isolate() { @@ -664,6 +697,8 @@ class WeakRefs { DCHECK(weak_objects_.weak_cells.IsEmpty()); DCHECK(weak_objects_.bytecode_flushing_candidates.IsEmpty()); DCHECK(weak_objects_.flushed_js_functions.IsEmpty()); + DCHECK(weak_objects_.discovered_ephemerons.IsEmpty()); + // DCHECK(weak_objects_.next_ephemerons.IsEmpty()); if (have_code_to_deoptimize_) { // Some code objects were marked for deoptimization during the GC. From ac905fbc3eb8feb7474f731d9ca324d7f1167c80 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 28 Jun 2021 16:47:36 +1000 Subject: [PATCH 40/65] WIP --- mmtk/src/api.rs | 5 ++- v8/third_party/heap/mmtk/mmtk-visitors.h | 39 ++++++++++++++++-------- v8/third_party/heap/mmtk/weak-refs.h | 34 ++++++++++++++++++++- 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 8c45ca6..2c9b54d 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -84,8 +84,11 @@ pub extern "C" fn alloc( size: usize, align: usize, offset: isize, - semantics: AllocationSemantics, + mut semantics: AllocationSemantics, ) -> Address { + if semantics == AllocationSemantics::Code { + semantics = AllocationSemantics::Los; + } let a = memory_manager::alloc::(mutator, size, align, offset, semantics); unsafe { memory_manager::post_alloc::(mutator, a.to_object_reference(), size, semantics); } a diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index a1d4cf2..031043c 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -101,6 +101,13 @@ class MMTkCustomRootBodyVisitor final : public i::ObjectVisitor { } private: + V8_INLINE void ProcessEdge(i::HeapObject host, i::ObjectSlot slot) { + auto object = *slot; + if (!object.IsHeapObject()) return; + trace_root_((void*) &object, context_); + *slot = object; + } + v8::internal::Heap* heap_; TraceRootFn trace_root_; void* context_; @@ -190,9 +197,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { int size = i::WeakCell::BodyDescriptor::SizeOf(map, weak_cell); this->VisitMapPointer(weak_cell); - skip_weak_ = true; i::WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this); - skip_weak_ = false; // i::HeapObject target = weak_cell.relaxed_target(); // i::HeapObject unregister_token = weak_cell.relaxed_unregister_token(); // concrete_visitor()->SynchronizePageAccess(target); @@ -214,9 +219,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } V8_INLINE void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { - skip_weak_ = true; VisitJSObjectSubclass(map, weak_ref); - skip_weak_ = false; // if (size == 0) return 0; if (weak_ref.target().IsHeapObject()) { // i::HeapObject target = i::HeapObject::cast(weak_ref.target()); @@ -238,18 +241,14 @@ class MMTkEdgeVisitor: public i::HeapVisitor { if (!ShouldVisit(object)) return; int size = i::BytecodeArray::BodyDescriptor::SizeOf(map, object); this->VisitMapPointer(object); - skip_weak_ = true; i::BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); - skip_weak_ = false; // if (!is_forced_gc_) { object.MakeOlder(); // } } V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { - skip_weak_ = true; VisitJSObjectSubclass(map, object); - skip_weak_ = false; // Check if the JSFunction needs reset due to bytecode being flushed. if (/*bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode &&*/ object.NeedsResetDueToFlushedBytecode()) { @@ -261,9 +260,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // if (!ShouldVisit(shared_info)) return; int size = i::SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); VisitMapPointer(shared_info); - skip_weak_ = true; i::SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); - skip_weak_ = false; auto data = shared_info.function_data(v8::kAcquireLoad); if (data.IsHeapObject() || data.IsWeak()) { @@ -304,11 +301,22 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // default implementation treats them as strong pointers. Visitors who want to // ignore them must override this function with empty. virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { - if (!skip_weak_) VisitPointers(host, start, end); + // for (auto p = start; p < end; ++p) { + // printf("@weak? %p.%p -> %p\n", (void*) host.ptr(), (void*) p.address(), (void*) (*p).ptr()); + // i::HeapObject object; + // if (i::ObjectSlot::kCanBeWeak && (*p).GetHeapObjectIfWeak(&object)) { + // // printf("@weak %p.%p -> %p\n", (void*) host.ptr(), (void*) p.address(), (void*) (*p).ptr()); + // // auto s = i::HeapObjectSlot(p.address()); + // // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); + // } else if ((*p).GetHeapObjectIfStrong(&object)) { + + // printf("@string %p.%p -> %p\n", (void*) host.ptr(), (void*) p.address(), (void*) (*p).ptr()); + // auto s = i::HeapObjectSlot(p.address()); + // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); + // } + // } } - bool skip_weak_ = false; - #endif virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { @@ -466,6 +474,11 @@ class MMTkHeapVerifier: public i::RootVisitor, public i::ObjectVisitor { printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } CHECK(tph::Heap::IsValidHeapObject(o)); + CHECK(is_live(o)); + // if (get_forwarded_ref(o)) { + // printf("Unforwarded edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); + // } + // CHECK(!get_forwarded_ref(o)); mark_stack_.push_back(o); } } diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index c50406f..0453809 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -46,7 +46,7 @@ class MMTkWeakObjectRetainer: public i::WeakObjectRetainer { // LOG("RetainAs %p\n", (void*) object.ptr()); if (object == i::Object()) return object; auto heap_object = i::HeapObject::cast(object); - if (tph::Heap::IsValidHeapObject(heap_object)) { + if (is_live(heap_object)) { auto f = mmtk_get_forwarded_object(heap_object); if (f != nullptr) { // LOG("%p -> %p\n", (void*) object.ptr(), (void*) f); @@ -635,6 +635,36 @@ class WeakRefs { void ProcessEphemerons() { Flush(); + class XRootVisitor: public i::RootVisitor { + public: + explicit XRootVisitor(std::function& trace): trace_(trace) {} + + virtual void VisitRootPointer(i::Root root, const char* description, i::FullObjectSlot p) override final { + ProcessRootEdge(root, p); + } + virtual void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override final { + for (auto p = start; p < end; ++p) ProcessRootEdge(root, p); + } + virtual void VisitRootPointers(i::Root root, const char* description, i::OffHeapObjectSlot start, i::OffHeapObjectSlot end) override final { + for (auto p = start; p < end; ++p) ProcessRootEdge(root, p); + } + private: + V8_INLINE void ProcessRootEdge(i::Root root, i::FullObjectSlot slot) { + i::HeapObject object; + if ((*slot).GetHeapObject(&object)) { + trace_((void*) slot.address()); + } + } + std::function& trace_; + }; + + { + XRootVisitor root_visitor(trace_); + isolate()->global_handles()->IterateWeakRootsForFinalizers(&root_visitor); + // isolate()->global_handles()->IterateWeakRootsForPhantomHandles(&root_visitor); + } + + i::Ephemeron ephemeron; DCHECK(weak_objects_.current_ephemerons.IsEmpty()); @@ -688,6 +718,8 @@ class WeakRefs { ClearWeakCollections(); ClearJSWeakRefs(); + PROFILE(heap()->isolate(), WeakCodeClearEvent()); + MarkDependentCodeForDeoptimization(); DCHECK(weak_objects_.transition_arrays.IsEmpty()); From b9f5be898ffd985f9f560c281a8a90dde81bf050 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 28 Jun 2021 17:21:50 +1000 Subject: [PATCH 41/65] Check --- v8/third_party/heap/mmtk/mmtk-visitors.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 031043c..c99feca 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -475,10 +475,10 @@ class MMTkHeapVerifier: public i::RootVisitor, public i::ObjectVisitor { } CHECK(tph::Heap::IsValidHeapObject(o)); CHECK(is_live(o)); - // if (get_forwarded_ref(o)) { - // printf("Unforwarded edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); - // } - // CHECK(!get_forwarded_ref(o)); + if (get_forwarded_ref(o)) { + printf("Unforwarded edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); + } + CHECK(!get_forwarded_ref(o)); mark_stack_.push_back(o); } } From 7dcd0a80bc81d209972e32b27f3236a501d4ecbe Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 29 Jun 2021 11:00:09 +1000 Subject: [PATCH 42/65] Fix liveness check --- mmtk/src/api.rs | 10 +++-- mmtk/src/object_archive.rs | 2 +- v8/third_party/heap/mmtk/mmtk-visitors.h | 9 +++-- v8/third_party/heap/mmtk/mmtk.cc | 5 ++- v8/third_party/heap/mmtk/mmtk.h | 1 + v8/third_party/heap/mmtk/weak-refs.h | 51 +++++++++++++----------- 6 files changed, 45 insertions(+), 33 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 2c9b54d..265f77c 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -84,11 +84,8 @@ pub extern "C" fn alloc( size: usize, align: usize, offset: isize, - mut semantics: AllocationSemantics, + semantics: AllocationSemantics, ) -> Address { - if semantics == AllocationSemantics::Code { - semantics = AllocationSemantics::Los; - } let a = memory_manager::alloc::(mutator, size, align, offset, semantics); unsafe { memory_manager::post_alloc::(mutator, a.to_object_reference(), size, semantics); } a @@ -149,6 +146,11 @@ pub extern "C" fn is_live_object(object: ObjectReference) -> bool { object.is_live() } +#[no_mangle] +pub extern "C" fn is_live_object2(object: ObjectReference) -> bool { + object.is_reachable() +} + #[no_mangle] pub extern "C" fn is_mapped_object(object: ObjectReference) -> bool { object.is_mapped() diff --git a/mmtk/src/object_archive.rs b/mmtk/src/object_archive.rs index 23b56c4..3db1526 100644 --- a/mmtk/src/object_archive.rs +++ b/mmtk/src/object_archive.rs @@ -133,7 +133,7 @@ impl ObjectArchive { let mut new_space_map = HashMap::new(); for object in &self.untagged_objects { debug_assert_eq!(object.to_address().as_usize() & 0b11, 0); - if object.is_live() { + if object.is_reachable() || self.space_map[&object] == 0 { let new_object = object.get_forwarded_object().unwrap_or(*object); debug_assert_eq!(new_object.to_address().as_usize() & 0b11, 0); new_objects.push(new_object); diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index c99feca..a54543a 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -149,7 +149,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { auto value_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); - if (is_live(key)) { + if (is_live2(key)) { if (auto f = get_forwarded_ref(key)) { key_slot.store(*f); } @@ -355,7 +355,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { private: template V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { - DCHECK(mmtk::is_live(host)); + DCHECK(mmtk::is_live2(host)); DCHECK(!mmtk::get_forwarded_ref(host)); i::HeapObject object; if ((*slot).GetHeapObjectIfStrong(&object)) { @@ -474,7 +474,10 @@ class MMTkHeapVerifier: public i::RootVisitor, public i::ObjectVisitor { printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } CHECK(tph::Heap::IsValidHeapObject(o)); - CHECK(is_live(o)); + if (!is_live2(o)) { + printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); + } + CHECK(is_live2(o)); if (get_forwarded_ref(o)) { printf("Unforwarded edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 9a73d43..c0f9778 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -171,7 +171,10 @@ bool Heap::InLargeObjectSpace(Address address) { } bool Heap::IsValidHeapObject(HeapObject object) { - return is_live_object(reinterpret_cast(object.address())); + if (InReadOnlySpace(object.address())) { + return true; + } + return is_live_object2(reinterpret_cast(object.address())); } void Heap::ResetIterator() { diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 0ee43f2..c7f03e8 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -72,6 +72,7 @@ extern void post_alloc(MMTk_Mutator mutator, void* refer, int bytes, int allocator); extern bool is_live_object(void* ref); +extern bool is_live_object2(void* ref); extern bool is_mapped_object(void* ref); extern bool is_mapped_address(void* addr); extern void modify_check(void *mmtk, void* ref); diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 0453809..6f78b2d 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -22,8 +22,11 @@ namespace i = v8::internal; namespace base = v8::base; namespace tph = v8::internal::third_party_heap; -bool is_live(i::HeapObject o) { - return is_live_object(reinterpret_cast(o.address())); +bool is_live2(i::HeapObject o) { + if (tph::Heap::InReadOnlySpace(o.address())) { + return true; + } + return is_live_object2(reinterpret_cast(o.address())); } i::MaybeObject to_weakref(i::HeapObject o) { @@ -46,7 +49,7 @@ class MMTkWeakObjectRetainer: public i::WeakObjectRetainer { // LOG("RetainAs %p\n", (void*) object.ptr()); if (object == i::Object()) return object; auto heap_object = i::HeapObject::cast(object); - if (is_live(heap_object)) { + if (is_live2(heap_object)) { auto f = mmtk_get_forwarded_object(heap_object); if (f != nullptr) { // LOG("%p -> %p\n", (void*) object.ptr(), (void*) f); @@ -79,7 +82,7 @@ class InternalizedStringTableCleaner : public i::RootVisitor { if (o.IsHeapObject()) { auto heap_object = i::HeapObject::cast(o); DCHECK(!i::Heap::InYoungGeneration(heap_object)); - if (!is_live(heap_object)) { + if (!is_live2(heap_object)) { pointers_removed_++; // Set the entry to the_hole_value (as deleted). p.store(i::StringTable::deleted_element()); @@ -109,7 +112,7 @@ class ExternalStringTableCleaner : public i::RootVisitor { i::Object o = *p; if (o.IsHeapObject()) { auto heap_object = i::HeapObject::cast(o); - if (!is_live(heap_object)) { + if (!is_live2(heap_object)) { if (o.IsExternalString()) { heap_->FinalizeExternalString(i::String::cast(o)); } else { @@ -189,7 +192,7 @@ class WeakRefs { // Mark the uncompiled data as black, and ensure all fields have already been // marked. - DCHECK(is_live(inferred_name)); + DCHECK(is_live2(inferred_name)); // DCHECK(is_live(uncompiled_data)); trace_((void*) &uncompiled_data); @@ -214,7 +217,7 @@ class WeakRefs { flushing_candidate.set_function_data(*f, v8::kReleaseStore); } } - if (!is_live(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { + if (!is_live2(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { FlushBytecodeFromSFI(flushing_candidate); } else { DCHECK(!get_forwarded_ref(flushing_candidate.GetBytecodeArray(heap()->isolate()))); @@ -255,7 +258,7 @@ class WeakRefs { continue; } auto parent = i::Map::cast(map.constructor_or_back_pointer()); - bool parent_is_alive = is_live(parent); + bool parent_is_alive = is_live2(parent); if (parent_is_alive) { DCHECK(!get_forwarded_ref(parent)); } @@ -277,7 +280,7 @@ class WeakRefs { DCHECK(isolate()->has_active_deserializer()); DCHECK_EQ(raw_target.ToSmi(), i::Deserializer::uninitialized_field_value()); return false; - } else if (!is_live(i::TransitionsAccessor::GetTargetFromRaw(raw_target))) { + } else if (!is_live2(i::TransitionsAccessor::GetTargetFromRaw(raw_target))) { return true; } else { DCHECK(!get_forwarded_ref(i::TransitionsAccessor::GetTargetFromRaw(raw_target))); @@ -298,7 +301,7 @@ class WeakRefs { for (int i = 0; i < num_transitions; ++i) { i::Map target = transitions.GetTarget(i); DCHECK_EQ(target.constructor_or_back_pointer(), map); - if (!is_live(target)) { + if (!is_live2(target)) { if (!descriptors.is_null() && target.instance_descriptors(isolate()) == descriptors) { DCHECK(!target.is_prototype_map()); @@ -395,7 +398,7 @@ class WeakRefs { while (weak_objects_.ephemeron_hash_tables.Pop(kMainThreadTask, &table)) { for (i::InternalIndex i : table.IterateEntries()) { auto key = i::HeapObject::cast(table.KeyAt(i)); - if (!is_live(key)) { + if (!is_live2(key)) { table.RemoveEntry(i); } else { if (auto f = get_forwarded_ref(key)) { @@ -411,7 +414,7 @@ class WeakRefs { } } for (auto it = heap()->ephemeron_remembered_set_.begin(); it != heap()->ephemeron_remembered_set_.end();) { - if (!is_live(it->first)) { + if (!is_live2(it->first)) { it = heap()->ephemeron_remembered_set_.erase(it); } else { DCHECK(!get_forwarded_ref(it->first)); @@ -430,7 +433,7 @@ class WeakRefs { i::MaybeObjectSlot location(slot.second); if ((*location)->GetHeapObjectIfWeak(&value)) { DCHECK(!value.IsCell()); - if (is_live(value)) { + if (is_live2(value)) { // The value of the weak reference is alive. // RecordSlot(slot.first, HeapObjectSlot(location), value); auto forwarded = get_forwarded_ref(value); @@ -467,9 +470,9 @@ class WeakRefs { } void ClearPotentialSimpleMapTransition(i::Map dead_target) { - DCHECK(!is_live(dead_target)); + DCHECK(!is_live2(dead_target)); auto potential_parent = dead_target.constructor_or_back_pointer(); - if (is_live(i::HeapObject::cast(potential_parent))) { + if (is_live2(i::HeapObject::cast(potential_parent))) { if (auto f = get_forwarded_ref(i::HeapObject::cast(potential_parent))) { potential_parent = *f; } @@ -477,10 +480,10 @@ class WeakRefs { if (potential_parent.IsMap()) { auto parent = i::Map::cast(potential_parent); i::DisallowGarbageCollection no_gc_obviously; - if (is_live(parent)) { + if (is_live2(parent)) { DCHECK(!get_forwarded_ref(parent)); } - if (is_live(parent) && i::TransitionsAccessor(isolate(), parent, &no_gc_obviously).HasSimpleTransitionTo(dead_target)) { + if (is_live2(parent) && i::TransitionsAccessor(isolate(), parent, &no_gc_obviously).HasSimpleTransitionTo(dead_target)) { ClearPotentialSimpleMapTransition(parent, dead_target); } } @@ -503,7 +506,7 @@ class WeakRefs { i::JSWeakRef weak_ref; while (weak_objects_.js_weak_refs.Pop(kMainThreadTask, &weak_ref)) { auto target = i::HeapObject::cast(weak_ref.target()); - if (!is_live(target)) { + if (!is_live2(target)) { weak_ref.set_target(i::ReadOnlyRoots(isolate()).undefined_value()); } else { auto forwarded = get_forwarded_ref(weak_ref.target()); @@ -521,12 +524,12 @@ class WeakRefs { // } }; auto target = i::HeapObject::cast(weak_cell.target()); - if (!is_live(target)) { + if (!is_live2(target)) { DCHECK(!target.IsUndefined()); // The value of the WeakCell is dead. auto finalization_registry = i::JSFinalizationRegistry::cast(weak_cell.finalization_registry()); if (auto f = get_forwarded_ref(finalization_registry)) finalization_registry = i::JSFinalizationRegistry::cast(*f); - DCHECK(is_live(finalization_registry)); + DCHECK(is_live2(finalization_registry)); if (!finalization_registry.scheduled_for_cleanup()) { heap()->EnqueueDirtyJSFinalizationRegistry(finalization_registry, gc_notify_updated_slot); } @@ -548,7 +551,7 @@ class WeakRefs { i::ObjectSlot slot = weak_cell.RawField(i::WeakCell::kUnregisterTokenOffset); i::HeapObject unregister_token = i::HeapObject::cast(*slot);//weak_cell.unregister_token(); - if (!is_live(unregister_token)) { + if (!is_live2(unregister_token)) { // The unregister token is dead. Remove any corresponding entries in the // key map. Multiple WeakCell with the same token will have all their // unregister_token field set to undefined when processing the first @@ -579,7 +582,7 @@ class WeakRefs { while (weak_objects_.weak_objects_in_code.Pop(kMainThreadTask, &weak_object_in_code)) { auto object = weak_object_in_code.first; auto code = weak_object_in_code.second; - auto object_is_live = is_live(object); + auto object_is_live = is_live2(object); if (!object_is_live && !code.embedded_objects_cleared()) { if (!code.marked_for_deoptimization()) { code.SetMarkedForDeoptimization("weak objects"); @@ -620,8 +623,8 @@ class WeakRefs { } void ProcessEphemeron(i::Ephemeron ephemeron) { - if (is_live(ephemeron.key)) { - if (!is_live(ephemeron.value)) { + if (is_live2(ephemeron.key)) { + if (!is_live2(ephemeron.value)) { trace_((void*) &ephemeron.value); } } else { From bf8c16b7f8c69b466f4eecd3ebcd77cc1a0f9b29 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 29 Jun 2021 12:03:22 +1000 Subject: [PATCH 43/65] cleanup --- mmtk/src/api.rs | 13 +++-- mmtk/src/object_archive.rs | 6 +-- v8/third_party/heap/mmtk/mmtk-visitors.h | 8 +-- v8/third_party/heap/mmtk/mmtk.cc | 5 +- v8/third_party/heap/mmtk/mmtk.h | 31 ++++++++++- v8/third_party/heap/mmtk/weak-refs.h | 69 ++++++++---------------- 6 files changed, 65 insertions(+), 67 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 265f77c..e1adea3 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -142,13 +142,12 @@ pub extern "C" fn scan_region(mmtk: &mut MMTK) { } #[no_mangle] -pub extern "C" fn is_live_object(object: ObjectReference) -> bool { - object.is_live() -} - -#[no_mangle] -pub extern "C" fn is_live_object2(object: ObjectReference) -> bool { - object.is_reachable() +pub extern "C" fn mmtk_object_is_live(object: ObjectReference) -> usize { + debug_assert_eq!(object.to_address().as_usize() & 0b11, 0); + if crate::object_archive::global_object_archive().object_to_space(object.to_address()) == Some(0) { + return 1; + } + if object.is_reachable() { 1 } else { 0 } } #[no_mangle] diff --git a/mmtk/src/object_archive.rs b/mmtk/src/object_archive.rs index 3db1526..5482caf 100644 --- a/mmtk/src/object_archive.rs +++ b/mmtk/src/object_archive.rs @@ -54,7 +54,7 @@ pub extern "C" fn tph_archive_inner_to_obj( #[no_mangle] pub extern "C" fn tph_archive_obj_to_space(arch: *mut c_void, obj_ptr: *mut c_void) -> u8 { let arch = unsafe { Box::from_raw(arch as *mut ObjectArchive) }; - let res = arch.object_to_space(Address::from_mut_ptr(obj_ptr)); + let res = arch.object_to_space(Address::from_mut_ptr(obj_ptr)).unwrap(); Box::into_raw(arch); res } @@ -122,10 +122,10 @@ impl ObjectArchive { self.untagged_objects[idx].to_address() } - pub fn object_to_space(&self, mut obj_addr: Address) -> u8 { + pub fn object_to_space(&self, mut obj_addr: Address) -> Option { obj_addr = unsafe { Address::from_usize(obj_addr.as_usize() & !0b11) }; debug_assert_eq!(obj_addr.as_usize() & 0b11, 0); - unsafe { self.space_map[&obj_addr.to_object_reference()] } + unsafe { self.space_map.get(&obj_addr.to_object_reference()).cloned() } } pub fn update(&mut self) { diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index a54543a..cf77442 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -149,7 +149,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { auto value_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); - if (is_live2(key)) { + if (is_live(key)) { if (auto f = get_forwarded_ref(key)) { key_slot.store(*f); } @@ -355,7 +355,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { private: template V8_INLINE void ProcessEdge(i::HeapObject host, TSlot slot) { - DCHECK(mmtk::is_live2(host)); + DCHECK(mmtk::is_live(host)); DCHECK(!mmtk::get_forwarded_ref(host)); i::HeapObject object; if ((*slot).GetHeapObjectIfStrong(&object)) { @@ -474,10 +474,10 @@ class MMTkHeapVerifier: public i::RootVisitor, public i::ObjectVisitor { printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } CHECK(tph::Heap::IsValidHeapObject(o)); - if (!is_live2(o)) { + if (!is_live(o)) { printf("Dead edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } - CHECK(is_live2(o)); + CHECK(is_live(o)); if (get_forwarded_ref(o)) { printf("Unforwarded edge %p.%p -> %p\n", (void*) host.ptr(), (void*) edge, (void*) o.ptr()); } diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index c0f9778..014b4e8 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -171,10 +171,7 @@ bool Heap::InLargeObjectSpace(Address address) { } bool Heap::IsValidHeapObject(HeapObject object) { - if (InReadOnlySpace(object.address())) { - return true; - } - return is_live_object2(reinterpret_cast(object.address())); + return mmtk_object_is_live(reinterpret_cast(object.address())) != 0; } void Heap::ResetIterator() { diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index c7f03e8..f87feaa 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -4,6 +4,8 @@ #include "src/heap/third-party/heap-api.h" #include "src/base/address-region.h" #include "src/heap/heap.h" +#include "src/objects/objects.h" +#include "src/objects/objects-inl.h" #include "src/execution/isolate.h" #include #include @@ -71,8 +73,7 @@ extern void* alloc_slow(MMTk_Mutator mutator, size_t size, extern void post_alloc(MMTk_Mutator mutator, void* refer, int bytes, int allocator); -extern bool is_live_object(void* ref); -extern bool is_live_object2(void* ref); +extern size_t mmtk_object_is_live(void* ref); extern bool is_mapped_object(void* ref); extern bool is_mapped_address(void* addr); extern void modify_check(void *mmtk, void* ref); @@ -164,4 +165,30 @@ extern void* mmtk_get_forwarded_object(v8::internal::Object o); } #endif +// Helpers + +namespace mmtk { + namespace i = v8::internal; + namespace base = v8::base; + namespace tph = v8::internal::third_party_heap; + + V8_INLINE bool is_live(i::HeapObject o) { + return mmtk_object_is_live(reinterpret_cast(o.address())) != 0; + } + + V8_INLINE i::MaybeObject to_weakref(i::HeapObject o) { + DCHECK(o.IsStrong()); + return i::MaybeObject::MakeWeak(i::MaybeObject::FromObject(o)); + } + + V8_INLINE base::Optional get_forwarded_ref(i::HeapObject o) { + auto f = mmtk_get_forwarded_object(o); + if (f != nullptr) { + auto x = i::HeapObject::cast(i::Object((i::Address) f)); + return x; + } + return base::nullopt; + } +} + #endif // MMTK_H \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 6f78b2d..10b17eb 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -18,38 +18,13 @@ extern v8::internal::Heap* v8_heap; namespace mmtk { -namespace i = v8::internal; -namespace base = v8::base; -namespace tph = v8::internal::third_party_heap; - -bool is_live2(i::HeapObject o) { - if (tph::Heap::InReadOnlySpace(o.address())) { - return true; - } - return is_live_object2(reinterpret_cast(o.address())); -} - -i::MaybeObject to_weakref(i::HeapObject o) { - DCHECK(o.IsStrong()); - return i::MaybeObject::MakeWeak(i::MaybeObject::FromObject(o)); -} - -base::Optional get_forwarded_ref(i::HeapObject o) { - auto f = mmtk_get_forwarded_object(o); - if (f != nullptr) { - auto x = i::HeapObject::cast(i::Object((i::Address) f)); - return x; - } - return base::nullopt; -} - class MMTkWeakObjectRetainer: public i::WeakObjectRetainer { public: virtual i::Object RetainAs(i::Object object) override final { // LOG("RetainAs %p\n", (void*) object.ptr()); if (object == i::Object()) return object; auto heap_object = i::HeapObject::cast(object); - if (is_live2(heap_object)) { + if (is_live(heap_object)) { auto f = mmtk_get_forwarded_object(heap_object); if (f != nullptr) { // LOG("%p -> %p\n", (void*) object.ptr(), (void*) f); @@ -82,7 +57,7 @@ class InternalizedStringTableCleaner : public i::RootVisitor { if (o.IsHeapObject()) { auto heap_object = i::HeapObject::cast(o); DCHECK(!i::Heap::InYoungGeneration(heap_object)); - if (!is_live2(heap_object)) { + if (!is_live(heap_object)) { pointers_removed_++; // Set the entry to the_hole_value (as deleted). p.store(i::StringTable::deleted_element()); @@ -112,7 +87,7 @@ class ExternalStringTableCleaner : public i::RootVisitor { i::Object o = *p; if (o.IsHeapObject()) { auto heap_object = i::HeapObject::cast(o); - if (!is_live2(heap_object)) { + if (!is_live(heap_object)) { if (o.IsExternalString()) { heap_->FinalizeExternalString(i::String::cast(o)); } else { @@ -192,7 +167,7 @@ class WeakRefs { // Mark the uncompiled data as black, and ensure all fields have already been // marked. - DCHECK(is_live2(inferred_name)); + DCHECK(is_live(inferred_name)); // DCHECK(is_live(uncompiled_data)); trace_((void*) &uncompiled_data); @@ -217,7 +192,7 @@ class WeakRefs { flushing_candidate.set_function_data(*f, v8::kReleaseStore); } } - if (!is_live2(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { + if (!is_live(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { FlushBytecodeFromSFI(flushing_candidate); } else { DCHECK(!get_forwarded_ref(flushing_candidate.GetBytecodeArray(heap()->isolate()))); @@ -258,7 +233,7 @@ class WeakRefs { continue; } auto parent = i::Map::cast(map.constructor_or_back_pointer()); - bool parent_is_alive = is_live2(parent); + bool parent_is_alive = is_live(parent); if (parent_is_alive) { DCHECK(!get_forwarded_ref(parent)); } @@ -280,7 +255,7 @@ class WeakRefs { DCHECK(isolate()->has_active_deserializer()); DCHECK_EQ(raw_target.ToSmi(), i::Deserializer::uninitialized_field_value()); return false; - } else if (!is_live2(i::TransitionsAccessor::GetTargetFromRaw(raw_target))) { + } else if (!is_live(i::TransitionsAccessor::GetTargetFromRaw(raw_target))) { return true; } else { DCHECK(!get_forwarded_ref(i::TransitionsAccessor::GetTargetFromRaw(raw_target))); @@ -301,7 +276,7 @@ class WeakRefs { for (int i = 0; i < num_transitions; ++i) { i::Map target = transitions.GetTarget(i); DCHECK_EQ(target.constructor_or_back_pointer(), map); - if (!is_live2(target)) { + if (!is_live(target)) { if (!descriptors.is_null() && target.instance_descriptors(isolate()) == descriptors) { DCHECK(!target.is_prototype_map()); @@ -398,7 +373,7 @@ class WeakRefs { while (weak_objects_.ephemeron_hash_tables.Pop(kMainThreadTask, &table)) { for (i::InternalIndex i : table.IterateEntries()) { auto key = i::HeapObject::cast(table.KeyAt(i)); - if (!is_live2(key)) { + if (!is_live(key)) { table.RemoveEntry(i); } else { if (auto f = get_forwarded_ref(key)) { @@ -414,7 +389,7 @@ class WeakRefs { } } for (auto it = heap()->ephemeron_remembered_set_.begin(); it != heap()->ephemeron_remembered_set_.end();) { - if (!is_live2(it->first)) { + if (!is_live(it->first)) { it = heap()->ephemeron_remembered_set_.erase(it); } else { DCHECK(!get_forwarded_ref(it->first)); @@ -433,7 +408,7 @@ class WeakRefs { i::MaybeObjectSlot location(slot.second); if ((*location)->GetHeapObjectIfWeak(&value)) { DCHECK(!value.IsCell()); - if (is_live2(value)) { + if (is_live(value)) { // The value of the weak reference is alive. // RecordSlot(slot.first, HeapObjectSlot(location), value); auto forwarded = get_forwarded_ref(value); @@ -470,9 +445,9 @@ class WeakRefs { } void ClearPotentialSimpleMapTransition(i::Map dead_target) { - DCHECK(!is_live2(dead_target)); + DCHECK(!is_live(dead_target)); auto potential_parent = dead_target.constructor_or_back_pointer(); - if (is_live2(i::HeapObject::cast(potential_parent))) { + if (is_live(i::HeapObject::cast(potential_parent))) { if (auto f = get_forwarded_ref(i::HeapObject::cast(potential_parent))) { potential_parent = *f; } @@ -480,10 +455,10 @@ class WeakRefs { if (potential_parent.IsMap()) { auto parent = i::Map::cast(potential_parent); i::DisallowGarbageCollection no_gc_obviously; - if (is_live2(parent)) { + if (is_live(parent)) { DCHECK(!get_forwarded_ref(parent)); } - if (is_live2(parent) && i::TransitionsAccessor(isolate(), parent, &no_gc_obviously).HasSimpleTransitionTo(dead_target)) { + if (is_live(parent) && i::TransitionsAccessor(isolate(), parent, &no_gc_obviously).HasSimpleTransitionTo(dead_target)) { ClearPotentialSimpleMapTransition(parent, dead_target); } } @@ -506,7 +481,7 @@ class WeakRefs { i::JSWeakRef weak_ref; while (weak_objects_.js_weak_refs.Pop(kMainThreadTask, &weak_ref)) { auto target = i::HeapObject::cast(weak_ref.target()); - if (!is_live2(target)) { + if (!is_live(target)) { weak_ref.set_target(i::ReadOnlyRoots(isolate()).undefined_value()); } else { auto forwarded = get_forwarded_ref(weak_ref.target()); @@ -524,12 +499,12 @@ class WeakRefs { // } }; auto target = i::HeapObject::cast(weak_cell.target()); - if (!is_live2(target)) { + if (!is_live(target)) { DCHECK(!target.IsUndefined()); // The value of the WeakCell is dead. auto finalization_registry = i::JSFinalizationRegistry::cast(weak_cell.finalization_registry()); if (auto f = get_forwarded_ref(finalization_registry)) finalization_registry = i::JSFinalizationRegistry::cast(*f); - DCHECK(is_live2(finalization_registry)); + DCHECK(is_live(finalization_registry)); if (!finalization_registry.scheduled_for_cleanup()) { heap()->EnqueueDirtyJSFinalizationRegistry(finalization_registry, gc_notify_updated_slot); } @@ -551,7 +526,7 @@ class WeakRefs { i::ObjectSlot slot = weak_cell.RawField(i::WeakCell::kUnregisterTokenOffset); i::HeapObject unregister_token = i::HeapObject::cast(*slot);//weak_cell.unregister_token(); - if (!is_live2(unregister_token)) { + if (!is_live(unregister_token)) { // The unregister token is dead. Remove any corresponding entries in the // key map. Multiple WeakCell with the same token will have all their // unregister_token field set to undefined when processing the first @@ -582,7 +557,7 @@ class WeakRefs { while (weak_objects_.weak_objects_in_code.Pop(kMainThreadTask, &weak_object_in_code)) { auto object = weak_object_in_code.first; auto code = weak_object_in_code.second; - auto object_is_live = is_live2(object); + auto object_is_live = is_live(object); if (!object_is_live && !code.embedded_objects_cleared()) { if (!code.marked_for_deoptimization()) { code.SetMarkedForDeoptimization("weak objects"); @@ -623,8 +598,8 @@ class WeakRefs { } void ProcessEphemeron(i::Ephemeron ephemeron) { - if (is_live2(ephemeron.key)) { - if (!is_live2(ephemeron.value)) { + if (is_live(ephemeron.key)) { + if (!is_live(ephemeron.value)) { trace_((void*) &ephemeron.value); } } else { From c7bf052a6ab401fd28f115ba6930d167967def8a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 29 Jun 2021 13:20:23 +1000 Subject: [PATCH 44/65] Fix space check --- mmtk/src/api.rs | 8 ++++-- mmtk/src/object_archive.rs | 43 +++--------------------------- mmtk/src/object_model.rs | 1 - v8/third_party/heap/mmtk/mmtk.cc | 40 +++++----------------------- v8/third_party/heap/mmtk/mmtk.h | 45 +++++++++++++++++++++++++++----- 5 files changed, 56 insertions(+), 81 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index e1adea3..d059cc1 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -66,8 +66,12 @@ pub extern "C" fn bind_mutator( #[no_mangle] pub unsafe extern "C" fn mmtk_in_space(mmtk: &'static MMTK, object: ObjectReference, space: AllocationSemantics) -> i32 { match space { + AllocationSemantics::Default => mmtk.plan.in_default_space(object) as _, AllocationSemantics::ReadOnly => mmtk.plan.base().ro_space.in_space(object) as _, - _ => unreachable!() + AllocationSemantics::Immortal => mmtk.plan.common().immortal.in_space(object) as _, + AllocationSemantics::Los => mmtk.plan.common().los.in_space(object) as _, + AllocationSemantics::Code => mmtk.plan.base().code_space.in_space(object) as _, + AllocationSemantics::LargeCode => mmtk.plan.base().code_lo_space.in_space(object) as _, } } @@ -144,7 +148,7 @@ pub extern "C" fn scan_region(mmtk: &mut MMTK) { #[no_mangle] pub extern "C" fn mmtk_object_is_live(object: ObjectReference) -> usize { debug_assert_eq!(object.to_address().as_usize() & 0b11, 0); - if crate::object_archive::global_object_archive().object_to_space(object.to_address()) == Some(0) { + if crate::SINGLETON.plan.base().ro_space.in_space(object) { return 1; } if object.is_reachable() { 1 } else { 0 } diff --git a/mmtk/src/object_archive.rs b/mmtk/src/object_archive.rs index 5482caf..da52e23 100644 --- a/mmtk/src/object_archive.rs +++ b/mmtk/src/object_archive.rs @@ -1,8 +1,7 @@ use libc::c_void; use mmtk::util::ObjectReference; use mmtk::util::address::Address; -use std::collections::HashMap; -use std::sync::Mutex; +use mmtk::policy::space::Space; lazy_static! { @@ -51,25 +50,16 @@ pub extern "C" fn tph_archive_inner_to_obj( res.to_mut_ptr() } -#[no_mangle] -pub extern "C" fn tph_archive_obj_to_space(arch: *mut c_void, obj_ptr: *mut c_void) -> u8 { - let arch = unsafe { Box::from_raw(arch as *mut ObjectArchive) }; - let res = arch.object_to_space(Address::from_mut_ptr(obj_ptr)).unwrap(); - Box::into_raw(arch); - res -} - #[no_mangle] pub extern "C" fn tph_archive_insert( arch: *mut c_void, obj_ptr: *mut c_void, iso_ptr: *mut c_void, - space: u8, ) { let obj_addr = Address::from_mut_ptr(obj_ptr); let iso_addr = Address::from_mut_ptr(iso_ptr); let mut arch = unsafe { Box::from_raw(arch as *mut ObjectArchive) }; - arch.insert_object(obj_addr, iso_addr, space); + arch.insert_object(obj_addr, iso_addr); Box::into_raw(arch); } @@ -84,7 +74,6 @@ pub extern "C" fn tph_archive_remove(arch: *mut c_void, obj_ptr: *mut c_void) { #[derive(Default)] pub struct ObjectArchive { untagged_objects: Vec, - space_map: HashMap, iter_pos: usize, iter_len: usize, } @@ -94,11 +83,10 @@ impl ObjectArchive { Default::default() } - pub fn insert_object(&mut self, addr: Address, _isolate: Address, space: u8) { + pub fn insert_object(&mut self, addr: Address, _isolate: Address) { let untagged_object = unsafe { Address::from_usize(addr.as_usize() & !0b11).to_object_reference() }; - self.space_map.insert(untagged_object, space); match self.untagged_objects.binary_search_by(|o| o.to_address().cmp(&untagged_object.to_address())) { Ok(_) => unreachable!(), Err(idx) => { @@ -109,7 +97,6 @@ impl ObjectArchive { pub fn remove_object(&mut self, addr: Address) { let untagged_object = unsafe { Address::from_usize(addr.as_usize() & !0b11).to_object_reference() }; - self.space_map.remove(&untagged_object); let index = self.untagged_objects.iter().position(|x| *x == untagged_object).unwrap(); self.untagged_objects.remove(index); } @@ -122,41 +109,19 @@ impl ObjectArchive { self.untagged_objects[idx].to_address() } - pub fn object_to_space(&self, mut obj_addr: Address) -> Option { - obj_addr = unsafe { Address::from_usize(obj_addr.as_usize() & !0b11) }; - debug_assert_eq!(obj_addr.as_usize() & 0b11, 0); - unsafe { self.space_map.get(&obj_addr.to_object_reference()).cloned() } - } - pub fn update(&mut self) { let mut new_objects = vec![]; - let mut new_space_map = HashMap::new(); for object in &self.untagged_objects { debug_assert_eq!(object.to_address().as_usize() & 0b11, 0); - if object.is_reachable() || self.space_map[&object] == 0 { + if object.is_reachable() || crate::SINGLETON.plan.base().ro_space.in_space(*object) { let new_object = object.get_forwarded_object().unwrap_or(*object); debug_assert_eq!(new_object.to_address().as_usize() & 0b11, 0); new_objects.push(new_object); - new_space_map.insert(new_object, self.space_map[&object]); } } new_objects.dedup(); new_objects.sort_by(|a, b| a.to_address().cmp(&b.to_address())); self.untagged_objects = new_objects; - self.space_map = new_space_map; - } - - pub fn copy(&mut self, from: Address, to: Address) { - lazy_static! { - static ref LOCK: Mutex<()> = Mutex::default(); - } - let _lock = LOCK.lock().unwrap(); - unsafe { - let space = self.space_map[&from.to_object_reference()]; - self.space_map.insert(to.to_object_reference(), space); - // self.untagged_objects.push(to.to_object_reference()); - // self.untagged_objects.sort_by(|a, b| a.to_address().cmp(&b.to_address())); - } } pub fn reset_iterator(&mut self) { diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 264022a..3617b22 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -27,7 +27,6 @@ impl ObjectModel for VMObjectModel { let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, unsafe { Address::zero() }, bytes, allocator); unsafe { ((*UPCALLS).on_move_event)(from, to_obj, bytes) }; - super::object_archive::global_object_archive().copy(src, dst); to_obj } diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 014b4e8..039532c 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -90,24 +90,10 @@ AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, Alloc TPHData* tph_data_ = get_tph_data(this); bool large_object = size_in_bytes > kMaxRegularHeapObjectSize; size_t align_bytes = (type == AllocationType::kCode) ? kCodeAlignment : (align == kWordAligned) ? kSystemPointerSize : (align == kDoubleAligned) ? kDoubleSize : kSystemPointerSize; - // Get MMTk space that the object should be allocated to. - int space = (type == AllocationType::kCode) ? 3 : (type == AllocationType::kReadOnly) ? 4 : (large_object || type == AllocationType::kMap) ? 2 : 0; + auto mmtk_allocator = mmtk::GetAllocationSemanticForV8AllocationType(type, large_object); Address result = - reinterpret_cast
(alloc(tph_mutator_, size_in_bytes, align_bytes, 0, space)); - // Remember the V8 internal `AllocationSpace` for this object. - // This is required to pass various V8 internal space checks. - // TODO(wenyuzhao): Use MMTk's vm-specific spaces for allocation instead of remembering the `AllocationSpace`s. - AllocationSpace allocation_space; - if (type == AllocationType::kCode) { - allocation_space = large_object ? CODE_LO_SPACE : CODE_SPACE; - } else if (type == AllocationType::kReadOnly) { - allocation_space = RO_SPACE; - } else if (type == AllocationType::kMap) { - allocation_space = MAP_SPACE; - } else { - allocation_space = large_object ? LO_SPACE : OLD_SPACE; - } - tph_archive_insert(tph_data_->archive(), reinterpret_cast(result), tph_data_->isolate(), uint8_t(allocation_space)); + reinterpret_cast
(alloc(tph_mutator_, size_in_bytes, align_bytes, 0, (int) mmtk_allocator)); + tph_archive_insert(tph_data_->archive(), reinterpret_cast(result), tph_data_->isolate()); HeapObject rtn = HeapObject::FromAddress(result); return rtn; } @@ -131,22 +117,10 @@ bool Heap::CollectGarbage() { return true; } -// Uninitialized space tag -constexpr AllocationSpace kNoSpace = AllocationSpace(255); - -// Checks whether the address is *logically* in the allocation_space. -// This does not related the real MMTk space that contains the address, -// but the V8 internal space expected by the runtime. -// -// TODO: Currently we record the space tag for each object. In the future we -// need to link each allocation_space to a real MMTk space. -bool Heap::InSpace(Address address, AllocationSpace allocation_space) { - for (auto tph_data : *tph_data_list) { - auto space = AllocationSpace(tph_archive_obj_to_space(tph_data->archive(), reinterpret_cast(address))); - if (space == kNoSpace) continue; - return space == allocation_space; - } - UNREACHABLE(); +bool Heap::InSpace(Address address, AllocationSpace v8_space) { + auto mmtk_space = mmtk::GetAllocationSemanticForV8Space(v8_space); + auto mmtk = get_tph_data(v8_heap->tp_heap_.get())->mmtk_heap(); + return mmtk_in_space(mmtk, (void*) address, (size_t) mmtk_space) != 0; } bool Heap::IsImmovable(HeapObject object) { diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index f87feaa..fe6f98e 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -22,6 +22,44 @@ typedef void* MMTk_TraceLocal; typedef void* MMTk_Heap; typedef void* MMTk_Heap_Archive; +namespace mmtk { + namespace i = v8::internal; + namespace base = v8::base; + namespace tph = v8::internal::third_party_heap; + + enum class MMTkAllocationSemantic: uint8_t { + kDefault = 0, + kImmortal = 1, + kLos = 2, + kCode = 3, + kReadOnly = 4, + kLargeCode = 5, + }; + + V8_INLINE MMTkAllocationSemantic GetAllocationSemanticForV8Space(i::AllocationSpace space) { + switch (space) { + case i::RO_SPACE: return mmtk::MMTkAllocationSemantic::kReadOnly; + case i::OLD_SPACE: return mmtk::MMTkAllocationSemantic::kDefault; + case i::CODE_SPACE: return mmtk::MMTkAllocationSemantic::kCode; + case i::MAP_SPACE: return mmtk::MMTkAllocationSemantic::kImmortal; + case i::LO_SPACE: return mmtk::MMTkAllocationSemantic::kLos; + case i::CODE_LO_SPACE: return mmtk::MMTkAllocationSemantic::kLargeCode; + default: UNREACHABLE(); + } + } + + V8_INLINE MMTkAllocationSemantic GetAllocationSemanticForV8AllocationType(i::AllocationType type, bool large) { + if (type == i::AllocationType::kCode) { + return large ? MMTkAllocationSemantic::kLargeCode : MMTkAllocationSemantic::kCode; + } else if (type == i::AllocationType::kReadOnly) { + return MMTkAllocationSemantic::kReadOnly; + } else if (type == i::AllocationType::kMap) { + return MMTkAllocationSemantic::kImmortal; + } else { + return large ? MMTkAllocationSemantic::kLos : MMTkAllocationSemantic::kDefault; + } + } +} namespace v8 { namespace internal { @@ -101,12 +139,11 @@ extern void start_worker(void *tls, void* worker); extern MMTk_Heap v8_new_heap(void* calls, size_t heap_size); extern void* tph_archive_new(); extern void tph_archive_delete(void* arch); -extern void tph_archive_insert(void* arch, void* object, void* isolate, uint8_t space); +extern void tph_archive_insert(void* arch, void* object, void* isolate); extern void tph_archive_remove(void* arch, void* object); extern void tph_archive_iter_reset(void* arch); extern void* tph_archive_iter_next(void* arch); extern void* tph_archive_inner_to_obj(void* arch, void* inner_ptr); -extern uint8_t tph_archive_obj_to_space(void* arch, void* obj_ptr); extern int mmtk_in_space(void* mmtk, void* object, size_t space); extern void release_buffer(void** buffer, size_t len, size_t cap); @@ -168,10 +205,6 @@ extern void* mmtk_get_forwarded_object(v8::internal::Object o); // Helpers namespace mmtk { - namespace i = v8::internal; - namespace base = v8::base; - namespace tph = v8::internal::third_party_heap; - V8_INLINE bool is_live(i::HeapObject o) { return mmtk_object_is_live(reinterpret_cast(o.address())) != 0; } From c4b5554f5472855f9bb669989cea0e0de288cfcc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 29 Jun 2021 15:16:24 +1000 Subject: [PATCH 45/65] Fix build error --- mmtk/Cargo.toml | 2 +- mmtk/src/lib.rs | 1 + mmtk/src/object_model.rs | 44 ++++++++++++++++++------ v8/third_party/heap/mmtk/mmtk-visitors.h | 1 - 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1c01117..0cdb08a 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -32,5 +32,5 @@ mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a8499062f9df749a6 # mmtk = { path = "../repos/mmtk-core" } [features] -default = ["mmtk/code_space", "mmtk/ro_space", "mmtk/side_gc_header"] +default = ["mmtk/code_space", "mmtk/ro_space"] nogc = [] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 24d054e..614fb37 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -1,5 +1,6 @@ #![feature(vec_into_raw_parts)] #![feature(thread_local)] +#![feature(const_option)] extern crate libc; extern crate mmtk; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 1f135a3..6f0c5b9 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,6 +1,8 @@ use std::sync::atomic::Ordering; use super::UPCALLS; +use mmtk::util::constants::{LOG_BITS_IN_WORD, LOG_BYTES_IN_PAGE, LOG_BYTES_IN_WORD}; +use mmtk::util::metadata::side_metadata::{LOCAL_SIDE_METADATA_VM_BASE_ADDRESS, SideMetadataSpec}; use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec, MetadataSpec}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::*; @@ -17,28 +19,48 @@ const DUMMY_METADATA: MetadataSpec = MetadataSpec::InHeader(HeaderMetadataSpec { impl ObjectModel for VMObjectModel { const GLOBAL_LOG_BIT_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_FORWARDING_POINTER_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_FORWARDING_BITS_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_MARK_BIT_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_LOS_MARK_NURSERY_SPEC: MetadataSpec = DUMMY_METADATA; + const LOCAL_FORWARDING_POINTER_SPEC: MetadataSpec = MetadataSpec::InHeader(HeaderMetadataSpec { + bit_offset: 0, + num_of_bits: LOG_BITS_IN_WORD, + }); + const LOCAL_FORWARDING_BITS_SPEC: MetadataSpec = MetadataSpec::OnSide(SideMetadataSpec { + is_global: false, + offset: LOCAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(), + log_num_of_bits: 1, + log_min_obj_size: LOG_BYTES_IN_WORD as usize, + }); + const LOCAL_MARK_BIT_SPEC: MetadataSpec = MetadataSpec::OnSide(SideMetadataSpec { + is_global: false, + offset: Self::LOCAL_FORWARDING_BITS_SPEC.as_side().unwrap().accumulated_size(), + log_num_of_bits: 0, + log_min_obj_size: LOG_BYTES_IN_WORD as usize, + }); + const LOCAL_LOS_MARK_NURSERY_SPEC: MetadataSpec = MetadataSpec::OnSide(SideMetadataSpec { + is_global: false, + offset: Self::LOCAL_MARK_BIT_SPEC.as_side().unwrap().accumulated_size(), + log_num_of_bits: 1, + log_min_obj_size: LOG_BYTES_IN_PAGE as usize, + }); fn load_metadata( - _metadata_spec: HeaderMetadataSpec, - _object: ObjectReference, + metadata_spec: HeaderMetadataSpec, + object: ObjectReference, _mask: Option, _atomic_ordering: Option, ) -> usize { - unimplemented!() + debug_assert_eq!(metadata_spec, Self::LOCAL_FORWARDING_POINTER_SPEC.as_header().unwrap()); + unsafe { object.to_address().load() } } fn store_metadata( - _metadata_spec: HeaderMetadataSpec, - _object: ObjectReference, - _val: usize, + metadata_spec: HeaderMetadataSpec, + object: ObjectReference, + val: usize, _mask: Option, _atomic_ordering: Option, ) { - unimplemented!() + debug_assert_eq!(metadata_spec, Self::LOCAL_FORWARDING_POINTER_SPEC.as_header().unwrap()); + unsafe { object.to_address().store(val) } } fn compare_exchange_metadata( diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index cf77442..388eb30 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -158,7 +158,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { auto value_obj = table.ValueAt(i); if (value_obj.IsHeapObject()) { auto value = i::HeapObject::cast(value_obj); - printf("discover ephemeron key=%p value=%p\n", (void*) key.ptr(), (void*) value.ptr()); // concrete_visitor()->SynchronizePageAccess(value); // concrete_visitor()->RecordSlot(table, value_slot, value); From d9d668ed663f6e53a3ced91329cfb2cf0e24a751 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 29 Jun 2021 22:49:14 +1000 Subject: [PATCH 46/65] Update to new TPH interface --- v8/third_party/heap/mmtk/mmtk.cc | 2 +- v8/third_party/heap/mmtk/mmtk.h | 41 ++++++++++++++++++++++++++++ v8/third_party/heap/mmtk/weak-refs.h | 17 ++++++------ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 039532c..aaf1afb 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -110,7 +110,7 @@ const v8::base::AddressRegion& Heap::GetCodeRange() { } bool Heap::CollectGarbage() { - v8_heap->increase_gc_count(); + v8_heap->gc_count_++; v8_heap->SetGCState(v8::internal::Heap::MARK_COMPACT); handle_user_collection_request(get_tph_data(this)->mmtk_heap(), (void*) 0); v8_heap->SetGCState(v8::internal::Heap::NOT_IN_GC); diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index fe6f98e..2c1a064 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -224,4 +224,45 @@ namespace mmtk { } } + +namespace v8 { +namespace internal { +namespace third_party_heap { + +class Impl { + public: + V8_INLINE static void ProcessAllWeakReferences(v8::internal::Heap* heap, WeakObjectRetainer* retainer) { + heap->ProcessAllWeakReferences(retainer); + } + + V8_INLINE static void UpdateExternalStringTable(v8::internal::Heap* heap, RootVisitor* external_visitor) { + heap->external_string_table_.IterateAll(external_visitor); + heap->external_string_table_.CleanUpAll(); + } + + V8_INLINE static void EphemeronHashTable_RemoveEntry(EphemeronHashTable& table, InternalIndex entry) { + table.RemoveEntry(entry); + } + + V8_INLINE static void TransitionArray_SetNumberOfTransitions(TransitionArray& array, int number_of_transitions) { + array.SetNumberOfTransitions(number_of_transitions); + } + + V8_INLINE static int TransitionArray_Capacity(TransitionArray& array) { + return array.Capacity(); + } + + V8_INLINE static Map TransitionsAccessor_GetTargetFromRaw(MaybeObject raw) { + return TransitionsAccessor::GetTargetFromRaw(raw); + } + + V8_INLINE static bool TransitionsAccessor_HasSimpleTransitionTo(Isolate* isolate, Map parent, Map target, DisallowGarbageCollection* no_gc) { + return TransitionsAccessor(isolate, parent, no_gc).HasSimpleTransitionTo(target); + } +}; + +} +} +} + #endif // MMTK_H \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 10b17eb..29a2e80 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -255,10 +255,10 @@ class WeakRefs { DCHECK(isolate()->has_active_deserializer()); DCHECK_EQ(raw_target.ToSmi(), i::Deserializer::uninitialized_field_value()); return false; - } else if (!is_live(i::TransitionsAccessor::GetTargetFromRaw(raw_target))) { + } else if (!is_live(tph::Impl::TransitionsAccessor_GetTargetFromRaw(raw_target))) { return true; } else { - DCHECK(!get_forwarded_ref(i::TransitionsAccessor::GetTargetFromRaw(raw_target))); + DCHECK(!get_forwarded_ref(tph::Impl::TransitionsAccessor_GetTargetFromRaw(raw_target))); } } return false; @@ -307,10 +307,10 @@ class WeakRefs { // such that number_of_transitions() == 0. If this assumption changes, // TransitionArray::Insert() will need to deal with the case that a transition // array disappeared during GC. - int trim = transitions.Capacity() - transition_index; + int trim = tph::Impl::TransitionArray_Capacity(transitions) - transition_index; if (trim > 0) { heap()->RightTrimWeakFixedArray(transitions, trim * i::TransitionArray::kEntrySize); - transitions.SetNumberOfTransitions(transition_index); + tph::Impl::TransitionArray_SetNumberOfTransitions(transitions, transition_index); } return descriptors_owner_died; } @@ -374,7 +374,7 @@ class WeakRefs { for (i::InternalIndex i : table.IterateEntries()) { auto key = i::HeapObject::cast(table.KeyAt(i)); if (!is_live(key)) { - table.RemoveEntry(i); + tph::Impl::EphemeronHashTable_RemoveEntry(table, i); } else { if (auto f = get_forwarded_ref(key)) { auto key_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); @@ -458,7 +458,8 @@ class WeakRefs { if (is_live(parent)) { DCHECK(!get_forwarded_ref(parent)); } - if (is_live(parent) && i::TransitionsAccessor(isolate(), parent, &no_gc_obviously).HasSimpleTransitionTo(dead_target)) { + if (is_live(parent) && + tph::Impl::TransitionsAccessor_HasSimpleTransitionTo(isolate(), parent, dead_target, &no_gc_obviously)) { ClearPotentialSimpleMapTransition(parent, dead_target); } } @@ -683,13 +684,13 @@ class WeakRefs { string_table->NotifyElementsRemoved(internalized_visitor.PointersRemoved()); ExternalStringTableCleaner external_visitor(heap()); - heap()->UpdateExternalStringTable(&external_visitor); + tph::Impl::UpdateExternalStringTable(heap(), &external_visitor); } ClearOldBytecodeCandidates(); ClearFlushedJsFunctions(); { MMTkWeakObjectRetainer retainer; - heap()->ProcessAllWeakReferences(&retainer); + tph::Impl::ProcessAllWeakReferences(heap(), &retainer); } ClearFullMapTransitions(); ClearWeakReferences(); From ab95848ae52f266abe7a04a82f0b55bb248ae438 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 29 Jun 2021 23:30:00 +1000 Subject: [PATCH 47/65] Cleanup & remove TPHData --- v8/third_party/heap/mmtk/mmtk.cc | 62 +++----- v8/third_party/heap/mmtk/mmtk.h | 193 ++++++++++++------------ v8/third_party/heap/mmtk/mmtkUpcalls.cc | 10 +- 3 files changed, 118 insertions(+), 147 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index aaf1afb..4d6d05b 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -6,49 +6,25 @@ #include "src/heap/heap-inl.h" #include "log.h" +namespace mmtk { +thread_local MMTk_Mutator mutator = nullptr; +} + namespace v8 { namespace internal { namespace third_party_heap { - - -// Data structure required for Rust-MMTK - v8::internal::Heap* v8_heap = nullptr; - base::AddressRegion code_range_; -thread_local BumpAllocator* tph_mutator_ = nullptr; - -std::vector* tph_data_list = new std::vector(); - extern V8_Upcalls mmtk_upcalls; -TPHData* get_tph_data(Heap* tph) { - for (size_t i = 0; i < tph_data_list->size(); i++) - { - TPHData* tph_data_ = reinterpret_cast((*tph_data_list)[i]); - if (tph_data_->v8_tph() == tph) { - return tph_data_; - } - } - UNREACHABLE(); -} - -std::vector* all_mutators = new std::vector(); - -thread_local bool is_mutator = false; - inline void CheckMutator(Heap* tph) { - TPHData* tph_data_ = get_tph_data(tph); - if (tph_mutator_ == nullptr) { - tph_mutator_ = reinterpret_cast( - bind_mutator(tph_data_->mmtk_heap(), &tph_mutator_)); - tph_mutator_->tph_data = tph_data_; - all_mutators->push_back(tph_mutator_); + if (mmtk::mutator == nullptr) { + mmtk::mutator = bind_mutator(mmtk::get_mmtk_instance(tph), &mmtk::mutator); + tph->impl()->mutators_.push_back(mmtk::mutator); } - is_mutator = true; } static std::atomic_bool IsolateCreated { false }; @@ -67,12 +43,11 @@ std::unique_ptr Heap::New(v8::internal::Isolate* isolate) { auto isolate_created = IsolateCreated.exchange(true); DCHECK_WITH_MSG(!isolate_created, "Multiple isolates are not supported."); MMTK_LOG("New Isolate: %lx\n", (unsigned long) isolate); - MMTk_Heap new_heap = v8_new_heap(&mmtk_upcalls, FIXED_HEAP_SIZE); + MMTk_Heap mmtk_heap = v8_new_heap(&mmtk_upcalls, FIXED_HEAP_SIZE); // FIXME code_range_ = base::AddressRegion(0x60000000, (0xb0000000- 0x60000000)); // isolate->AddCodeRange(code_range_.begin(), code_range_.size()); auto v8_tph = std::make_unique(); - TPHData* tph_data = new TPHData(v8_tph.get(), new_heap, isolate, tph_archive_new()); - tph_data_list->push_back(tph_data); + v8_tph->impl_ = new Impl(mmtk_heap, isolate, tph_archive_new()); return v8_tph; } @@ -87,21 +62,19 @@ AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, Alloc if (!initialization_finished_ && type == AllocationType::kOld) { type = AllocationType::kMap; } - TPHData* tph_data_ = get_tph_data(this); bool large_object = size_in_bytes > kMaxRegularHeapObjectSize; size_t align_bytes = (type == AllocationType::kCode) ? kCodeAlignment : (align == kWordAligned) ? kSystemPointerSize : (align == kDoubleAligned) ? kDoubleSize : kSystemPointerSize; auto mmtk_allocator = mmtk::GetAllocationSemanticForV8AllocationType(type, large_object); Address result = - reinterpret_cast
(alloc(tph_mutator_, size_in_bytes, align_bytes, 0, (int) mmtk_allocator)); - tph_archive_insert(tph_data_->archive(), reinterpret_cast(result), tph_data_->isolate()); + reinterpret_cast
(alloc(mmtk::mutator, size_in_bytes, align_bytes, 0, (int) mmtk_allocator)); + tph_archive_insert(mmtk::get_object_archive(this), reinterpret_cast(result),mmtk::get_isolate(this)); HeapObject rtn = HeapObject::FromAddress(result); return rtn; } Address Heap::GetObjectFromInnerPointer(Address inner_pointer) { - TPHData* tph_data_ = get_tph_data(this); return reinterpret_cast
( - tph_archive_inner_to_obj(tph_data_->archive(), + tph_archive_inner_to_obj(mmtk::get_object_archive(this), reinterpret_cast(inner_pointer))); } @@ -112,14 +85,15 @@ const v8::base::AddressRegion& Heap::GetCodeRange() { bool Heap::CollectGarbage() { v8_heap->gc_count_++; v8_heap->SetGCState(v8::internal::Heap::MARK_COMPACT); - handle_user_collection_request(get_tph_data(this)->mmtk_heap(), (void*) 0); + handle_user_collection_request(mmtk::get_mmtk_instance(this), (void*) 0); v8_heap->SetGCState(v8::internal::Heap::NOT_IN_GC); return true; } bool Heap::InSpace(Address address, AllocationSpace v8_space) { auto mmtk_space = mmtk::GetAllocationSemanticForV8Space(v8_space); - auto mmtk = get_tph_data(v8_heap->tp_heap_.get())->mmtk_heap(); + // TODO(wenyuzhao): Infer isolate from address. May involves consulting the SFT. + auto mmtk = mmtk::get_mmtk_instance(v8_heap); return mmtk_in_space(mmtk, (void*) address, (size_t) mmtk_space) != 0; } @@ -149,13 +123,11 @@ bool Heap::IsValidHeapObject(HeapObject object) { } void Heap::ResetIterator() { - TPHData* tph_data_ = get_tph_data(this); - tph_archive_iter_reset(tph_data_->archive()); + tph_archive_iter_reset(mmtk::get_object_archive(this)); } HeapObject Heap::NextObject() { - TPHData* tph_data_ = get_tph_data(this); - void* obj_addr = tph_archive_iter_next(tph_data_->archive()); + void* obj_addr = tph_archive_iter_next(mmtk::get_object_archive(this)); if (obj_addr != nullptr) { return HeapObject::FromAddress(reinterpret_cast
(obj_addr)); } else { diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 2c1a064..f7beadd 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -22,81 +22,6 @@ typedef void* MMTk_TraceLocal; typedef void* MMTk_Heap; typedef void* MMTk_Heap_Archive; -namespace mmtk { - namespace i = v8::internal; - namespace base = v8::base; - namespace tph = v8::internal::third_party_heap; - - enum class MMTkAllocationSemantic: uint8_t { - kDefault = 0, - kImmortal = 1, - kLos = 2, - kCode = 3, - kReadOnly = 4, - kLargeCode = 5, - }; - - V8_INLINE MMTkAllocationSemantic GetAllocationSemanticForV8Space(i::AllocationSpace space) { - switch (space) { - case i::RO_SPACE: return mmtk::MMTkAllocationSemantic::kReadOnly; - case i::OLD_SPACE: return mmtk::MMTkAllocationSemantic::kDefault; - case i::CODE_SPACE: return mmtk::MMTkAllocationSemantic::kCode; - case i::MAP_SPACE: return mmtk::MMTkAllocationSemantic::kImmortal; - case i::LO_SPACE: return mmtk::MMTkAllocationSemantic::kLos; - case i::CODE_LO_SPACE: return mmtk::MMTkAllocationSemantic::kLargeCode; - default: UNREACHABLE(); - } - } - - V8_INLINE MMTkAllocationSemantic GetAllocationSemanticForV8AllocationType(i::AllocationType type, bool large) { - if (type == i::AllocationType::kCode) { - return large ? MMTkAllocationSemantic::kLargeCode : MMTkAllocationSemantic::kCode; - } else if (type == i::AllocationType::kReadOnly) { - return MMTkAllocationSemantic::kReadOnly; - } else if (type == i::AllocationType::kMap) { - return MMTkAllocationSemantic::kImmortal; - } else { - return large ? MMTkAllocationSemantic::kLos : MMTkAllocationSemantic::kDefault; - } - } -} - -namespace v8 { -namespace internal { - -class Isolate; - -namespace third_party_heap { - -class Heap; - -class TPHData { - Heap* v8_tph_; - MMTk_Heap mmtk_heap_; - v8::internal::Isolate* isolate_; - MMTk_Heap_Archive tph_archive_; - - public: - Heap* v8_tph() { return v8_tph_; } - MMTk_Heap mmtk_heap() { return mmtk_heap_; } - v8::internal::Isolate * isolate() { return isolate_; } - MMTk_Heap_Archive archive() { return tph_archive_; } - - TPHData(Heap* v8_tph, MMTk_Heap mmtk_heap, Isolate* isolate, MMTk_Heap_Archive tph_archive): - v8_tph_(v8_tph), mmtk_heap_(mmtk_heap), isolate_(isolate), tph_archive_(tph_archive) {} -}; - -class BumpAllocator { - public: - TPHData* tph_data; - uintptr_t cursor; - uintptr_t limit; - void* space; -}; - -} // third_party_heap -} // internal -} // v8 /** * Allocation */ @@ -204,27 +129,6 @@ extern void* mmtk_get_forwarded_object(v8::internal::Object o); // Helpers -namespace mmtk { - V8_INLINE bool is_live(i::HeapObject o) { - return mmtk_object_is_live(reinterpret_cast(o.address())) != 0; - } - - V8_INLINE i::MaybeObject to_weakref(i::HeapObject o) { - DCHECK(o.IsStrong()); - return i::MaybeObject::MakeWeak(i::MaybeObject::FromObject(o)); - } - - V8_INLINE base::Optional get_forwarded_ref(i::HeapObject o) { - auto f = mmtk_get_forwarded_object(o); - if (f != nullptr) { - auto x = i::HeapObject::cast(i::Object((i::Address) f)); - return x; - } - return base::nullopt; - } -} - - namespace v8 { namespace internal { namespace third_party_heap { @@ -259,10 +163,107 @@ class Impl { V8_INLINE static bool TransitionsAccessor_HasSimpleTransitionTo(Isolate* isolate, Map parent, Map target, DisallowGarbageCollection* no_gc) { return TransitionsAccessor(isolate, parent, no_gc).HasSimpleTransitionTo(target); } + + V8_INLINE static Heap* get_tp_heap(v8::internal::Heap* heap) { + return heap->tp_heap_.get(); + } + + V8_INLINE Impl(MMTk_Heap mmtk_heap, Isolate* isolate, MMTk_Heap_Archive tph_archive) + : mmtk_heap_(mmtk_heap), isolate_(isolate), tph_archive_(tph_archive) {} + + MMTk_Heap mmtk_heap_; + v8::internal::Isolate* isolate_; + MMTk_Heap_Archive tph_archive_; + std::vector mutators_ {}; +}; + +// TODO(wenyuzhao): We only support one heap at the moment. +extern v8::internal::Heap* v8_heap; + +} +} +} + +namespace mmtk { + +namespace i = v8::internal; +namespace base = v8::base; +namespace tph = v8::internal::third_party_heap; + +// TODO(wenyuzhao): Using of thread_local is incorrect for multiple isolates. +extern thread_local MMTk_Mutator mutator; + +enum class MMTkAllocationSemantic: uint8_t { + kDefault = 0, + kImmortal = 1, + kLos = 2, + kCode = 3, + kReadOnly = 4, + kLargeCode = 5, }; +V8_INLINE MMTkAllocationSemantic GetAllocationSemanticForV8Space(i::AllocationSpace space) { + switch (space) { + case i::RO_SPACE: return mmtk::MMTkAllocationSemantic::kReadOnly; + case i::OLD_SPACE: return mmtk::MMTkAllocationSemantic::kDefault; + case i::CODE_SPACE: return mmtk::MMTkAllocationSemantic::kCode; + case i::MAP_SPACE: return mmtk::MMTkAllocationSemantic::kImmortal; + case i::LO_SPACE: return mmtk::MMTkAllocationSemantic::kLos; + case i::CODE_LO_SPACE: return mmtk::MMTkAllocationSemantic::kLargeCode; + default: UNREACHABLE(); + } +} + +V8_INLINE MMTkAllocationSemantic GetAllocationSemanticForV8AllocationType(i::AllocationType type, bool large) { + if (type == i::AllocationType::kCode) { + return large ? MMTkAllocationSemantic::kLargeCode : MMTkAllocationSemantic::kCode; + } else if (type == i::AllocationType::kReadOnly) { + return MMTkAllocationSemantic::kReadOnly; + } else if (type == i::AllocationType::kMap) { + return MMTkAllocationSemantic::kImmortal; + } else { + return large ? MMTkAllocationSemantic::kLos : MMTkAllocationSemantic::kDefault; + } +} + +V8_INLINE bool is_live(i::HeapObject o) { + return mmtk_object_is_live(reinterpret_cast(o.address())) != 0; +} + +V8_INLINE i::MaybeObject to_weakref(i::HeapObject o) { + DCHECK(o.IsStrong()); + return i::MaybeObject::MakeWeak(i::MaybeObject::FromObject(o)); +} + +V8_INLINE base::Optional get_forwarded_ref(i::HeapObject o) { + auto f = mmtk_get_forwarded_object(o); + if (f != nullptr) { + auto x = i::HeapObject::cast(i::Object((i::Address) f)); + return x; + } + return base::nullopt; +} + +V8_INLINE std::vector& get_mmtk_mutators(i::Heap* heap) { + return tph::Impl::get_tp_heap(heap)->impl()->mutators_; } + +V8_INLINE MMTk_Heap get_mmtk_instance(i::Heap* heap) { + return tph::Impl::get_tp_heap(heap)->impl()->mmtk_heap_; } + +V8_INLINE MMTk_Heap get_mmtk_instance(tph::Heap* tp_heap) { + return tp_heap->impl()->mmtk_heap_; +} + +V8_INLINE MMTk_Heap get_object_archive(tph::Heap* tp_heap) { + return tp_heap->impl()->tph_archive_; +} + +V8_INLINE i::Isolate* get_isolate(tph::Heap* tp_heap) { + return tp_heap->impl()->isolate_; +} + } #endif // MMTK_H \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 7a28f18..c5e948c 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -85,18 +85,16 @@ static void* mmtk_get_mmtk_mutator(void* tls) { UNIMPLEMENTED(); } -extern thread_local bool is_mutator; - static bool mmtk_is_mutator(void* tls) { - return is_mutator; + return mmtk::mutator != nullptr; } -extern std::vector* all_mutators; size_t index = 0; static void* mmtk_get_next_mutator() { - if (index >= all_mutators->size()) return nullptr; - return (*all_mutators)[index++]; + auto& mutators = mmtk::get_mmtk_mutators(v8_heap); + if (index >= mutators.size()) return nullptr; + return mutators[index++]; } static void mmtk_reset_mutator_iterator() { From b4dac238a8a569e31b8e6dee461134f7fec585d9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 30 Jun 2021 10:47:38 +1000 Subject: [PATCH 48/65] VisitMap --- v8/third_party/heap/mmtk/mmtk-visitors.h | 89 +++++++++++++++++++----- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 388eb30..0311b47 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -172,25 +172,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // return table.SizeFromMap(map); } - V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { - // if (!ShouldVisit(array)) return; - VisitMapPointer(array); - // int size = i::DescriptorArray::BodyDescriptor::SizeOf(map, array); - VisitPointers(array, array.GetFirstPointerSlot(), array.GetDescriptorSlot(0)); - VisitDescriptors(array, array.number_of_descriptors()); - } - - void VisitDescriptors(i::DescriptorArray descriptor_array, int number_of_own_descriptors) { - int16_t new_marked = static_cast(number_of_own_descriptors); - // int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors( - // heap_->gc_count(), new_marked); - // if (old_marked < new_marked) { - VisitPointers(descriptor_array, - i::MaybeObjectSlot(descriptor_array.GetDescriptorSlot(0)), - i::MaybeObjectSlot(descriptor_array.GetDescriptorSlot(new_marked))); - // } - } - V8_INLINE void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { // if (!ShouldVisit(weak_cell)) return; @@ -318,6 +299,76 @@ class MMTkEdgeVisitor: public i::HeapVisitor { #endif + V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { + VisitMapPointer(array); + VisitPointers(array, array.GetFirstPointerSlot(), array.GetDescriptorSlot(0)); + VisitDescriptors(array, array.number_of_descriptors()); + } + + void VisitDescriptors(i::DescriptorArray descriptor_array, int number_of_own_descriptors) { + int16_t new_marked = static_cast(number_of_own_descriptors); + // int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors( + // heap_->gc_count(), new_marked); + // if (old_marked < new_marked) { + VisitPointers(descriptor_array, + i::MaybeObjectSlot(descriptor_array.GetDescriptorSlot(0)), + i::MaybeObjectSlot(descriptor_array.GetDescriptorSlot(new_marked))); + // } + } + + V8_INLINE void VisitMap(i::Map meta_map, i::Map map) { + int size = i::Map::BodyDescriptor::SizeOf(meta_map, map); + size += VisitDescriptorsForMap(map); + // Mark the pointer fields of the Map. If there is a transitions array, it has + // been marked already, so it is fine that one of these fields contains a + // pointer to it. + i::Map::BodyDescriptor::IterateBody(meta_map, map, size, this); + } + + V8_INLINE int VisitDescriptorsForMap(i::Map map) { + if (!map.CanTransition()) return 0; + // Maps that can transition share their descriptor arrays and require + // special visiting logic to avoid memory leaks. + // Since descriptor arrays are potentially shared, ensure that only the + // descriptors that belong to this map are marked. The first time a + // non-empty descriptor array is marked, its header is also visited. The + // slot holding the descriptor array will be implicitly recorded when the + // pointer fields of this map are visited. + i::Object maybe_descriptors = + i::TaggedField::Acquire_Load( + heap_->isolate(), map); + // If the descriptors are a Smi, then this Map is in the process of being + // deserialized, and doesn't yet have an initialized descriptor field. + if (maybe_descriptors.IsSmi()) { + DCHECK_EQ(maybe_descriptors, i::Deserializer::uninitialized_field_value()); + return 0; + } + auto descriptors = i::DescriptorArray::cast(maybe_descriptors); + // Don't do any special processing of strong descriptor arrays, let them get + // marked through the normal visitor mechanism. + if (descriptors.IsStrongDescriptorArray()) { + return 0; + } + // Mark weak DescriptorArray + auto old_descriptors = descriptors; + trace_field_((void*) &descriptors, context_); + if (old_descriptors != descriptors) { + i::TaggedField::Release_Store(map, descriptors); + } + auto size = i::DescriptorArray::BodyDescriptor::SizeOf(descriptors.map(), descriptors); + int number_of_own_descriptors = map.NumberOfOwnDescriptors(); + if (number_of_own_descriptors) { + // It is possible that the concurrent marker observes the + // number_of_own_descriptors out of sync with the descriptors. In that + // case the marking write barrier for the descriptor array will ensure + // that all required descriptors are marked. The concurrent marker + // just should avoid crashing in that case. That's why we need the + // std::min() below. + VisitDescriptors(descriptors, std::min(number_of_own_descriptors, descriptors.number_of_descriptors())); + } + return size; + } + virtual void VisitPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { for (auto p = start; p < end; ++p) ProcessEdge(host, p); } From 30dffe99bb8ab756c741d143fd7d383d9e0e724a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 30 Jun 2021 14:22:43 +1000 Subject: [PATCH 49/65] Several fix --- mmtk/src/lib.rs | 6 +-- mmtk/src/scanning.rs | 6 +++ v8/third_party/heap/mmtk/mmtk-visitors.h | 51 ++++++++++-------------- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 6 +-- 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 614fb37..8da85c9 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -80,11 +80,11 @@ impl VMBinding for V8 { lazy_static! { pub static ref SINGLETON: MMTK = { if let Ok(threads) = env::var("MMTK_THREADS").map(|x| x.parse::().unwrap()) { - if threads > 8 { - env::set_var("MMTK_THREADS", "8"); + if threads > 7 { + env::set_var("MMTK_THREADS", "7"); } } else { - env::set_var("MMTK_THREADS", "8"); + env::set_var("MMTK_THREADS", "7"); } MMTK::new() }; diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 0a8fbd5..ea6e71d 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -62,11 +62,13 @@ impl Scanning for VMScanning { unsafe { let w = x as *mut GCWorker; debug_assert!(ROOT_OBJECTS.is_empty()); + ROOT_FLUSHED = false; ((*UPCALLS).process_ephemerons)(trace_root:: as _, w as _, (*w).ordinal); if !ROOT_OBJECTS.is_empty() { flush_roots::(&mut *w); } debug_assert!(ROOT_OBJECTS.is_empty()); + ROOT_FLUSHED } })); mmtk::memory_manager::add_work_packet( @@ -85,6 +87,7 @@ pub struct ProcessEphemerons>(PhantomData); impl> ProcessEphemerons { pub fn new() -> Self { + unreachable!(); Self(PhantomData) } } @@ -124,11 +127,14 @@ impl> GCWork for ScanAndForwardRoots { } pub(crate) static mut ROOT_OBJECTS: Vec = Vec::new(); +pub(crate) static mut ROOT_FLUSHED: bool = false; pub(crate) fn flush_roots>(_worker: &mut GCWorker) { + unsafe { ROOT_FLUSHED = true; } let mut buf = vec![]; unsafe { std::mem::swap(&mut buf, &mut ROOT_OBJECTS); } let scan_objects_work = mmtk::scheduler::gc_work::ScanObjects::::new(buf, false); + println!("Flush Roots"); mmtk::memory_manager::add_work_packet( &SINGLETON, WorkBucketStage::Closure, diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 0311b47..431af7a 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -117,9 +117,14 @@ class MMTkCustomRootBodyVisitor final : public i::ObjectVisitor { class MMTkEdgeVisitor: public i::HeapVisitor { public: explicit MMTkEdgeVisitor(i::Heap* heap, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id) - : heap_(heap), process_edges_(process_edges), trace_field_(trace_field), context_(context), task_id_(task_id) { + : heap_(heap), process_edges_(process_edges), task_id_(task_id) { + trace_field_ = [=](i::HeapObject o) -> base::Optional { + auto old = o; + trace_field((void*) &o, context); + return o != old ? base::make_optional(o) : base::nullopt; + }; USE(heap_); - DCHECK(task_id <= 8); + DCHECK(1 <= task_id && task_id <= 7); NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; cap_ = buf.cap; @@ -280,22 +285,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // visitors. They're used for e.g., lists that are recreated after GC. The // default implementation treats them as strong pointers. Visitors who want to // ignore them must override this function with empty. - virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final { - // for (auto p = start; p < end; ++p) { - // printf("@weak? %p.%p -> %p\n", (void*) host.ptr(), (void*) p.address(), (void*) (*p).ptr()); - // i::HeapObject object; - // if (i::ObjectSlot::kCanBeWeak && (*p).GetHeapObjectIfWeak(&object)) { - // // printf("@weak %p.%p -> %p\n", (void*) host.ptr(), (void*) p.address(), (void*) (*p).ptr()); - // // auto s = i::HeapObjectSlot(p.address()); - // // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); - // } else if ((*p).GetHeapObjectIfStrong(&object)) { - - // printf("@string %p.%p -> %p\n", (void*) host.ptr(), (void*) p.address(), (void*) (*p).ptr()); - // auto s = i::HeapObjectSlot(p.address()); - // weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); - // } - // } - } + virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final {} #endif @@ -350,9 +340,8 @@ class MMTkEdgeVisitor: public i::HeapVisitor { return 0; } // Mark weak DescriptorArray - auto old_descriptors = descriptors; - trace_field_((void*) &descriptors, context_); - if (old_descriptors != descriptors) { + if (auto forwarded = trace_field_(descriptors)) { + descriptors = i::DescriptorArray::cast(*forwarded); i::TaggedField::Release_Store(map, descriptors); } auto size = i::DescriptorArray::BodyDescriptor::SizeOf(descriptors.map(), descriptors); @@ -380,21 +369,22 @@ class MMTkEdgeVisitor: public i::HeapVisitor { virtual void VisitCodeTarget(i::Code host, i::RelocInfo* rinfo) override final { auto target = i::Code::GetCodeFromTargetAddress(rinfo->target_address()); DCHECK(!mmtk_is_movable(target)); - trace_field_((void*) &target, context_); - DCHECK_EQ(target, i::Code::GetCodeFromTargetAddress(rinfo->target_address())); + auto forwarded = trace_field_(target); + DCHECK(!forwarded); + USE(forwarded); } virtual void VisitEmbeddedPointer(i::Code host, i::RelocInfo* rinfo) override final { auto o = rinfo->target_object(); - auto f = mmtk::get_forwarded_ref(o); - if (f) { - rinfo->set_target_object(heap_, *f); + if (auto forwarded = mmtk::get_forwarded_ref(o)) { + rinfo->set_target_object(heap_, *forwarded); } else if (host.IsWeakObject(o) && WEAKREF_PROCESSING_BOOL) { DCHECK(i::RelocInfo::IsCodeTarget(rinfo->rmode()) || i::RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); weak_objects_->weak_objects_in_code.Push(task_id_, std::make_pair(o, host)); } else { - trace_field_((void*) &o, context_); - if (o != rinfo->target_object()) rinfo->set_target_object(heap_, o); + if (auto forwarded = trace_field_(o)) { + rinfo->set_target_object(heap_, *forwarded); + } } } @@ -439,13 +429,12 @@ class MMTkEdgeVisitor: public i::HeapVisitor { v8::internal::Heap* heap_; ProcessEdgesFn process_edges_; - TraceFieldFn trace_field_; - void* context_; + int task_id_; void** buffer_ = nullptr; size_t cap_ = 0; size_t cursor_ = 0; i::WeakObjects* weak_objects_ = mmtk::global_weakref_processor->weak_objects(); - int task_id_; + std::function(i::HeapObject)> trace_field_; }; diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index c5e948c..8d985eb 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -136,8 +136,8 @@ static void mmtk_on_move_event(void* from_address, void* to_address, size_t size static void mmtk_scan_roots(TraceRootFn trace_root, void* context, int task_id) { main_thread_synchronizer->RunMainThreadTask([=]() { - mmtk::MMTkRootVisitor root_visitor(v8_heap, trace_root, context, task_id); - mmtk::MMTkCustomRootBodyVisitor custom_root_body_visitor(v8_heap, trace_root, context, task_id); + mmtk::MMTkRootVisitor root_visitor(v8_heap, trace_root, context, kMainThreadTask); + mmtk::MMTkCustomRootBodyVisitor custom_root_body_visitor(v8_heap, trace_root, context, kMainThreadTask); v8_heap->IterateRoots(&root_visitor, {}); for (i::StackFrameIterator it(v8_heap->isolate(), v8_heap->isolate()->thread_local_top()); !it.done(); it.Advance()) { if (it.frame()->is_unoptimized()) break; @@ -154,7 +154,7 @@ static void mmtk_scan_roots(TraceRootFn trace_root, void* context, int task_id) } static void mmtk_scan_objects(void** objects, size_t count, ProcessEdgesFn process_edges, TraceFieldFn trace_field, void* context, int task_id) { - mmtk::MMTkEdgeVisitor visitor(v8_heap, process_edges, trace_field, context, task_id); + mmtk::MMTkEdgeVisitor visitor(v8_heap, process_edges, trace_field, context, task_id + 1); for (size_t i = 0; i < count; i++) { auto ptr = *(objects + i); DCHECK_EQ(((Address) ptr) & 1, 0); From cecff38beffe61c2439c2d45005850603cd85080 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 30 Jun 2021 17:04:25 +1000 Subject: [PATCH 50/65] Update (for gencopy) --- mmtk/src/object_model.rs | 14 +++++++------- mmtk/src/scanning.rs | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 6f0c5b9..675f2fb 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -2,7 +2,7 @@ use std::sync::atomic::Ordering; use super::UPCALLS; use mmtk::util::constants::{LOG_BITS_IN_WORD, LOG_BYTES_IN_PAGE, LOG_BYTES_IN_WORD}; -use mmtk::util::metadata::side_metadata::{LOCAL_SIDE_METADATA_VM_BASE_ADDRESS, SideMetadataSpec}; +use mmtk::util::metadata::side_metadata::{GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS, LOCAL_SIDE_METADATA_VM_BASE_ADDRESS, SideMetadataSpec}; use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec, MetadataSpec}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::*; @@ -12,13 +12,13 @@ use V8; pub struct VMObjectModel {} -const DUMMY_METADATA: MetadataSpec = MetadataSpec::InHeader(HeaderMetadataSpec { - bit_offset: 0, - num_of_bits: 0, -}); - impl ObjectModel for VMObjectModel { - const GLOBAL_LOG_BIT_SPEC: MetadataSpec = DUMMY_METADATA; + const GLOBAL_LOG_BIT_SPEC: MetadataSpec = MetadataSpec::OnSide(SideMetadataSpec { + is_global: true, + offset: GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(), + log_num_of_bits: 0, + log_min_obj_size: LOG_BYTES_IN_WORD as usize, + }); const LOCAL_FORWARDING_POINTER_SPEC: MetadataSpec = MetadataSpec::InHeader(HeaderMetadataSpec { bit_offset: 0, num_of_bits: LOG_BITS_IN_WORD, diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index ea6e71d..33ce03a 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -134,7 +134,6 @@ pub(crate) fn flush_roots>(_worker: &mut GCWorker::new(buf, false); - println!("Flush Roots"); mmtk::memory_manager::add_work_packet( &SINGLETON, WorkBucketStage::Closure, From b4bae757f2a826f7c21dc1703a2a8440f6fb5276 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Jul 2021 16:04:53 +1000 Subject: [PATCH 51/65] Redirect IsPendingAllocation --- v8/third_party/heap/mmtk/mmtk.cc | 6 +++++- v8/third_party/heap/mmtk/mmtk.h | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/v8/third_party/heap/mmtk/mmtk.cc b/v8/third_party/heap/mmtk/mmtk.cc index 4d6d05b..e761e5c 100644 --- a/v8/third_party/heap/mmtk/mmtk.cc +++ b/v8/third_party/heap/mmtk/mmtk.cc @@ -59,7 +59,7 @@ v8::internal::Isolate* Heap::GetIsolate(Address object_pointer) { // Address space in Rust is statically from 0x60000000 - 0xb0000000 AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, AllocationAlignment align) { CheckMutator(this); - if (!initialization_finished_ && type == AllocationType::kOld) { + if (!v8_heap->deserialization_complete() && type == AllocationType::kOld) { type = AllocationType::kMap; } bool large_object = size_in_bytes > kMaxRegularHeapObjectSize; @@ -72,6 +72,10 @@ AllocationResult Heap::Allocate(size_t size_in_bytes, AllocationType type, Alloc return rtn; } +bool Heap::IsPendingAllocation(HeapObject object) { + return false; +} + Address Heap::GetObjectFromInnerPointer(Address inner_pointer) { return reinterpret_cast
( tph_archive_inner_to_obj(mmtk::get_object_archive(this), diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index f7beadd..ad5f087 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -164,6 +164,10 @@ class Impl { return TransitionsAccessor(isolate, parent, no_gc).HasSimpleTransitionTo(target); } + V8_INLINE static void FlushNumberStringCache(v8::internal::Heap* heap) { + heap->FlushNumberStringCache(); + } + V8_INLINE static Heap* get_tp_heap(v8::internal::Heap* heap) { return heap->tp_heap_.get(); } From ba2492a67638ab18e6580dcedd77d06590c6fc8f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Jul 2021 16:05:04 +1000 Subject: [PATCH 52/65] Minor fix --- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 8d985eb..b08ffe7 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -11,6 +11,7 @@ #include "main-thread-sync.h" #include "log.h" #include "weak-refs.h" +#include "src/codegen/compilation-cache.h" namespace v8 { namespace internal { @@ -30,7 +31,8 @@ static void mmtk_stop_all_mutators(void *tls) { v8_heap->isolate()->descriptor_lookup_cache()->Clear(); RegExpResultsCache::Clear(v8_heap->string_split_cache()); RegExpResultsCache::Clear(v8_heap->regexp_multiple_cache()); - // v8_heap->FlushNumberStringCache(); + v8_heap->isolate()->compilation_cache()->MarkCompactPrologue(); + Impl::FlushNumberStringCache(v8_heap); int len = v8_heap->number_string_cache().length(); for (int i = 0; i < len; i++) { v8_heap->number_string_cache().set_undefined(i); From a017f0772aea216aa49cc5ade15a55c0eb63a2c0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Jul 2021 18:10:39 +1000 Subject: [PATCH 53/65] Override V8 weak object visiting code --- v8/third_party/heap/mmtk/mmtk-visitors.h | 2 +- v8/third_party/heap/mmtk/mmtk.h | 15 +- .../heap/mmtk/third-party-heap-impl.cc | 230 ++++++++++++++++++ v8/third_party/heap/mmtk/weak-refs.h | 5 +- 4 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 v8/third_party/heap/mmtk/third-party-heap-impl.cc diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 431af7a..d777e1c 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -330,7 +330,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { // If the descriptors are a Smi, then this Map is in the process of being // deserialized, and doesn't yet have an initialized descriptor field. if (maybe_descriptors.IsSmi()) { - DCHECK_EQ(maybe_descriptors, i::Deserializer::uninitialized_field_value()); + DCHECK_EQ(maybe_descriptors, i::Smi::uninitialized_deserialization_value()); return 0; } auto descriptors = i::DescriptorArray::cast(maybe_descriptors); diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index ad5f087..5799d12 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -133,10 +133,21 @@ namespace v8 { namespace internal { namespace third_party_heap { +namespace i = v8::internal; + class Impl { public: - V8_INLINE static void ProcessAllWeakReferences(v8::internal::Heap* heap, WeakObjectRetainer* retainer) { - heap->ProcessAllWeakReferences(retainer); + template + static v8::internal::Object VisitWeakList(v8::internal::Heap* heap, v8::internal::Object list, v8::internal::WeakObjectRetainer* retainer); + + V8_INLINE static void ProcessAllWeakReferences(v8::internal::Heap* heap, v8::internal::WeakObjectRetainer* retainer) { + heap->set_native_contexts_list(VisitWeakList(heap, heap->native_contexts_list(), retainer)); + heap->set_allocation_sites_list(VisitWeakList(heap, heap->allocation_sites_list(), retainer)); + auto head = VisitWeakList(heap, heap->dirty_js_finalization_registries_list(), retainer); + heap->set_dirty_js_finalization_registries_list(head); + if (head.IsUndefined(heap->isolate())) { + heap->set_dirty_js_finalization_registries_list_tail(head); + } } V8_INLINE static void UpdateExternalStringTable(v8::internal::Heap* heap, RootVisitor* external_visitor) { diff --git a/v8/third_party/heap/mmtk/third-party-heap-impl.cc b/v8/third_party/heap/mmtk/third-party-heap-impl.cc new file mode 100644 index 0000000..dd074ed --- /dev/null +++ b/v8/third_party/heap/mmtk/third-party-heap-impl.cc @@ -0,0 +1,230 @@ +#include "mmtk.h" +#include "src/heap/heap.h" +#include "src/objects/string-table.h" +#include "src/objects/visitors.h" +#include "src/objects/transitions-inl.h" +#include "src/heap/objects-visiting.h" +#include "src/heap/heap-inl.h" +#include "src/objects/js-weak-refs-inl.h" +#include "src/heap/mark-compact-inl.h" + +namespace v8 { +namespace internal { +namespace third_party_heap { + +namespace { + +using namespace v8::internal; +using Heap = v8::internal::Heap; + +static bool MustRecordSlots(Heap* heap) { + return false; +} + +template +struct WeakListVisitor; + +template +Object InternalVisitWeakList(Heap* heap, Object list, WeakObjectRetainer* retainer) { + Object undefined = ReadOnlyRoots(heap).undefined_value(); + Object head = undefined; + T tail; + bool record_slots = MustRecordSlots(heap); + + while (list != undefined) { + // Check whether to keep the candidate in the list. + T candidate = T::cast(list); + + Object retained = retainer->RetainAs(list); + + // Move to the next element before the WeakNext is cleared. + list = WeakListVisitor::WeakNext(retained != Object() ? T::cast(retained) + : candidate); + + if (retained != Object()) { + if (head == undefined) { + // First element in the list. + head = retained; + } else { + // Subsequent elements in the list. + DCHECK(!tail.is_null()); + WeakListVisitor::SetWeakNext(tail, retained); + if (record_slots) { + HeapObject slot_holder = WeakListVisitor::WeakNextHolder(tail); + int slot_offset = WeakListVisitor::WeakNextOffset(); + ObjectSlot slot = slot_holder.RawField(slot_offset); + MarkCompactCollector::RecordSlot(slot_holder, slot, + HeapObject::cast(retained)); + } + } + // Retained object is new tail. + DCHECK(!retained.IsUndefined(heap->isolate())); + candidate = T::cast(retained); + tail = candidate; + + // tail is a live object, visit it. + WeakListVisitor::VisitLiveObject(heap, tail, retainer); + + } else { + WeakListVisitor::VisitPhantomObject(heap, candidate); + } + } + + // Terminate the list if there is one or more elements. + if (!tail.is_null()) WeakListVisitor::SetWeakNext(tail, undefined); + return head; +} + +template +static void ClearWeakList(Heap* heap, Object list) { + Object undefined = ReadOnlyRoots(heap).undefined_value(); + while (list != undefined) { + T candidate = T::cast(list); + list = WeakListVisitor::WeakNext(candidate); + WeakListVisitor::SetWeakNext(candidate, undefined); + } +} + +template <> +struct WeakListVisitor { + static void SetWeakNext(CodeT code, Object next) { + CodeDataContainerFromCodeT(code).set_next_code_link( + next, UPDATE_WEAK_WRITE_BARRIER); + } + + static Object WeakNext(CodeT code) { + return CodeDataContainerFromCodeT(code).next_code_link(); + } + + static HeapObject WeakNextHolder(CodeT code) { + return CodeDataContainerFromCodeT(code); + } + + static int WeakNextOffset() { return CodeDataContainer::kNextCodeLinkOffset; } + + static void VisitLiveObject(Heap*, CodeT, WeakObjectRetainer*) {} + + static void VisitPhantomObject(Heap* heap, CodeT code) { + // Even though the code is dying, its code_data_container can still be + // alive. Clear the next_code_link slot to avoid a dangling pointer. + SetWeakNext(code, ReadOnlyRoots(heap).undefined_value()); + } +}; + +template <> +struct WeakListVisitor { + static void SetWeakNext(Context context, Object next) { + context.set(Context::NEXT_CONTEXT_LINK, next, UPDATE_WEAK_WRITE_BARRIER); + } + + static Object WeakNext(Context context) { + return context.next_context_link(); + } + + static HeapObject WeakNextHolder(Context context) { return context; } + + static int WeakNextOffset() { + return FixedArray::SizeFor(Context::NEXT_CONTEXT_LINK); + } + + static void VisitLiveObject(Heap* heap, Context context, + WeakObjectRetainer* retainer) { + if (heap->gc_state() == Heap::MARK_COMPACT) { + if (!V8_ENABLE_THIRD_PARTY_HEAP_BOOL) { + // Record the slots of the weak entries in the native context. + for (int idx = Context::FIRST_WEAK_SLOT; + idx < Context::NATIVE_CONTEXT_SLOTS; ++idx) { + ObjectSlot slot = context.RawField(Context::OffsetOfElementAt(idx)); + MarkCompactCollector::RecordSlot(context, slot, + HeapObject::cast(*slot)); + } + } + // Code objects are always allocated in Code space, we do not have to + // visit them during scavenges. + DoWeakList(heap, context, retainer, Context::OPTIMIZED_CODE_LIST); + DoWeakList(heap, context, retainer, + Context::DEOPTIMIZED_CODE_LIST); + } + } + + template + static void DoWeakList(i::Heap* heap, Context context, + WeakObjectRetainer* retainer, int index) { + // Visit the weak list, removing dead intermediate elements. + Object list_head = VisitWeakList(heap, context.get(index), retainer); + + // Update the list head. + context.set(index, list_head, UPDATE_WRITE_BARRIER); + + if (MustRecordSlots(heap)) { + // Record the updated slot if necessary. + ObjectSlot head_slot = context.RawField(FixedArray::SizeFor(index)); + heap->mark_compact_collector()->RecordSlot(context, head_slot, + HeapObject::cast(list_head)); + } + } + + static void VisitPhantomObject(Heap* heap, Context context) { + ClearWeakList(heap, context.get(Context::OPTIMIZED_CODE_LIST)); + ClearWeakList(heap, context.get(Context::DEOPTIMIZED_CODE_LIST)); + } +}; + + +template <> +struct WeakListVisitor { + static void SetWeakNext(AllocationSite obj, Object next) { + obj.set_weak_next(next, UPDATE_WEAK_WRITE_BARRIER); + } + + static Object WeakNext(AllocationSite obj) { return obj.weak_next(); } + + static HeapObject WeakNextHolder(AllocationSite obj) { return obj; } + + static int WeakNextOffset() { return AllocationSite::kWeakNextOffset; } + + static void VisitLiveObject(Heap*, AllocationSite, WeakObjectRetainer*) {} + + static void VisitPhantomObject(Heap*, AllocationSite) {} +}; + +template <> +struct WeakListVisitor { + static void SetWeakNext(JSFinalizationRegistry obj, Object next) { + obj.set_next_dirty(next, UPDATE_WEAK_WRITE_BARRIER); + } + + static Object WeakNext(JSFinalizationRegistry obj) { + return obj.next_dirty(); + } + + static HeapObject WeakNextHolder(JSFinalizationRegistry obj) { return obj; } + + static int WeakNextOffset() { + return JSFinalizationRegistry::kNextDirtyOffset; + } + + static void VisitLiveObject(Heap* heap, JSFinalizationRegistry obj, + WeakObjectRetainer*) { + heap->set_dirty_js_finalization_registries_list_tail(obj); + } + + static void VisitPhantomObject(Heap*, JSFinalizationRegistry) {} +}; + +} + +template +i::Object Impl::VisitWeakList(i::Heap* heap, i::Object list, i::WeakObjectRetainer* retainer) { + return InternalVisitWeakList(heap, list, retainer); +} + +template i::Object Impl::VisitWeakList(i::Heap* heap, i::Object list, i::WeakObjectRetainer* retainer); + +template i::Object Impl::VisitWeakList(i::Heap* heap, i::Object list, i::WeakObjectRetainer* retainer); + +template i::Object Impl::VisitWeakList(i::Heap* heap, i::Object list, i::WeakObjectRetainer* retainer); + +} +} +} \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 29a2e80..88a4abe 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -5,6 +5,7 @@ #include "src/objects/string-table.h" #include "src/objects/visitors.h" #include "src/objects/transitions-inl.h" +#include "src/heap/objects-visiting.h" namespace v8 { namespace internal { @@ -229,7 +230,7 @@ class WeakRefs { auto constructor_or_back_pointer = map.constructor_or_back_pointer(); if (constructor_or_back_pointer.IsSmi()) { DCHECK(isolate()->has_active_deserializer()); - DCHECK_EQ(constructor_or_back_pointer, i::Deserializer::uninitialized_field_value()); + DCHECK_EQ(constructor_or_back_pointer, i::Smi::uninitialized_deserialization_value()); continue; } auto parent = i::Map::cast(map.constructor_or_back_pointer()); @@ -253,7 +254,7 @@ class WeakRefs { if (raw_target.IsSmi()) { // This target is still being deserialized, DCHECK(isolate()->has_active_deserializer()); - DCHECK_EQ(raw_target.ToSmi(), i::Deserializer::uninitialized_field_value()); + DCHECK_EQ(raw_target.ToSmi(), i::Smi::uninitialized_deserialization_value()); return false; } else if (!is_live(tph::Impl::TransitionsAccessor_GetTargetFromRaw(raw_target))) { return true; From 94428f85b451759986a0890e778886848b391289 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 6 Jul 2021 16:52:16 +1000 Subject: [PATCH 54/65] no logging --- v8/third_party/heap/mmtk/log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v8/third_party/heap/mmtk/log.h b/v8/third_party/heap/mmtk/log.h index 84a0382..c5896d9 100644 --- a/v8/third_party/heap/mmtk/log.h +++ b/v8/third_party/heap/mmtk/log.h @@ -1,7 +1,7 @@ #ifndef MMTK_LOG_H #define MMTK_LOG_H -#define ENABLE_LOGGING 1 +#define ENABLE_LOGGING false #define MMTK_LOG(...) \ if (ENABLE_LOGGING) { \ From 664111a3a007e8d58b4bec1887ab86470a3672c2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 9 Jul 2021 15:11:09 +1000 Subject: [PATCH 55/65] Fix fold-allocated FixedArrays --- mmtk/src/api.rs | 10 ++++++++++ mmtk/src/object_model.rs | 4 ++-- mmtk/src/scanning.rs | 1 - v8/third_party/heap/mmtk/mmtkUpcalls.cc | 2 -- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index d059cc1..be91030 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -3,6 +3,7 @@ use libc::c_void; use mmtk::memory_manager; use mmtk::scheduler::GCWorker; use mmtk::util::opaque_pointer::*; +use mmtk::util::options::PlanSelector; use mmtk::util::{Address, ObjectReference}; use mmtk::AllocationSemantics; use mmtk::Mutator; @@ -92,6 +93,15 @@ pub extern "C" fn alloc( ) -> Address { let a = memory_manager::alloc::(mutator, size, align, offset, semantics); unsafe { memory_manager::post_alloc::(mutator, a.to_object_reference(), size, semantics); } + if PlanSelector::PageProtect == mutator.plan.options().plan && AllocationSemantics::Default == semantics { + // Possible `array_header_size` values that can be passed to [AllocateUninitializedJSArrayWithElements](https://source.chromium.org/chromium/chromium/src/+/main:v8/src/codegen/code-stub-assembler.h;l=1884). + let array_header_sizes = [0x20, 0x50, 0x58]; + for array_header_size in array_header_sizes { + unsafe { + memory_manager::post_alloc::(mutator, a.add(array_header_size).to_object_reference(), 0, semantics); + } + } + } a } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 675f2fb..2d2d2c8 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,7 +1,7 @@ use std::sync::atomic::Ordering; use super::UPCALLS; -use mmtk::util::constants::{LOG_BITS_IN_WORD, LOG_BYTES_IN_PAGE, LOG_BYTES_IN_WORD}; +use mmtk::util::constants::{LOG_BITS_IN_WORD, LOG_BYTES_IN_WORD}; use mmtk::util::metadata::side_metadata::{GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS, LOCAL_SIDE_METADATA_VM_BASE_ADDRESS, SideMetadataSpec}; use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec, MetadataSpec}; use mmtk::util::{Address, ObjectReference}; @@ -39,7 +39,7 @@ impl ObjectModel for VMObjectModel { is_global: false, offset: Self::LOCAL_MARK_BIT_SPEC.as_side().unwrap().accumulated_size(), log_num_of_bits: 1, - log_min_obj_size: LOG_BYTES_IN_PAGE as usize, + log_min_obj_size: LOG_BYTES_IN_WORD as usize, }); fn load_metadata( diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 33ce03a..e1346b7 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -88,7 +88,6 @@ pub struct ProcessEphemerons>(PhantomData); impl> ProcessEphemerons { pub fn new() -> Self { unreachable!(); - Self(PhantomData) } } diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index b08ffe7..9815bbd 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -25,8 +25,6 @@ static void mmtk_stop_all_mutators(void *tls) { MMTK_LOG("[mmtk_stop_all_mutators] START\n"); main_thread_synchronizer->RunMainThreadTask([=]() { main_thread_synchronizer->EnterSafepoint(v8_heap); - MMTK_LOG("[mmtk_stop_all_mutators] Verify heap\n"); - mmtk::MMTkHeapVerifier::Verify(v8_heap); MMTK_LOG("[mmtk_stop_all_mutators] Flush cache\n"); v8_heap->isolate()->descriptor_lookup_cache()->Clear(); RegExpResultsCache::Clear(v8_heap->string_split_cache()); From 425e92ae5d5b1878f2a12ba93af2a8df536c0ea8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 13 Jul 2021 13:10:49 +1000 Subject: [PATCH 56/65] update --- mmtk/src/collection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index d86f9df..7729687 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -57,7 +57,7 @@ impl Collection for VMCollection { unimplemented!() } - fn update_object_archive() { + fn vm_release() { global_object_archive().update(); } From 62777767c2ec5a23d90027a577493cee6005a7d9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 14 Jul 2021 17:30:22 +1000 Subject: [PATCH 57/65] Fix --- mmtk/src/api.rs | 9 ++++++++- mmtk/src/scanning.rs | 21 +++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index be91030..d7e606f 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -67,7 +67,14 @@ pub extern "C" fn bind_mutator( #[no_mangle] pub unsafe extern "C" fn mmtk_in_space(mmtk: &'static MMTK, object: ObjectReference, space: AllocationSemantics) -> i32 { match space { - AllocationSemantics::Default => mmtk.plan.in_default_space(object) as _, + AllocationSemantics::Default => { + (object.is_mapped() + && mmtk_in_space(mmtk, object, AllocationSemantics::ReadOnly) == 0 + && mmtk_in_space(mmtk, object, AllocationSemantics::Immortal) == 0 + && mmtk_in_space(mmtk, object, AllocationSemantics::Los) == 0 + && mmtk_in_space(mmtk, object, AllocationSemantics::Code) == 0 + && mmtk_in_space(mmtk, object, AllocationSemantics::LargeCode) == 0) as _ + }, AllocationSemantics::ReadOnly => mmtk.plan.base().ro_space.in_space(object) as _, AllocationSemantics::Immortal => mmtk.plan.common().immortal.in_space(object) as _, AllocationSemantics::Los => mmtk.plan.common().los.in_space(object) as _, diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index e1346b7..9288b34 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -58,7 +58,7 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots>(worker: &mut GCWorker) { let x = worker as *mut GCWorker as usize; - SINGLETON.scheduler.on_closure_end(Box::new(move || { + mmtk::memory_manager::on_closure_end(&SINGLETON, Box::new(move || { unsafe { let w = x as *mut GCWorker; debug_assert!(ROOT_OBJECTS.is_empty()); @@ -132,7 +132,24 @@ pub(crate) fn flush_roots>(_worker: &mut GCWorker::new(buf, false); + pub struct ScanRootObjects { + buffer: Vec, + phantom: PhantomData, + } + impl ScanRootObjects { + pub fn new(buffer: Vec) -> Self { + Self { + buffer, + phantom: PhantomData, + } + } + } + impl GCWork for ScanRootObjects { + fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + ::VMScanning::scan_objects::(&self.buffer, worker); + } + } + let scan_objects_work = ScanRootObjects::::new(buf); mmtk::memory_manager::add_work_packet( &SINGLETON, WorkBucketStage::Closure, From 92883df2083e0478489277919492250f9a8241e8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 21 Jul 2021 11:15:37 +1000 Subject: [PATCH 58/65] Update mmtk-core dependency --- mmtk/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index bd25e1f..fa8678e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -27,10 +27,10 @@ log = "*" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "1e7964656eb0988e3fa4f216c5f8dfb1dba7b31f" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "5d338222c1d02a1a3e7b9aecbda4f55285bd0e5b" } # Uncomment the following and fix the path to mmtk-core to build locally # mmtk = { path = "../repos/mmtk-core" } [features] -default = ["mmtk/code_space", "mmtk/ro_space"] +default = ["mmtk/code_space", "mmtk/ro_space", "mmtk/nogc_common_plan"] nogc = [] From 22cef95f94dec9ed4f998cd78aae114e22124716 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 23 Jul 2021 11:27:05 +1000 Subject: [PATCH 59/65] Fix build error --- mmtk/src/object_model.rs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 29e2510..c6d8e82 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,10 +1,8 @@ use std::sync::atomic::Ordering; use super::UPCALLS; -use mmtk::util::constants::{LOG_BITS_IN_WORD, LOG_BYTES_IN_WORD}; -use mmtk::util::metadata::side_metadata::{GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS, LOCAL_SIDE_METADATA_VM_BASE_ADDRESS, SideMetadataSpec}; -use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec, MetadataSpec}; -use mmtk::util::{Address, ObjectReference}; +use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec}; +use mmtk::util::{Address, ObjectReference, metadata}; use mmtk::vm::*; use mmtk::AllocationSemantics; use mmtk::CopyContext; @@ -13,31 +11,29 @@ use V8; pub struct VMObjectModel {} impl ObjectModel for VMObjectModel { - const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side(GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize()); + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = VMLocalForwardingPointerSpec::in_header(0); - const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = VMLocalForwardingBitsSpec::side(LOCAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize()); - const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side(Self::LOCAL_FORWARDING_BITS_SPEC.spec().as_side().unwrap().accumulated_size()); - const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = VMLocalLOSMarkNurserySpec::side(Self::LOCAL_MARK_BIT_SPEC.spec().as_side().unwrap().accumulated_size()); + const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = VMLocalForwardingBitsSpec::side_first(); + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_after(&Self::LOCAL_FORWARDING_BITS_SPEC.as_spec()); + const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = VMLocalLOSMarkNurserySpec::side_after(&Self::LOCAL_MARK_BIT_SPEC.as_spec()); fn load_metadata( metadata_spec: &HeaderMetadataSpec, object: ObjectReference, - _mask: Option, - _atomic_ordering: Option, + optional_mask: Option, + atomic_ordering: Option, ) -> usize { - debug_assert_eq!(metadata_spec, &Self::LOCAL_FORWARDING_POINTER_SPEC.as_header().unwrap()); - unsafe { object.to_address().load() } + metadata::header_metadata::load_metadata(metadata_spec, object, optional_mask, atomic_ordering) } fn store_metadata( metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, - _mask: Option, - _atomic_ordering: Option, + optional_mask: Option, + atomic_ordering: Option, ) { - debug_assert_eq!(metadata_spec, &Self::LOCAL_FORWARDING_POINTER_SPEC.as_header().unwrap()); - unsafe { object.to_address().store(val) } + metadata::header_metadata::store_metadata(metadata_spec, object, val, optional_mask, atomic_ordering) } fn compare_exchange_metadata( From 9aebd1bb7c9a67e337fe94ce1f4d3b570ce58d65 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 23 Jul 2021 12:08:33 +1000 Subject: [PATCH 60/65] cleanup --- mmtk/Cargo.toml | 5 ----- mmtk/src/api.rs | 5 ++++- mmtk/src/object_model.rs | 12 ++++++++++-- v8/third_party/heap/mmtk/mmtk-visitors.h | 2 +- v8/third_party/heap/mmtk/mmtk.h | 2 +- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index fa8678e..91d3109 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -8,12 +8,7 @@ name = "mmtk_v8" # be careful - LTO is only allowed for certain crate types crate-type = ["cdylib"] -[profile.dev] -panic = "abort" -opt-level = 3 - [profile.release] -panic = "abort" lto = true [dependencies] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index d7e606f..ca71a20 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -15,11 +15,13 @@ use V8_Upcalls; use UPCALLS; use V8; +/// Release an address buffer #[no_mangle] -pub unsafe extern "C" fn release_buffer(ptr: *mut Address, length: usize, capacity: usize) { +pub unsafe extern "C" fn mmtk_release_buffer(ptr: *mut Address, length: usize, capacity: usize) { let _vec = Vec::
::from_raw_parts(ptr, length, capacity); } +/// Check whether an object is movable. #[no_mangle] pub unsafe extern "C" fn mmtk_is_movable(object: ObjectReference) -> i32 { let object = { @@ -29,6 +31,7 @@ pub unsafe extern "C" fn mmtk_is_movable(object: ObjectReference) -> i32 { if object.is_movable() { 1 } else { 0 } } +/// Get the forwarding pointer, or NULL if the object is not forwarded #[no_mangle] pub unsafe extern "C" fn mmtk_get_forwarded_object(object: ObjectReference) -> *mut c_void { let tag = object.to_address().as_usize() & 0b11usize; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index c6d8e82..2efcbc7 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,6 +1,7 @@ use std::sync::atomic::Ordering; use super::UPCALLS; +use mmtk::util::metadata::MetadataSpec; use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec}; use mmtk::util::{Address, ObjectReference, metadata}; use mmtk::vm::*; @@ -24,16 +25,23 @@ impl ObjectModel for VMObjectModel { atomic_ordering: Option, ) -> usize { metadata::header_metadata::load_metadata(metadata_spec, object, optional_mask, atomic_ordering) + // unsafe { object.to_address().load() } } fn store_metadata( metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, - optional_mask: Option, + _optional_mask: Option, atomic_ordering: Option, ) { - metadata::header_metadata::store_metadata(metadata_spec, object, val, optional_mask, atomic_ordering) + metadata::header_metadata::store_metadata( + metadata_spec, + object, + val, + None, + atomic_ordering, + ); } fn compare_exchange_metadata( diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index d777e1c..06395fe 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -133,7 +133,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { virtual ~MMTkEdgeVisitor() { if (cursor_ > 0) flush(); if (buffer_ != NULL) { - release_buffer(buffer_, cursor_, cap_); + mmtk_release_buffer(buffer_, cursor_, cap_); } } diff --git a/v8/third_party/heap/mmtk/mmtk.h b/v8/third_party/heap/mmtk/mmtk.h index 5799d12..4352c02 100644 --- a/v8/third_party/heap/mmtk/mmtk.h +++ b/v8/third_party/heap/mmtk/mmtk.h @@ -71,7 +71,7 @@ extern void* tph_archive_iter_next(void* arch); extern void* tph_archive_inner_to_obj(void* arch, void* inner_ptr); extern int mmtk_in_space(void* mmtk, void* object, size_t space); -extern void release_buffer(void** buffer, size_t len, size_t cap); +extern void mmtk_release_buffer(void** buffer, size_t len, size_t cap); typedef struct { void** buf; From 7560cecc4f0d28f40e367e42b25d9894262753ea Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 23 Jul 2021 13:21:49 +1000 Subject: [PATCH 61/65] Cleanup --- mmtk/src/lib.rs | 2 ++ mmtk/src/object_model.rs | 10 +++------- mmtk/src/scanning.rs | 22 +--------------------- 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 8da85c9..3cfb9fd 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -79,6 +79,8 @@ impl VMBinding for V8 { lazy_static! { pub static ref SINGLETON: MMTK = { + // V8 can only support up to 8 worker threads. + // Set MMTK_THREADS = 7 here to exclude the main thread -- it undertakes part of the worker's job. if let Ok(threads) = env::var("MMTK_THREADS").map(|x| x.parse::().unwrap()) { if threads > 7 { env::set_var("MMTK_THREADS", "7"); diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 2efcbc7..f03690b 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,7 +1,6 @@ use std::sync::atomic::Ordering; - +use std::ptr; use super::UPCALLS; -use mmtk::util::metadata::MetadataSpec; use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec}; use mmtk::util::{Address, ObjectReference, metadata}; use mmtk::vm::*; @@ -25,7 +24,6 @@ impl ObjectModel for VMObjectModel { atomic_ordering: Option, ) -> usize { metadata::header_metadata::load_metadata(metadata_spec, object, optional_mask, atomic_ordering) - // unsafe { object.to_address().load() } } fn store_metadata( @@ -83,10 +81,8 @@ impl ObjectModel for VMObjectModel { let bytes = unsafe { ((*UPCALLS).get_object_size)(from) }; let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, allocator); // Copy - // println!("Copy {:?} -> {:?}", from, dst); - let src = from.to_address(); - for i in 0..bytes { - unsafe { (dst + i).store((src + i).load::()) }; + unsafe { + ptr::copy_nonoverlapping::(from.to_address().to_ptr(), dst.to_mut_ptr(), bytes); } let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, unsafe { Address::zero() }, bytes, allocator); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 9288b34..93e6775 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -83,27 +83,6 @@ impl Scanning for VMScanning { } } -pub struct ProcessEphemerons>(PhantomData); - -impl> ProcessEphemerons { - pub fn new() -> Self { - unreachable!(); - } -} - -impl> GCWork for ProcessEphemerons { - fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { - unsafe { - debug_assert!(ROOT_OBJECTS.is_empty()); - ((*UPCALLS).process_ephemerons)(trace_root:: as _, worker as *mut _ as _, worker.ordinal); - if !ROOT_OBJECTS.is_empty() { - flush_roots::(worker); - } - debug_assert!(ROOT_OBJECTS.is_empty()); - } - } -} - pub struct ScanAndForwardRoots>(PhantomData); impl> ScanAndForwardRoots { @@ -125,6 +104,7 @@ impl> GCWork for ScanAndForwardRoots { } } +/// No locks since we always use single-threaded root scanning. pub(crate) static mut ROOT_OBJECTS: Vec = Vec::new(); pub(crate) static mut ROOT_FLUSHED: bool = false; From fdcc01c96386fdfe9cdf87fc62fde59b769bee42 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 23 Jul 2021 15:38:37 +1000 Subject: [PATCH 62/65] Cleanup --- v8/third_party/heap/mmtk/mmtk-visitors.h | 78 +++------------------- v8/third_party/heap/mmtk/mmtkUpcalls.cc | 2 +- v8/third_party/heap/mmtk/weak-refs.h | 83 ++++-------------------- 3 files changed, 23 insertions(+), 140 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index 06395fe..ec539cb 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -16,14 +16,6 @@ #include "weak-refs.h" #include -#define WEAKREF_PROCESSING - -#ifdef WEAKREF_PROCESSING -#define WEAKREF_PROCESSING_BOOL true -#else -#define WEAKREF_PROCESSING_BOOL false -#endif - namespace mmtk { namespace i = v8::internal; @@ -137,23 +129,14 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } } - // V8_INLINE bool AllowDefaultJSObjectVisit() { return false; } - #ifdef WEAKREF_PROCESSING void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { - // if (!concrete_visitor()->ShouldVisit(table)) return 0; weak_objects_->ephemeron_hash_tables.Push(task_id_, table); for (auto i : table.IterateEntries()) { auto key_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); auto key = i::HeapObject::cast(table.KeyAt(i)); - - // VisitPointer(table, key_slot); - // concrete_visitor()->SynchronizePageAccess(key); - // concrete_visitor()->RecordSlot(table, key_slot, key); - auto value_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); - if (is_live(key)) { if (auto f = get_forwarded_ref(key)) { key_slot.store(*f); @@ -163,62 +146,28 @@ class MMTkEdgeVisitor: public i::HeapVisitor { auto value_obj = table.ValueAt(i); if (value_obj.IsHeapObject()) { auto value = i::HeapObject::cast(value_obj); - // concrete_visitor()->SynchronizePageAccess(value); - // concrete_visitor()->RecordSlot(table, value_slot, value); - - // // Revisit ephemerons with both key and value unreachable at end - // // of concurrent marking cycle. - // if (concrete_visitor()->marking_state()->IsWhite(value)) { weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); - // } } } } - // return table.SizeFromMap(map); } V8_INLINE void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { - // if (!ShouldVisit(weak_cell)) return; - int size = i::WeakCell::BodyDescriptor::SizeOf(map, weak_cell); this->VisitMapPointer(weak_cell); i::WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this); - // i::HeapObject target = weak_cell.relaxed_target(); - // i::HeapObject unregister_token = weak_cell.relaxed_unregister_token(); - // concrete_visitor()->SynchronizePageAccess(target); - // concrete_visitor()->SynchronizePageAccess(unregister_token); - // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target) && - // concrete_visitor()->marking_state()->IsBlackOrGrey(unregister_token)) { - // // Record the slots inside the WeakCell, since the IterateBody above - // // didn't visit it. - // ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); - // concrete_visitor()->RecordSlot(weak_cell, slot, target); - // slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); - // concrete_visitor()->RecordSlot(weak_cell, slot, unregister_token); - // } else { - // WeakCell points to a potentially dead object or a dead unregister - // token. We have to process them when we know the liveness of the whole - // transitive closure. - weak_objects_->weak_cells.Push(task_id_, weak_cell); - // } + // WeakCell points to a potentially dead object or a dead unregister + // token. We have to process them when we know the liveness of the whole + // transitive closure. + weak_objects_->weak_cells.Push(task_id_, weak_cell); } V8_INLINE void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { VisitJSObjectSubclass(map, weak_ref); - // if (size == 0) return 0; if (weak_ref.target().IsHeapObject()) { - // i::HeapObject target = i::HeapObject::cast(weak_ref.target()); - // SynchronizePageAccess(target); - // if (concrete_visitor()->marking_state()->IsBlackOrGrey(target)) { - // // Record the slot inside the JSWeakRef, since the - // // VisitJSObjectSubclass above didn't visit it. - // ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); - // concrete_visitor()->RecordSlot(weak_ref, slot, target); - // } else { - // JSWeakRef points to a potentially dead object. We have to process - // them when we know the liveness of the whole transitive closure. - weak_objects_->js_weak_refs.Push(task_id_, weak_ref); - // } + // JSWeakRef points to a potentially dead object. We have to process + // them when we know the liveness of the whole transitive closure. + weak_objects_->js_weak_refs.Push(task_id_, weak_ref); } } @@ -227,9 +176,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { int size = i::BytecodeArray::BodyDescriptor::SizeOf(map, object); this->VisitMapPointer(object); i::BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); - // if (!is_forced_gc_) { - object.MakeOlder(); - // } + object.MakeOlder(); } V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { @@ -242,7 +189,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { - // if (!ShouldVisit(shared_info)) return; int size = i::SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); VisitMapPointer(shared_info); i::SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); @@ -252,8 +198,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { if (auto f = mmtk::get_forwarded_ref(i::HeapObject::cast(data))) { shared_info.set_function_data(*f, v8::kReleaseStore); } - // trace_field_((void*) &data, context_); - // shared_info.set_function_data(data, v8::kReleaseStore); } // If the SharedFunctionInfo has old bytecode, mark it as flushable, @@ -267,14 +211,12 @@ class MMTkEdgeVisitor: public i::HeapVisitor { template void VisitJSObjectSubclass(i::Map map, T object) { - // if (!ShouldVisit(object)) return; VisitMapPointer(object); int size = T::BodyDescriptor::SizeOf(map, object); T::BodyDescriptor::IterateBody(map, object, size, this); } V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { - // if (!ShouldVisit(array)) return; VisitMapPointer(array); int size = i::TransitionArray::BodyDescriptor::SizeOf(map, array); i::TransitionArray::BodyDescriptor::IterateBody(map, array, size, this); @@ -297,6 +239,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { void VisitDescriptors(i::DescriptorArray descriptor_array, int number_of_own_descriptors) { int16_t new_marked = static_cast(number_of_own_descriptors); + // Note: Always trace all the element in descriptor_arrays. // int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors( // heap_->gc_count(), new_marked); // if (old_marked < new_marked) { @@ -403,9 +346,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } else if (TSlot::kCanBeWeak && (*slot).GetHeapObjectIfWeak(&object)) { if (!WEAKREF_PROCESSING_BOOL) { PushEdge((void*) slot.address()); - // } else if (auto f = mmtk::get_forwarded_ref(object)) { - // i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); - // s.StoreHeapObject(*f); } else { i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); diff --git a/v8/third_party/heap/mmtk/mmtkUpcalls.cc b/v8/third_party/heap/mmtk/mmtkUpcalls.cc index 9815bbd..5e11681 100644 --- a/v8/third_party/heap/mmtk/mmtkUpcalls.cc +++ b/v8/third_party/heap/mmtk/mmtkUpcalls.cc @@ -193,6 +193,6 @@ V8_Upcalls mmtk_upcalls = { mmtk_on_move_event, mmtk_process_ephemerons, }; -} // namespace third_party_heap +} // namespace third_party_heap } // namespace internal } // namespace v8 \ No newline at end of file diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 88a4abe..3487e5a 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -7,6 +7,14 @@ #include "src/objects/transitions-inl.h" #include "src/heap/objects-visiting.h" +#define WEAKREF_PROCESSING + +#ifdef WEAKREF_PROCESSING +#define WEAKREF_PROCESSING_BOOL true +#else +#define WEAKREF_PROCESSING_BOOL false +#endif + namespace v8 { namespace internal { namespace third_party_heap { @@ -22,19 +30,13 @@ namespace mmtk { class MMTkWeakObjectRetainer: public i::WeakObjectRetainer { public: virtual i::Object RetainAs(i::Object object) override final { - // LOG("RetainAs %p\n", (void*) object.ptr()); if (object == i::Object()) return object; auto heap_object = i::HeapObject::cast(object); if (is_live(heap_object)) { auto f = mmtk_get_forwarded_object(heap_object); - if (f != nullptr) { - // LOG("%p -> %p\n", (void*) object.ptr(), (void*) f); - return i::Object((i::Address) f); - } - // LOG("%p is dead 1 \n", (void*) object.ptr()); + if (f != nullptr) return i::Object((i::Address) f); return object; } else { - // LOG("%p is dead 2 \n", (void*) object.ptr()); return i::Object(); } } @@ -131,18 +133,7 @@ class WeakRefs { // Replace bytecode array with an uncompiled data array. auto compiled_data = shared_info.GetBytecodeArray(isolate()); - // auto compiled_data_start = compiled_data.address(); int compiled_data_size = compiled_data.Size(); - // auto chunk = MemoryChunk::FromAddress(compiled_data_start); - - // Clear any recorded slots for the compiled data as being invalid. - // DCHECK_NULL(chunk->sweeping_slot_set()); - // RememberedSet::RemoveRange( - // chunk, compiled_data_start, compiled_data_start + compiled_data_size, - // SlotSet::FREE_EMPTY_BUCKETS); - // RememberedSet::RemoveRange( - // chunk, compiled_data_start, compiled_data_start + compiled_data_size, - // SlotSet::FREE_EMPTY_BUCKETS); // Swap the map, using set_map_after_allocation to avoid verify heap checks // which are not necessary since we are doing this during the GC atomic pause. @@ -169,11 +160,8 @@ class WeakRefs { // Mark the uncompiled data as black, and ensure all fields have already been // marked. DCHECK(is_live(inferred_name)); - // DCHECK(is_live(uncompiled_data)); trace_((void*) &uncompiled_data); - // auto forwarded = trace_((void*) &uncompiled_data); - // if (forwarded) uncompiled_data = i::UncompiledData::cast(*forwarded); // Use the raw function data setter to avoid validity checks, since we're // performing the unusual task of decompiling. @@ -198,11 +186,6 @@ class WeakRefs { } else { DCHECK(!get_forwarded_ref(flushing_candidate.GetBytecodeArray(heap()->isolate()))); } - // Now record the slot, which has either been updated to an uncompiled data, - // or is the BytecodeArray which is still alive. - // auto slot = - // flushing_candidate.RawField(SharedFunctionInfo::kFunctionDataOffset); - // RecordSlot(flushing_candidate, slot, HeapObject::cast(*slot)); } } @@ -288,13 +271,8 @@ class WeakRefs { if (i != transition_index) { i::Name key = transitions.GetKey(i); transitions.SetKey(transition_index, key); - // i::HeapObjectSlot key_slot = transitions.GetKeySlot(transition_index); - // RecordSlot(transitions, key_slot, key); i::MaybeObject raw_target = transitions.GetRawTarget(i); transitions.SetRawTarget(transition_index, raw_target); - // i::HeapObjectSlot target_slot = - // transitions.GetTargetSlot(transition_index); - // RecordSlot(transitions, target_slot, raw_target->GetHeapObject()); } transition_index++; } @@ -323,12 +301,6 @@ class WeakRefs { DCHECK_LE(0, new_nof_all_descriptors); auto start = array.GetDescriptorSlot(new_nof_all_descriptors).address(); auto end = array.GetDescriptorSlot(old_nof_all_descriptors).address(); - // MemoryChunk* chunk = MemoryChunk::FromHeapObject(array); - // DCHECK_NULL(chunk->sweeping_slot_set()); - // RememberedSet::RemoveRange(chunk, start, end, - // SlotSet::FREE_EMPTY_BUCKETS); - // RememberedSet::RemoveRange(chunk, start, end, - // SlotSet::FREE_EMPTY_BUCKETS); heap()->CreateFillerObjectAt(start, static_cast(end - start), i::ClearRecordedSlots::kNo); array.set_number_of_all_descriptors(new_nof_all_descriptors); } @@ -421,27 +393,7 @@ class WeakRefs { } location.store(cleared_weak_ref); } - } /*else if ((*location)->GetHeapObjectIfStrong(&value)) { - DCHECK(!value.IsCell()); - if (is_live(value)) { - // The value of the weak reference is alive. - // RecordSlot(slot.first, HeapObjectSlot(location), value); - auto forwarded = get_forwarded_ref(value); - if (forwarded) { - printf("[WeakRef] Strong %p -> %p\n", (void*) value.ptr(), (void*) forwarded->ptr()); - location.store(to_weakref(*forwarded)); - } else { - printf("[WeakRef] Strong %p \n", (void*) value.ptr()); - } - } else { - printf("[WeakRef] Strong Dead %p\n", (void*) value.ptr()); - if (value.IsMap()) { - // The map is non-live. - ClearPotentialSimpleMapTransition(i::Map::cast(value)); - } - location.store(cleared_weak_ref); - } - }*/ + } } } @@ -488,9 +440,6 @@ class WeakRefs { } else { auto forwarded = get_forwarded_ref(weak_ref.target()); if (forwarded) weak_ref.set_target(*forwarded); - // The value of the JSWeakRef is alive. - // i::ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); - // RecordSlot(weak_ref, slot, target); } } i::WeakCell weak_cell; @@ -521,9 +470,6 @@ class WeakRefs { if (auto f = get_forwarded_ref(target)) { slot.store(*f); } - // The value of the WeakCell is alive. - // i::ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); - // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); } i::ObjectSlot slot = weak_cell.RawField(i::WeakCell::kUnregisterTokenOffset); @@ -546,9 +492,6 @@ class WeakRefs { if (auto f = get_forwarded_ref(unregister_token)) { slot.store(*f); } - // The unregister_token is alive. - // ObjectSlot slot = weak_cell.RawField(WeakCell::kUnregisterTokenOffset); - // RecordSlot(weak_cell, slot, HeapObject::cast(*slot)); } } heap()->PostFinalizationRegistryCleanupTaskIfNeeded(); @@ -615,9 +558,9 @@ class WeakRefs { void ProcessEphemerons() { Flush(); - class XRootVisitor: public i::RootVisitor { + class InternalRootVisitor: public i::RootVisitor { public: - explicit XRootVisitor(std::function& trace): trace_(trace) {} + explicit InternalRootVisitor(std::function& trace): trace_(trace) {} virtual void VisitRootPointer(i::Root root, const char* description, i::FullObjectSlot p) override final { ProcessRootEdge(root, p); @@ -639,7 +582,7 @@ class WeakRefs { }; { - XRootVisitor root_visitor(trace_); + InternalRootVisitor root_visitor(trace_); isolate()->global_handles()->IterateWeakRootsForFinalizers(&root_visitor); // isolate()->global_handles()->IterateWeakRootsForPhantomHandles(&root_visitor); } From fd768aa82da9681eb68ca3824729de0853f80fc4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 23 Jul 2021 16:44:31 +1000 Subject: [PATCH 63/65] Cleanup --- v8/third_party/heap/mmtk/weak-refs.h | 32 ---------------------------- 1 file changed, 32 deletions(-) diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 3487e5a..71a285d 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -558,36 +558,6 @@ class WeakRefs { void ProcessEphemerons() { Flush(); - class InternalRootVisitor: public i::RootVisitor { - public: - explicit InternalRootVisitor(std::function& trace): trace_(trace) {} - - virtual void VisitRootPointer(i::Root root, const char* description, i::FullObjectSlot p) override final { - ProcessRootEdge(root, p); - } - virtual void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override final { - for (auto p = start; p < end; ++p) ProcessRootEdge(root, p); - } - virtual void VisitRootPointers(i::Root root, const char* description, i::OffHeapObjectSlot start, i::OffHeapObjectSlot end) override final { - for (auto p = start; p < end; ++p) ProcessRootEdge(root, p); - } - private: - V8_INLINE void ProcessRootEdge(i::Root root, i::FullObjectSlot slot) { - i::HeapObject object; - if ((*slot).GetHeapObject(&object)) { - trace_((void*) slot.address()); - } - } - std::function& trace_; - }; - - { - InternalRootVisitor root_visitor(trace_); - isolate()->global_handles()->IterateWeakRootsForFinalizers(&root_visitor); - // isolate()->global_handles()->IterateWeakRootsForPhantomHandles(&root_visitor); - } - - i::Ephemeron ephemeron; DCHECK(weak_objects_.current_ephemerons.IsEmpty()); @@ -616,8 +586,6 @@ class WeakRefs { Flush(); have_code_to_deoptimize_ = false; { - // TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR_STRING_TABLE); - // Prune the string table removing all strings only pointed to by the // string table. Cannot use string_table() here because the string // table is marked. From de973a23703225a9cedcebdf5ee078127580518f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 3 Aug 2021 20:21:32 +1000 Subject: [PATCH 64/65] Remove weakref processing code --- v8/third_party/heap/mmtk/mmtk-visitors.h | 111 +---- v8/third_party/heap/mmtk/weak-refs.h | 570 +---------------------- 2 files changed, 13 insertions(+), 668 deletions(-) diff --git a/v8/third_party/heap/mmtk/mmtk-visitors.h b/v8/third_party/heap/mmtk/mmtk-visitors.h index ec539cb..0530417 100644 --- a/v8/third_party/heap/mmtk/mmtk-visitors.h +++ b/v8/third_party/heap/mmtk/mmtk-visitors.h @@ -120,6 +120,7 @@ class MMTkEdgeVisitor: public i::HeapVisitor { NewBuffer buf = process_edges(NULL, 0, 0); buffer_ = buf.buf; cap_ = buf.cap; + USE(task_id_); } virtual ~MMTkEdgeVisitor() { @@ -129,108 +130,6 @@ class MMTkEdgeVisitor: public i::HeapVisitor { } } -#ifdef WEAKREF_PROCESSING - void VisitEphemeronHashTable(i::Map map, i::EphemeronHashTable table) { - weak_objects_->ephemeron_hash_tables.Push(task_id_, table); - - for (auto i : table.IterateEntries()) { - auto key_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); - auto key = i::HeapObject::cast(table.KeyAt(i)); - auto value_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); - if (is_live(key)) { - if (auto f = get_forwarded_ref(key)) { - key_slot.store(*f); - } - VisitPointer(table, value_slot); - } else { - auto value_obj = table.ValueAt(i); - if (value_obj.IsHeapObject()) { - auto value = i::HeapObject::cast(value_obj); - weak_objects_->discovered_ephemerons.Push(task_id_, i::Ephemeron{key, value}); - } - } - } - } - - V8_INLINE void VisitWeakCell(i::Map map, i::WeakCell weak_cell) { - int size = i::WeakCell::BodyDescriptor::SizeOf(map, weak_cell); - this->VisitMapPointer(weak_cell); - i::WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this); - // WeakCell points to a potentially dead object or a dead unregister - // token. We have to process them when we know the liveness of the whole - // transitive closure. - weak_objects_->weak_cells.Push(task_id_, weak_cell); - } - - V8_INLINE void VisitJSWeakRef(i::Map map, i::JSWeakRef weak_ref) { - VisitJSObjectSubclass(map, weak_ref); - if (weak_ref.target().IsHeapObject()) { - // JSWeakRef points to a potentially dead object. We have to process - // them when we know the liveness of the whole transitive closure. - weak_objects_->js_weak_refs.Push(task_id_, weak_ref); - } - } - - V8_INLINE void VisitBytecodeArray(i::Map map, i::BytecodeArray object) { - if (!ShouldVisit(object)) return; - int size = i::BytecodeArray::BodyDescriptor::SizeOf(map, object); - this->VisitMapPointer(object); - i::BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); - object.MakeOlder(); - } - - V8_INLINE void VisitJSFunction(i::Map map, i::JSFunction object) { - VisitJSObjectSubclass(map, object); - // Check if the JSFunction needs reset due to bytecode being flushed. - if (/*bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode &&*/ - object.NeedsResetDueToFlushedBytecode()) { - weak_objects_->flushed_js_functions.Push(task_id_, object); - } - } - - V8_INLINE void VisitSharedFunctionInfo(i::Map map, i::SharedFunctionInfo shared_info) { - int size = i::SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); - VisitMapPointer(shared_info); - i::SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); - - auto data = shared_info.function_data(v8::kAcquireLoad); - if (data.IsHeapObject() || data.IsWeak()) { - if (auto f = mmtk::get_forwarded_ref(i::HeapObject::cast(data))) { - shared_info.set_function_data(*f, v8::kReleaseStore); - } - } - - // If the SharedFunctionInfo has old bytecode, mark it as flushable, - // otherwise visit the function data field strongly. - if (shared_info.ShouldFlushBytecode(i::Heap::GetBytecodeFlushMode(heap_->isolate()))) { - weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); - } else { - VisitPointer(shared_info, shared_info.RawField(i::SharedFunctionInfo::kFunctionDataOffset)); - } - } - - template - void VisitJSObjectSubclass(i::Map map, T object) { - VisitMapPointer(object); - int size = T::BodyDescriptor::SizeOf(map, object); - T::BodyDescriptor::IterateBody(map, object, size, this); - } - - V8_INLINE void VisitTransitionArray(i::Map map, i::TransitionArray array) { - VisitMapPointer(array); - int size = i::TransitionArray::BodyDescriptor::SizeOf(map, array); - i::TransitionArray::BodyDescriptor::IterateBody(map, array, size, this); - weak_objects_->transition_arrays.Push(task_id_, array); - } - - // Custom weak pointers must be ignored by the GC but not other - // visitors. They're used for e.g., lists that are recreated after GC. The - // default implementation treats them as strong pointers. Visitors who want to - // ignore them must override this function with empty. - virtual void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) override final {} - -#endif - V8_INLINE void VisitDescriptorArray(i::Map map, i::DescriptorArray array) { VisitMapPointer(array); VisitPointers(array, array.GetFirstPointerSlot(), array.GetDescriptorSlot(0)); @@ -322,8 +221,8 @@ class MMTkEdgeVisitor: public i::HeapVisitor { if (auto forwarded = mmtk::get_forwarded_ref(o)) { rinfo->set_target_object(heap_, *forwarded); } else if (host.IsWeakObject(o) && WEAKREF_PROCESSING_BOOL) { - DCHECK(i::RelocInfo::IsCodeTarget(rinfo->rmode()) || i::RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); - weak_objects_->weak_objects_in_code.Push(task_id_, std::make_pair(o, host)); + // TODO: Enable weak ref processing + UNIMPLEMENTED(); } else { if (auto forwarded = trace_field_(o)) { rinfo->set_target_object(heap_, *forwarded); @@ -347,8 +246,8 @@ class MMTkEdgeVisitor: public i::HeapVisitor { if (!WEAKREF_PROCESSING_BOOL) { PushEdge((void*) slot.address()); } else { - i::HeapObjectSlot s = i::HeapObjectSlot(slot.address()); - weak_objects_->weak_references.Push(task_id_, std::make_pair(host, s)); + // TODO: Enable weak ref processing + UNIMPLEMENTED(); } } } diff --git a/v8/third_party/heap/mmtk/weak-refs.h b/v8/third_party/heap/mmtk/weak-refs.h index 71a285d..482ae1d 100644 --- a/v8/third_party/heap/mmtk/weak-refs.h +++ b/v8/third_party/heap/mmtk/weak-refs.h @@ -7,7 +7,8 @@ #include "src/objects/transitions-inl.h" #include "src/heap/objects-visiting.h" -#define WEAKREF_PROCESSING +// TODO: Enable weak ref processing +// #define WEAKREF_PROCESSING #ifdef WEAKREF_PROCESSING #define WEAKREF_PROCESSING_BOOL true @@ -42,536 +43,18 @@ class MMTkWeakObjectRetainer: public i::WeakObjectRetainer { } }; -class InternalizedStringTableCleaner : public i::RootVisitor { - public: - explicit InternalizedStringTableCleaner(i::Heap* heap) - : heap_(heap), pointers_removed_(0) {} - - void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override { - UNREACHABLE(); - } - - void VisitRootPointers(i::Root root, const char* description, i::OffHeapObjectSlot start, i::OffHeapObjectSlot end) override { - DCHECK_EQ(root, i::Root::kStringTable); - // Visit all HeapObject pointers in [start, end). - auto isolate = heap_->isolate(); - for (auto p = start; p < end; ++p) { - i::Object o = p.load(isolate); - if (o.IsHeapObject()) { - auto heap_object = i::HeapObject::cast(o); - DCHECK(!i::Heap::InYoungGeneration(heap_object)); - if (!is_live(heap_object)) { - pointers_removed_++; - // Set the entry to the_hole_value (as deleted). - p.store(i::StringTable::deleted_element()); - } else { - auto forwarded = get_forwarded_ref(heap_object); - if (forwarded) p.store(*forwarded); - } - } - } - } - - int PointersRemoved() { return pointers_removed_; } - - private: - i::Heap* heap_; - int pointers_removed_; -}; - -class ExternalStringTableCleaner : public i::RootVisitor { - public: - explicit ExternalStringTableCleaner(i::Heap* heap) : heap_(heap) {} - - void VisitRootPointers(i::Root root, const char* description, i::FullObjectSlot start, i::FullObjectSlot end) override { - // Visit all HeapObject pointers in [start, end). - auto the_hole = i::ReadOnlyRoots(heap_).the_hole_value(); - for (auto p = start; p < end; ++p) { - i::Object o = *p; - if (o.IsHeapObject()) { - auto heap_object = i::HeapObject::cast(o); - if (!is_live(heap_object)) { - if (o.IsExternalString()) { - heap_->FinalizeExternalString(i::String::cast(o)); - } else { - // The original external string may have been internalized. - DCHECK(o.IsThinString()); - } - // Set the entry to the_hole_value (as deleted). - p.store(the_hole); - } else { - auto forwarded = get_forwarded_ref(heap_object); - if (forwarded) p.store(*forwarded); - } - } - } - } - - private: - i::Heap* heap_; -}; - - class WeakRefs { static constexpr int kMainThreadTask = 0; - i::WeakObjects weak_objects_; - - void FlushBytecodeFromSFI(i::SharedFunctionInfo shared_info) { - DCHECK(shared_info.HasBytecodeArray()); - // Retain objects required for uncompiled data. - auto inferred_name = shared_info.inferred_name(); - int start_position = shared_info.StartPosition(); - int end_position = shared_info.EndPosition(); - - shared_info.DiscardCompiledMetadata(isolate(), [](i::HeapObject object, i::ObjectSlot slot, i::HeapObject target) { - // RecordSlot(object, slot, target); - }); - - // The size of the bytecode array should always be larger than an - // UncompiledData object. - STATIC_ASSERT(i::BytecodeArray::SizeFor(0) >= i::UncompiledDataWithoutPreparseData::kSize); - - // Replace bytecode array with an uncompiled data array. - auto compiled_data = shared_info.GetBytecodeArray(isolate()); - int compiled_data_size = compiled_data.Size(); - - // Swap the map, using set_map_after_allocation to avoid verify heap checks - // which are not necessary since we are doing this during the GC atomic pause. - compiled_data.set_map_after_allocation( - i::ReadOnlyRoots(heap()).uncompiled_data_without_preparse_data_map(), - i::SKIP_WRITE_BARRIER); - - // Create a filler object for any left over space in the bytecode array. - if (!heap()->IsLargeObject(compiled_data)) { - heap()->CreateFillerObjectAt( - compiled_data.address() + i::UncompiledDataWithoutPreparseData::kSize, - compiled_data_size - i::UncompiledDataWithoutPreparseData::kSize, - i::ClearRecordedSlots::kNo); - } - - // Initialize the uncompiled data. - auto uncompiled_data = i::UncompiledData::cast(compiled_data); - uncompiled_data.InitAfterBytecodeFlush( - inferred_name, start_position, end_position, - [](i::HeapObject object, i::ObjectSlot slot, i::HeapObject target) { - // RecordSlot(object, slot, target); - }); - - // Mark the uncompiled data as black, and ensure all fields have already been - // marked. - DCHECK(is_live(inferred_name)); - - trace_((void*) &uncompiled_data); - - // Use the raw function data setter to avoid validity checks, since we're - // performing the unusual task of decompiling. - shared_info.set_function_data(uncompiled_data, v8::kReleaseStore); - DCHECK(!shared_info.is_compiled()); - } - - void ClearOldBytecodeCandidates() { - DCHECK(i::FLAG_flush_bytecode || weak_objects_.bytecode_flushing_candidates.IsEmpty()); - i::SharedFunctionInfo flushing_candidate; - while (weak_objects_.bytecode_flushing_candidates.Pop(kMainThreadTask, &flushing_candidate)) { - // If the BytecodeArray is dead, flush it, which will replace the field with - // an uncompiled data object. - auto data = flushing_candidate.function_data(v8::kAcquireLoad); - if (data.IsHeapObject() || data.IsWeak()) { - if (auto f = get_forwarded_ref(i::HeapObject::cast(data))) { - flushing_candidate.set_function_data(*f, v8::kReleaseStore); - } - } - if (!is_live(flushing_candidate.GetBytecodeArray(heap()->isolate()))) { - FlushBytecodeFromSFI(flushing_candidate); - } else { - DCHECK(!get_forwarded_ref(flushing_candidate.GetBytecodeArray(heap()->isolate()))); - } - } - } - - void ClearFlushedJsFunctions() { - DCHECK(i::FLAG_flush_bytecode || weak_objects_.flushed_js_functions.IsEmpty()); - i::JSFunction flushed_js_function; - while (weak_objects_.flushed_js_functions.Pop(kMainThreadTask, &flushed_js_function)) { - auto gc_notify_updated_slot = [](i::HeapObject object, i::ObjectSlot slot, i::Object target) { - // RecordSlot(object, slot, HeapObject::cast(target)); - }; - flushed_js_function.ResetIfBytecodeFlushed(gc_notify_updated_slot); - } - } - - void ClearFullMapTransitions() { - i::TransitionArray array; - while (weak_objects_.transition_arrays.Pop(kMainThreadTask, &array)) { - int num_transitions = array.number_of_entries(); - if (num_transitions > 0) { - i::Map map; - // The array might contain "undefined" elements because it's not yet - // filled. Allow it. - if (array.GetTargetIfExists(0, isolate(), &map)) { - DCHECK(!map.is_null()); // Weak pointers aren't cleared yet. - auto constructor_or_back_pointer = map.constructor_or_back_pointer(); - if (constructor_or_back_pointer.IsSmi()) { - DCHECK(isolate()->has_active_deserializer()); - DCHECK_EQ(constructor_or_back_pointer, i::Smi::uninitialized_deserialization_value()); - continue; - } - auto parent = i::Map::cast(map.constructor_or_back_pointer()); - bool parent_is_alive = is_live(parent); - if (parent_is_alive) { - DCHECK(!get_forwarded_ref(parent)); - } - auto descriptors = parent_is_alive ? parent.instance_descriptors(isolate()) : i::DescriptorArray(); - bool descriptors_owner_died = CompactTransitionArray(parent, array, descriptors); - if (descriptors_owner_died) { - TrimDescriptorArray(parent, descriptors); - } - } - } - } - } - - bool TransitionArrayNeedsCompaction(i::TransitionArray transitions, int num_transitions) { - for (int i = 0; i < num_transitions; ++i) { - i::MaybeObject raw_target = transitions.GetRawTarget(i); - if (raw_target.IsSmi()) { - // This target is still being deserialized, - DCHECK(isolate()->has_active_deserializer()); - DCHECK_EQ(raw_target.ToSmi(), i::Smi::uninitialized_deserialization_value()); - return false; - } else if (!is_live(tph::Impl::TransitionsAccessor_GetTargetFromRaw(raw_target))) { - return true; - } else { - DCHECK(!get_forwarded_ref(tph::Impl::TransitionsAccessor_GetTargetFromRaw(raw_target))); - } - } - return false; - } - - bool CompactTransitionArray(i::Map map, i::TransitionArray transitions, i::DescriptorArray descriptors) { - DCHECK(!map.is_prototype_map()); - int num_transitions = transitions.number_of_entries(); - if (!TransitionArrayNeedsCompaction(transitions, num_transitions)) { - return false; - } - bool descriptors_owner_died = false; - int transition_index = 0; - // Compact all live transitions to the left. - for (int i = 0; i < num_transitions; ++i) { - i::Map target = transitions.GetTarget(i); - DCHECK_EQ(target.constructor_or_back_pointer(), map); - if (!is_live(target)) { - if (!descriptors.is_null() && - target.instance_descriptors(isolate()) == descriptors) { - DCHECK(!target.is_prototype_map()); - descriptors_owner_died = true; - } - } else { - DCHECK(!get_forwarded_ref(target)); - if (i != transition_index) { - i::Name key = transitions.GetKey(i); - transitions.SetKey(transition_index, key); - i::MaybeObject raw_target = transitions.GetRawTarget(i); - transitions.SetRawTarget(transition_index, raw_target); - } - transition_index++; - } - } - // If there are no transitions to be cleared, return. - if (transition_index == num_transitions) { - DCHECK(!descriptors_owner_died); - return false; - } - // Note that we never eliminate a transition array, though we might right-trim - // such that number_of_transitions() == 0. If this assumption changes, - // TransitionArray::Insert() will need to deal with the case that a transition - // array disappeared during GC. - int trim = tph::Impl::TransitionArray_Capacity(transitions) - transition_index; - if (trim > 0) { - heap()->RightTrimWeakFixedArray(transitions, trim * i::TransitionArray::kEntrySize); - tph::Impl::TransitionArray_SetNumberOfTransitions(transitions, transition_index); - } - return descriptors_owner_died; - } - - void RightTrimDescriptorArray(i::DescriptorArray array, int descriptors_to_trim) { - int old_nof_all_descriptors = array.number_of_all_descriptors(); - int new_nof_all_descriptors = old_nof_all_descriptors - descriptors_to_trim; - DCHECK_LT(0, descriptors_to_trim); - DCHECK_LE(0, new_nof_all_descriptors); - auto start = array.GetDescriptorSlot(new_nof_all_descriptors).address(); - auto end = array.GetDescriptorSlot(old_nof_all_descriptors).address(); - heap()->CreateFillerObjectAt(start, static_cast(end - start), i::ClearRecordedSlots::kNo); - array.set_number_of_all_descriptors(new_nof_all_descriptors); - } - - void TrimDescriptorArray(i::Map map, i::DescriptorArray descriptors) { - int number_of_own_descriptors = map.NumberOfOwnDescriptors(); - if (number_of_own_descriptors == 0) { - DCHECK(descriptors == i::ReadOnlyRoots(heap()).empty_descriptor_array()); - return; - } - int to_trim = descriptors.number_of_all_descriptors() - number_of_own_descriptors; - if (to_trim > 0) { - descriptors.set_number_of_descriptors(number_of_own_descriptors); - RightTrimDescriptorArray(descriptors, to_trim); - TrimEnumCache(map, descriptors); - descriptors.Sort(); - } - DCHECK(descriptors.number_of_descriptors() == number_of_own_descriptors); - map.set_owns_descriptors(true); - } - - void TrimEnumCache(i::Map map, i::DescriptorArray descriptors) { - int live_enum = map.EnumLength(); - if (live_enum == i::kInvalidEnumCacheSentinel) { - live_enum = map.NumberOfEnumerableProperties(); - } - if (live_enum == 0) return descriptors.ClearEnumCache(); - auto enum_cache = descriptors.enum_cache(); - - auto keys = enum_cache.keys(); - int to_trim = keys.length() - live_enum; - if (to_trim <= 0) return; - heap()->RightTrimFixedArray(keys, to_trim); - - auto indices = enum_cache.indices(); - to_trim = indices.length() - live_enum; - if (to_trim <= 0) return; - heap()->RightTrimFixedArray(indices, to_trim); - } - - void ClearWeakCollections() { - i::EphemeronHashTable table; - while (weak_objects_.ephemeron_hash_tables.Pop(kMainThreadTask, &table)) { - for (i::InternalIndex i : table.IterateEntries()) { - auto key = i::HeapObject::cast(table.KeyAt(i)); - if (!is_live(key)) { - tph::Impl::EphemeronHashTable_RemoveEntry(table, i); - } else { - if (auto f = get_forwarded_ref(key)) { - auto key_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToIndex(i)); - key_slot.store(*f); - } - auto value = i::HeapObject::cast(table.ValueAt(i)); - if (auto f = get_forwarded_ref(value)) { - auto value_slot = table.RawFieldOfElementAt(i::EphemeronHashTable::EntryToValueIndex(i)); - value_slot.store(*f); - } - } - } - } - for (auto it = heap()->ephemeron_remembered_set_.begin(); it != heap()->ephemeron_remembered_set_.end();) { - if (!is_live(it->first)) { - it = heap()->ephemeron_remembered_set_.erase(it); - } else { - DCHECK(!get_forwarded_ref(it->first)); - ++it; - } - } - } - - void ClearWeakReferences() { - std::pair slot; - auto cleared_weak_ref = i::HeapObjectReference::ClearedValue(isolate()); - while (weak_objects_.weak_references.Pop(kMainThreadTask, &slot)) { - i::HeapObject value; - // The slot could have been overwritten, so we have to treat it - // as MaybeObjectSlot. - i::MaybeObjectSlot location(slot.second); - if ((*location)->GetHeapObjectIfWeak(&value)) { - DCHECK(!value.IsCell()); - if (is_live(value)) { - // The value of the weak reference is alive. - // RecordSlot(slot.first, HeapObjectSlot(location), value); - auto forwarded = get_forwarded_ref(value); - if (forwarded) location.store(to_weakref(*forwarded)); - } else { - if (value.IsMap()) { - // The map is non-live. - ClearPotentialSimpleMapTransition(i::Map::cast(value)); - } - location.store(cleared_weak_ref); - } - } - } - } - - void ClearPotentialSimpleMapTransition(i::Map dead_target) { - DCHECK(!is_live(dead_target)); - auto potential_parent = dead_target.constructor_or_back_pointer(); - if (is_live(i::HeapObject::cast(potential_parent))) { - if (auto f = get_forwarded_ref(i::HeapObject::cast(potential_parent))) { - potential_parent = *f; - } - } - if (potential_parent.IsMap()) { - auto parent = i::Map::cast(potential_parent); - i::DisallowGarbageCollection no_gc_obviously; - if (is_live(parent)) { - DCHECK(!get_forwarded_ref(parent)); - } - if (is_live(parent) && - tph::Impl::TransitionsAccessor_HasSimpleTransitionTo(isolate(), parent, dead_target, &no_gc_obviously)) { - ClearPotentialSimpleMapTransition(parent, dead_target); - } - } - } - - void ClearPotentialSimpleMapTransition(i::Map map, i::Map dead_target) { - DCHECK(!map.is_prototype_map()); - DCHECK(!dead_target.is_prototype_map()); - DCHECK_EQ(map.raw_transitions(), i::HeapObjectReference::Weak(dead_target)); - // Take ownership of the descriptor array. - int number_of_own_descriptors = map.NumberOfOwnDescriptors(); - auto descriptors = map.instance_descriptors(isolate()); - if (descriptors == dead_target.instance_descriptors(isolate()) && number_of_own_descriptors > 0) { - TrimDescriptorArray(map, descriptors); - DCHECK(descriptors.number_of_descriptors() == number_of_own_descriptors); - } - } - - void ClearJSWeakRefs() { - i::JSWeakRef weak_ref; - while (weak_objects_.js_weak_refs.Pop(kMainThreadTask, &weak_ref)) { - auto target = i::HeapObject::cast(weak_ref.target()); - if (!is_live(target)) { - weak_ref.set_target(i::ReadOnlyRoots(isolate()).undefined_value()); - } else { - auto forwarded = get_forwarded_ref(weak_ref.target()); - if (forwarded) weak_ref.set_target(*forwarded); - } - } - i::WeakCell weak_cell; - while (weak_objects_.weak_cells.Pop(kMainThreadTask, &weak_cell)) { - auto gc_notify_updated_slot = [](i::HeapObject object, i::ObjectSlot slot, i::Object target) { - // if (target.IsHeapObject()) { - // RecordSlot(object, slot, HeapObject::cast(target)); - // } - }; - auto target = i::HeapObject::cast(weak_cell.target()); - if (!is_live(target)) { - DCHECK(!target.IsUndefined()); - // The value of the WeakCell is dead. - auto finalization_registry = i::JSFinalizationRegistry::cast(weak_cell.finalization_registry()); - if (auto f = get_forwarded_ref(finalization_registry)) finalization_registry = i::JSFinalizationRegistry::cast(*f); - DCHECK(is_live(finalization_registry)); - if (!finalization_registry.scheduled_for_cleanup()) { - heap()->EnqueueDirtyJSFinalizationRegistry(finalization_registry, gc_notify_updated_slot); - } - // We're modifying the pointers in WeakCell and JSFinalizationRegistry - // during GC; thus we need to record the slots it writes. The normal write - // barrier is not enough, since it's disabled before GC. - weak_cell.Nullify(isolate(), gc_notify_updated_slot); - DCHECK(finalization_registry.NeedsCleanup()); - DCHECK(finalization_registry.scheduled_for_cleanup()); - } else { - i::ObjectSlot slot = weak_cell.RawField(i::WeakCell::kTargetOffset); - if (auto f = get_forwarded_ref(target)) { - slot.store(*f); - } - } - - i::ObjectSlot slot = weak_cell.RawField(i::WeakCell::kUnregisterTokenOffset); - i::HeapObject unregister_token = i::HeapObject::cast(*slot);//weak_cell.unregister_token(); - if (!is_live(unregister_token)) { - // The unregister token is dead. Remove any corresponding entries in the - // key map. Multiple WeakCell with the same token will have all their - // unregister_token field set to undefined when processing the first - // WeakCell. Like above, we're modifying pointers during GC, so record the - // slots. - auto undefined = i::ReadOnlyRoots(isolate()).undefined_value(); - auto finalization_registry = i::JSFinalizationRegistry::cast(weak_cell.finalization_registry()); - finalization_registry.RemoveUnregisterToken( - i::JSReceiver::cast(unregister_token), isolate(), - [undefined](i::WeakCell matched_cell) { - matched_cell.set_unregister_token(undefined); - }, - gc_notify_updated_slot); - } else { - if (auto f = get_forwarded_ref(unregister_token)) { - slot.store(*f); - } - } - } - heap()->PostFinalizationRegistryCleanupTaskIfNeeded(); - } - - void MarkDependentCodeForDeoptimization() { - std::pair weak_object_in_code; - while (weak_objects_.weak_objects_in_code.Pop(kMainThreadTask, &weak_object_in_code)) { - auto object = weak_object_in_code.first; - auto code = weak_object_in_code.second; - auto object_is_live = is_live(object); - if (!object_is_live && !code.embedded_objects_cleared()) { - if (!code.marked_for_deoptimization()) { - code.SetMarkedForDeoptimization("weak objects"); - have_code_to_deoptimize_ = true; - } - code.ClearEmbeddedObjects(heap()); - DCHECK(code.embedded_objects_cleared()); - } else if (object_is_live) { - auto f = mmtk::get_forwarded_ref(object); - if (f) { - int mode_mask = i::RelocInfo::EmbeddedObjectModeMask() | (1 << i::RelocInfo::CODE_TARGET); - for (i::RelocIterator it(code, mode_mask); !it.done(); it.next()) { - DCHECK(i::RelocInfo::IsEmbeddedObjectMode(it.rinfo()->rmode())); - if (it.rinfo()->target_object() == object) - it.rinfo()->set_target_object(heap(), *f, i::SKIP_WRITE_BARRIER); - } - } - } - } - } + i::WeakObjects weak_objects_; bool have_code_to_deoptimize_ = false; - void Flush() { - for (int i = 0; i < 8; i++) { - weak_objects_.transition_arrays.FlushToGlobal(i); - weak_objects_.ephemeron_hash_tables.FlushToGlobal(i); - weak_objects_.current_ephemerons.FlushToGlobal(i); - weak_objects_.next_ephemerons.FlushToGlobal(i); - weak_objects_.discovered_ephemerons.FlushToGlobal(i); - weak_objects_.weak_references.FlushToGlobal(i); - weak_objects_.js_weak_refs.FlushToGlobal(i); - weak_objects_.weak_cells.FlushToGlobal(i); - weak_objects_.weak_objects_in_code.FlushToGlobal(i); - weak_objects_.bytecode_flushing_candidates.FlushToGlobal(i); - weak_objects_.flushed_js_functions.FlushToGlobal(i); - } - } - - void ProcessEphemeron(i::Ephemeron ephemeron) { - if (is_live(ephemeron.key)) { - if (!is_live(ephemeron.value)) { - trace_((void*) &ephemeron.value); - } - } else { - weak_objects_.next_ephemerons.Push(kMainThreadTask, ephemeron); - } - } - public: - - void ProcessEphemerons() { - Flush(); - - i::Ephemeron ephemeron; - - DCHECK(weak_objects_.current_ephemerons.IsEmpty()); - weak_objects_.current_ephemerons.Swap(weak_objects_.next_ephemerons); - while (weak_objects_.current_ephemerons.Pop(kMainThreadTask, &ephemeron)) { - ProcessEphemeron(ephemeron); - } - while (weak_objects_.discovered_ephemerons.Pop(kMainThreadTask, &ephemeron)) { - ProcessEphemeron(ephemeron); - } + // Do nothing at the moment. + // TODO: Fix this } - std::function trace_ = [](void*) { UNREACHABLE(); }; - static i::Isolate* isolate() { return heap()->isolate(); } @@ -583,52 +66,15 @@ class WeakRefs { } void ClearNonLiveReferences() { - Flush(); have_code_to_deoptimize_ = false; - { - // Prune the string table removing all strings only pointed to by the - // string table. Cannot use string_table() here because the string - // table is marked. - v8::internal::StringTable* string_table = heap()->isolate()->string_table(); - InternalizedStringTableCleaner internalized_visitor(heap()); - string_table->DropOldData(); - string_table->IterateElements(&internalized_visitor); - string_table->NotifyElementsRemoved(internalized_visitor.PointersRemoved()); - - ExternalStringTableCleaner external_visitor(heap()); - tph::Impl::UpdateExternalStringTable(heap(), &external_visitor); - } - ClearOldBytecodeCandidates(); - ClearFlushedJsFunctions(); { MMTkWeakObjectRetainer retainer; tph::Impl::ProcessAllWeakReferences(heap(), &retainer); } - ClearFullMapTransitions(); - ClearWeakReferences(); - ClearWeakCollections(); - ClearJSWeakRefs(); - - PROFILE(heap()->isolate(), WeakCodeClearEvent()); - - MarkDependentCodeForDeoptimization(); - - DCHECK(weak_objects_.transition_arrays.IsEmpty()); - DCHECK(weak_objects_.weak_references.IsEmpty()); - DCHECK(weak_objects_.weak_objects_in_code.IsEmpty()); - DCHECK(weak_objects_.js_weak_refs.IsEmpty()); - DCHECK(weak_objects_.weak_cells.IsEmpty()); - DCHECK(weak_objects_.bytecode_flushing_candidates.IsEmpty()); - DCHECK(weak_objects_.flushed_js_functions.IsEmpty()); - DCHECK(weak_objects_.discovered_ephemerons.IsEmpty()); - // DCHECK(weak_objects_.next_ephemerons.IsEmpty()); - - if (have_code_to_deoptimize_) { - // Some code objects were marked for deoptimization during the GC. - i::Deoptimizer::DeoptimizeMarkedCode(isolate()); - have_code_to_deoptimize_ = false; - } + DCHECK(!have_code_to_deoptimize_); } + + std::function trace_ = [](void*) { UNREACHABLE(); }; }; WeakRefs* global_weakref_processor = new WeakRefs(); From 8346d29a783c10c09fd8e97c5429ecd7d2718d75 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 25 Nov 2021 13:40:43 +1100 Subject: [PATCH 65/65] minor update --- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4bd77e6..6e72426 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -27,5 +27,5 @@ mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "8e50e2c3fa8b45f7e # mmtk = { path = "../repos/mmtk-core" } [features] -default = ["mmtk/code_space", "mmtk/ro_space", "mmtk/nogc_common_plan"] +default = ["mmtk/code_space", "mmtk/ro_space"] nogc = [] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 8b2c94d..b8b41fd 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -49,7 +49,7 @@ pub extern "C" fn v8_new_heap(calls: *const V8_Upcalls, heap_size: usize) -> *mu }; let mmtk: *const MMTK = &*crate::SINGLETON; memory_manager::gc_init(unsafe { &mut *(mmtk as *mut MMTK) }, heap_size); - enable_collection(unsafe { &mut *(mmtk as *mut MMTK) }, VMThread::UNINITIALIZED); + initialize_collection(unsafe { &mut *(mmtk as *mut MMTK) }, VMThread::UNINITIALIZED); mmtk as *mut c_void }