1 /* 2 * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "classfile/classLoaderDataGraph.hpp" 26 #include "classfile/stringTable.hpp" 27 #include "classfile/systemDictionary.hpp" 28 #include "code/codeCache.hpp" 29 #include "gc/epsilon/epsilonHeap.hpp" 30 #include "gc/epsilon/epsilonMemoryPool.hpp" 31 #include "gc/epsilon/epsilonThreadLocalData.hpp" 32 #include "gc/shared/gcArguments.hpp" 33 #include "gc/shared/barrierSet.inline.hpp" 34 #include "gc/shared/gcTraceTime.inline.hpp" 35 #include "gc/shared/markBitMap.inline.hpp" 36 #include "gc/shared/strongRootsScope.hpp" 37 #include "gc/shared/preservedMarks.inline.hpp" 38 #include "gc/shared/weakProcessor.hpp" 39 #include "memory/allocation.inline.hpp" 40 #include "memory/iterator.inline.hpp" 41 #include "memory/resourceArea.hpp" 42 #include "memory/universe.hpp" 43 #include "oops/compressedOops.inline.hpp" 44 #include "oops/markWord.inline.hpp" 45 #include "runtime/biasedLocking.hpp" 46 #include "runtime/globals.hpp" 47 #include "runtime/objectMonitor.inline.hpp" 48 #include "runtime/thread.hpp" 49 #include "runtime/vmOperations.hpp" 50 #include "runtime/vmThread.hpp" 51 #include "utilities/stack.inline.hpp" 52 #include "services/management.hpp" 53 54 jint EpsilonHeap::initialize() { 55 size_t align = HeapAlignment; 56 size_t init_byte_size = align_up(InitialHeapSize, align); 57 size_t max_byte_size = align_up(MaxHeapSize, align); 58 59 // Initialize backing storage 60 ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, align); 61 _virtual_space.initialize(heap_rs, init_byte_size); 62 63 MemRegion committed_region((HeapWord*)_virtual_space.low(), (HeapWord*)_virtual_space.high()); 64 MemRegion reserved_region((HeapWord*)_virtual_space.low_boundary(), (HeapWord*)_virtual_space.high_boundary()); 65 66 initialize_reserved_region(reserved_region.start(), reserved_region.end()); 67 68 _space = new ContiguousSpace(); 69 _space->initialize(committed_region, /* clear_space = */ true, /* mangle_space = */ true); 70 71 // Precompute hot fields 72 _max_tlab_size = MIN2(CollectedHeap::max_tlab_size(), align_object_size(EpsilonMaxTLABSize / HeapWordSize)); 73 _step_counter_update = MIN2<size_t>(max_byte_size / 16, EpsilonUpdateCountersStep); 74 _step_heap_print = (EpsilonPrintHeapSteps == 0) ? SIZE_MAX : (max_byte_size / EpsilonPrintHeapSteps); 75 _decay_time_ns = (int64_t) EpsilonTLABDecayTime * NANOSECS_PER_MILLISEC; 76 77 // Enable monitoring 78 _monitoring_support = new EpsilonMonitoringSupport(this); 79 _last_counter_update = 0; 80 _last_heap_print = 0; 81 82 // Install barrier set 83 BarrierSet::set_barrier_set(new EpsilonBarrierSet()); 84 85 size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); 86 size_t _bitmap_size = MarkBitMap::compute_size(heap_rs.size()); 87 _bitmap_size = align_up(_bitmap_size, bitmap_page_size); 88 89 // Initialize marking bitmap, but not commit it yet 90 if (EpsilonSlidingGC) { 91 ReservedSpace bitmap(_bitmap_size, bitmap_page_size); 92 MemTracker::record_virtual_memory_type(bitmap.base(), mtGC); 93 _bitmap_region = MemRegion((HeapWord *) bitmap.base(), bitmap.size() / HeapWordSize); 94 MemRegion heap_region = MemRegion((HeapWord *) heap_rs.base(), heap_rs.size() / HeapWordSize); 95 _bitmap.initialize(heap_region, _bitmap_region); 96 } 97 98 // All done, print out the configuration 99 if (init_byte_size != max_byte_size) { 100 log_info(gc)("Resizeable heap; starting at " SIZE_FORMAT "M, max: " SIZE_FORMAT "M, step: " SIZE_FORMAT "M", 101 init_byte_size / M, max_byte_size / M, EpsilonMinHeapExpand / M); 102 } else { 103 log_info(gc)("Non-resizeable heap; start/max: " SIZE_FORMAT "M", init_byte_size / M); 104 } 105 106 if (UseTLAB) { 107 log_info(gc)("Using TLAB allocation; max: " SIZE_FORMAT "K", _max_tlab_size * HeapWordSize / K); 108 if (EpsilonElasticTLAB) { 109 log_info(gc)("Elastic TLABs enabled; elasticity: %.2fx", EpsilonTLABElasticity); 110 } 111 if (EpsilonElasticTLABDecay) { 112 log_info(gc)("Elastic TLABs decay enabled; decay time: " SIZE_FORMAT "ms", EpsilonTLABDecayTime); 113 } 114 } else { 115 log_info(gc)("Not using TLAB allocation"); 116 } 117 118 return JNI_OK; 119 } 120 121 void EpsilonHeap::post_initialize() { 122 CollectedHeap::post_initialize(); 123 } 124 125 void EpsilonHeap::initialize_serviceability() { 126 _pool = new EpsilonMemoryPool(this); 127 _memory_manager.add_pool(_pool); 128 } 129 130 GrowableArray<GCMemoryManager*> EpsilonHeap::memory_managers() { 131 GrowableArray<GCMemoryManager*> memory_managers(1); 132 memory_managers.append(&_memory_manager); 133 return memory_managers; 134 } 135 136 GrowableArray<MemoryPool*> EpsilonHeap::memory_pools() { 137 GrowableArray<MemoryPool*> memory_pools(1); 138 memory_pools.append(_pool); 139 return memory_pools; 140 } 141 142 size_t EpsilonHeap::unsafe_max_tlab_alloc(Thread* thr) const { 143 // Return max allocatable TLAB size, and let allocation path figure out 144 // the actual TLAB allocation size. 145 return _max_tlab_size; 146 } 147 148 EpsilonHeap* EpsilonHeap::heap() { 149 CollectedHeap* heap = Universe::heap(); 150 assert(heap != NULL, "Uninitialized access to EpsilonHeap::heap()"); 151 assert(heap->kind() == CollectedHeap::Epsilon, "Not an Epsilon heap"); 152 return (EpsilonHeap*)heap; 153 } 154 155 HeapWord* EpsilonHeap::allocate_work(size_t size) { 156 assert(is_object_aligned(size), "Allocation size should be aligned: " SIZE_FORMAT, size); 157 158 HeapWord* res = _space->par_allocate(size); 159 160 while (res == NULL) { 161 // Allocation failed, attempt expansion, and retry: 162 MutexLocker ml(Heap_lock); 163 164 size_t space_left = max_capacity() - capacity(); 165 size_t want_space = MAX2(size, EpsilonMinHeapExpand); 166 167 if (want_space < space_left) { 168 // Enough space to expand in bulk: 169 bool expand = _virtual_space.expand_by(want_space); 170 assert(expand, "Should be able to expand"); 171 } else if (size < space_left) { 172 // No space to expand in bulk, and this allocation is still possible, 173 // take all the remaining space: 174 bool expand = _virtual_space.expand_by(space_left); 175 assert(expand, "Should be able to expand"); 176 } else { 177 // No space left: 178 return NULL; 179 } 180 181 _space->set_end((HeapWord *) _virtual_space.high()); 182 res = _space->par_allocate(size); 183 } 184 185 size_t used = _space->used(); 186 187 // Allocation successful, update counters 188 { 189 size_t last = _last_counter_update; 190 if ((used - last >= _step_counter_update) && Atomic::cmpxchg(used, &_last_counter_update, last) == last) { 191 _monitoring_support->update_counters(); 192 } 193 } 194 195 // ...and print the occupancy line, if needed 196 { 197 size_t last = _last_heap_print; 198 if ((used - last >= _step_heap_print) && Atomic::cmpxchg(used, &_last_heap_print, last) == last) { 199 print_heap_info(used); 200 print_metaspace_info(); 201 } 202 } 203 204 assert(is_object_aligned(res), "Object should be aligned: " PTR_FORMAT, p2i(res)); 205 return res; 206 } 207 208 HeapWord* EpsilonHeap::allocate_new_tlab(size_t min_size, 209 size_t requested_size, 210 size_t* actual_size) { 211 Thread* thread = Thread::current(); 212 213 // Defaults in case elastic paths are not taken 214 bool fits = true; 215 size_t size = requested_size; 216 size_t ergo_tlab = requested_size; 217 int64_t time = 0; 218 219 if (EpsilonElasticTLAB) { 220 ergo_tlab = EpsilonThreadLocalData::ergo_tlab_size(thread); 221 222 if (EpsilonElasticTLABDecay) { 223 int64_t last_time = EpsilonThreadLocalData::last_tlab_time(thread); 224 time = (int64_t) os::javaTimeNanos(); 225 226 assert(last_time <= time, "time should be monotonic"); 227 228 // If the thread had not allocated recently, retract the ergonomic size. 229 // This conserves memory when the thread had initial burst of allocations, 230 // and then started allocating only sporadically. 231 if (last_time != 0 && (time - last_time > _decay_time_ns)) { 232 ergo_tlab = 0; 233 EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0); 234 } 235 } 236 237 // If we can fit the allocation under current TLAB size, do so. 238 // Otherwise, we want to elastically increase the TLAB size. 239 fits = (requested_size <= ergo_tlab); 240 if (!fits) { 241 size = (size_t) (ergo_tlab * EpsilonTLABElasticity); 242 } 243 } 244 245 // Always honor boundaries 246 size = MAX2(min_size, MIN2(_max_tlab_size, size)); 247 248 // Always honor alignment 249 size = align_up(size, MinObjAlignment); 250 251 // Check that adjustments did not break local and global invariants 252 assert(is_object_aligned(size), 253 "Size honors object alignment: " SIZE_FORMAT, size); 254 assert(min_size <= size, 255 "Size honors min size: " SIZE_FORMAT " <= " SIZE_FORMAT, min_size, size); 256 assert(size <= _max_tlab_size, 257 "Size honors max size: " SIZE_FORMAT " <= " SIZE_FORMAT, size, _max_tlab_size); 258 assert(size <= CollectedHeap::max_tlab_size(), 259 "Size honors global max size: " SIZE_FORMAT " <= " SIZE_FORMAT, size, CollectedHeap::max_tlab_size()); 260 261 if (log_is_enabled(Trace, gc)) { 262 ResourceMark rm; 263 log_trace(gc)("TLAB size for \"%s\" (Requested: " SIZE_FORMAT "K, Min: " SIZE_FORMAT 264 "K, Max: " SIZE_FORMAT "K, Ergo: " SIZE_FORMAT "K) -> " SIZE_FORMAT "K", 265 thread->name(), 266 requested_size * HeapWordSize / K, 267 min_size * HeapWordSize / K, 268 _max_tlab_size * HeapWordSize / K, 269 ergo_tlab * HeapWordSize / K, 270 size * HeapWordSize / K); 271 } 272 273 // All prepared, let's do it! 274 HeapWord* res = allocate_or_collect_work(size); 275 276 if (res != NULL) { 277 // Allocation successful 278 *actual_size = size; 279 if (EpsilonElasticTLABDecay) { 280 EpsilonThreadLocalData::set_last_tlab_time(thread, time); 281 } 282 if (EpsilonElasticTLAB && !fits) { 283 // If we requested expansion, this is our new ergonomic TLAB size 284 EpsilonThreadLocalData::set_ergo_tlab_size(thread, size); 285 } 286 } else { 287 // Allocation failed, reset ergonomics to try and fit smaller TLABs 288 if (EpsilonElasticTLAB) { 289 EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0); 290 } 291 } 292 293 return res; 294 } 295 296 HeapWord* EpsilonHeap::mem_allocate(size_t size, bool *gc_overhead_limit_was_exceeded) { 297 *gc_overhead_limit_was_exceeded = false; 298 return allocate_or_collect_work(size); 299 } 300 301 void EpsilonHeap::collect(GCCause::Cause cause) { 302 switch (cause) { 303 case GCCause::_metadata_GC_threshold: 304 case GCCause::_metadata_GC_clear_soft_refs: 305 // Receiving these causes means the VM itself entered the safepoint for metadata collection. 306 // While Epsilon does not do GC, it has to perform sizing adjustments, otherwise we would 307 // re-enter the safepoint again very soon. 308 309 assert(SafepointSynchronize::is_at_safepoint(), "Expected at safepoint"); 310 log_info(gc)("GC request for \"%s\" is handled", GCCause::to_string(cause)); 311 MetaspaceGC::compute_new_size(); 312 print_metaspace_info(); 313 break; 314 default: 315 if (EpsilonSlidingGC) { 316 if (SafepointSynchronize::is_at_safepoint()) { 317 entry_collect(cause); 318 } else { 319 vmentry_collect(cause); 320 } 321 } else { 322 log_info(gc)("GC request for \"%s\" is ignored", GCCause::to_string(cause)); 323 } 324 } 325 _monitoring_support->update_counters(); 326 } 327 328 void EpsilonHeap::do_full_collection(bool clear_all_soft_refs) { 329 collect(gc_cause()); 330 } 331 332 void EpsilonHeap::safe_object_iterate(ObjectClosure *cl) { 333 _space->safe_object_iterate(cl); 334 } 335 336 void EpsilonHeap::print_on(outputStream *st) const { 337 st->print_cr("Epsilon Heap"); 338 339 // Cast away constness: 340 ((VirtualSpace)_virtual_space).print_on(st); 341 342 st->print_cr("Allocation space:"); 343 _space->print_on(st); 344 345 MetaspaceUtils::print_on(st); 346 } 347 348 void EpsilonHeap::print_tracing_info() const { 349 print_heap_info(used()); 350 print_metaspace_info(); 351 } 352 353 void EpsilonHeap::print_heap_info(size_t used) const { 354 size_t reserved = max_capacity(); 355 size_t committed = capacity(); 356 357 if (reserved != 0) { 358 log_info(gc)("Heap: " SIZE_FORMAT "%s reserved, " SIZE_FORMAT "%s (%.2f%%) committed, " 359 SIZE_FORMAT "%s (%.2f%%) used", 360 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved), 361 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed), 362 committed * 100.0 / reserved, 363 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used), 364 used * 100.0 / reserved); 365 } else { 366 log_info(gc)("Heap: no reliable data"); 367 } 368 } 369 370 void EpsilonHeap::print_metaspace_info() const { 371 size_t reserved = MetaspaceUtils::reserved_bytes(); 372 size_t committed = MetaspaceUtils::committed_bytes(); 373 size_t used = MetaspaceUtils::used_bytes(); 374 375 if (reserved != 0) { 376 log_info(gc, metaspace)("Metaspace: " SIZE_FORMAT "%s reserved, " SIZE_FORMAT "%s (%.2f%%) committed, " 377 SIZE_FORMAT "%s (%.2f%%) used", 378 byte_size_in_proper_unit(reserved), proper_unit_for_byte_size(reserved), 379 byte_size_in_proper_unit(committed), proper_unit_for_byte_size(committed), 380 committed * 100.0 / reserved, 381 byte_size_in_proper_unit(used), proper_unit_for_byte_size(used), 382 used * 100.0 / reserved); 383 } else { 384 log_info(gc, metaspace)("Metaspace: no reliable data"); 385 } 386 } 387 388 // ------------------ EXPERIMENTAL MARK-COMPACT ------------------------------- 389 // 390 // This implements a trivial Lisp2-style sliding collector: 391 // https://en.wikipedia.org/wiki/Mark-compact_algorithm#LISP2_algorithm 392 // 393 // The goal for this implementation is to be as simple as possible, ignoring 394 // non-trivial performance optimizations. This collector does not implement 395 // reference processing: no soft/weak/phantom/finalizeable references are ever 396 // cleared. It also does not implement class unloading and other runtime 397 // cleanups. 398 // 399 400 // VM operation that executes collection cycle under safepoint 401 class VM_EpsilonCollect: public VM_Operation { 402 private: 403 const GCCause::Cause _cause; 404 EpsilonHeap* const _heap; 405 static size_t _last_used; 406 public: 407 VM_EpsilonCollect(GCCause::Cause cause) : VM_Operation(), 408 _cause(cause), 409 _heap(EpsilonHeap::heap()) {}; 410 411 VM_Operation::VMOp_Type type() const { return VMOp_EpsilonCollect; } 412 const char* name() const { return "Epsilon Collection"; } 413 414 virtual bool doit_prologue() { 415 // Need to take the Heap lock before managing backing storage. 416 // This also naturally serializes GC requests, and allows us to coalesce 417 // back-to-back allocation failure requests from many threads. There is no 418 // need to handle allocation failure that comes without allocations since 419 // last complete GC. Waiting for 1% of heap allocated before starting next 420 // GC seems to resolve most races. 421 Heap_lock->lock(); 422 size_t used = _heap->used(); 423 size_t capacity = _heap->capacity(); 424 size_t allocated = used > _last_used ? used - _last_used : 0; 425 if (_cause != GCCause::_allocation_failure || allocated > capacity / 100) { 426 return true; 427 } else { 428 Heap_lock->unlock(); 429 return false; 430 } 431 } 432 433 virtual void doit() { 434 _heap->entry_collect(_cause); 435 } 436 437 virtual void doit_epilogue() { 438 _last_used = _heap->used(); 439 Heap_lock->unlock(); 440 } 441 }; 442 443 size_t VM_EpsilonCollect::_last_used = 0; 444 445 void EpsilonHeap::vmentry_collect(GCCause::Cause cause) { 446 VM_EpsilonCollect vmop(cause); 447 VMThread::execute(&vmop); 448 } 449 450 HeapWord* EpsilonHeap::allocate_or_collect_work(size_t size) { 451 HeapWord* res = allocate_work(size); 452 if (res == NULL && EpsilonSlidingGC) { 453 vmentry_collect(GCCause::_allocation_failure); 454 res = allocate_work(size); 455 } 456 return res; 457 } 458 459 typedef Stack<oop, mtGC> EpsilonMarkStack; 460 461 void EpsilonHeap::do_roots(OopClosure* cl, bool everything) { 462 // Need to tell runtime we are about to walk the roots with 1 thread 463 StrongRootsScope scope(1); 464 465 // Need to adapt oop closure for some special root types. 466 CLDToOopClosure clds(cl, ClassLoaderData::_claim_none); 467 MarkingCodeBlobClosure blobs(cl, CodeBlobToOopClosure::FixRelocations); 468 469 // Walk all these different parts of runtime roots. Some roots require 470 // holding the lock when walking them. 471 { 472 MutexLocker lock(CodeCache_lock, Mutex::_no_safepoint_check_flag); 473 CodeCache::blobs_do(&blobs); 474 } 475 { 476 MutexLocker lock(ClassLoaderDataGraph_lock); 477 ClassLoaderDataGraph::cld_do(&clds); 478 } 479 Universe::oops_do(cl); 480 Management::oops_do(cl); 481 JvmtiExport::oops_do(cl); 482 JNIHandles::oops_do(cl); 483 WeakProcessor::oops_do(cl); 484 ObjectSynchronizer::oops_do(cl); 485 SystemDictionary::oops_do(cl); 486 Threads::possibly_parallel_oops_do(false, cl, &blobs); 487 488 // This is implicitly handled by other roots, and we only want to 489 // touch these during verification. 490 if (everything) { 491 StringTable::oops_do(cl); 492 } 493 } 494 495 // Walk the marking bitmap and call object closure on every marked object. 496 // This is much faster that walking a (very sparse) parsable heap, but it 497 // takes up to 1/64-th of heap size for the bitmap. 498 void EpsilonHeap::walk_bitmap(ObjectClosure* cl) { 499 HeapWord* limit = _space->top(); 500 HeapWord* addr = _bitmap.get_next_marked_addr(_space->bottom(), limit); 501 while (addr < limit) { 502 oop obj = oop(addr); 503 assert(_bitmap.is_marked(obj), "sanity"); 504 cl->do_object(obj); 505 addr += 1; 506 if (addr < limit) { 507 addr = _bitmap.get_next_marked_addr(addr, limit); 508 } 509 } 510 } 511 512 class EpsilonScanOopClosure : public BasicOopIterateClosure { 513 private: 514 EpsilonMarkStack* const _stack; 515 MarkBitMap* const _bitmap; 516 517 template <class T> 518 void do_oop_work(T* p) { 519 // p is the pointer to memory location where oop is, load the value 520 // from it, unpack the compressed reference, if needed: 521 T o = RawAccess<>::oop_load(p); 522 if (!CompressedOops::is_null(o)) { 523 oop obj = CompressedOops::decode_not_null(o); 524 525 // Object is discovered. See if it is marked already. If not, 526 // mark and push it on mark stack for further traversal. Non-atomic 527 // check and set would do, as this closure is called by single thread. 528 if (!_bitmap->is_marked(obj)) { 529 _bitmap->mark((HeapWord*)obj); 530 _stack->push(obj); 531 } 532 } 533 } 534 535 public: 536 EpsilonScanOopClosure(EpsilonMarkStack* stack, MarkBitMap* bitmap) : 537 _stack(stack), _bitmap(bitmap) {} 538 virtual void do_oop(oop* p) { do_oop_work(p); } 539 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 540 }; 541 542 class EpsilonCalcNewLocationObjectClosure : public ObjectClosure { 543 private: 544 HeapWord* _compact_point; 545 PreservedMarks* const _preserved_marks; 546 547 public: 548 EpsilonCalcNewLocationObjectClosure(HeapWord* start, PreservedMarks* pm) : 549 _compact_point(start), 550 _preserved_marks(pm) {} 551 552 void do_object(oop obj) { 553 // Record the new location of the object: it is current compaction point. 554 // If object stays at the same location (which is true for objects in 555 // dense prefix, that we would normally get), do not bother recording the 556 // move, letting downstream code ignore it. 557 if ((HeapWord*)obj != _compact_point) { 558 markWord mark = obj->mark_raw(); 559 if (mark.must_be_preserved(obj->klass())) { 560 _preserved_marks->push(obj, mark); 561 } 562 obj->forward_to(oop(_compact_point)); 563 } 564 _compact_point += obj->size(); 565 } 566 567 HeapWord* compact_point() { 568 return _compact_point; 569 } 570 }; 571 572 class EpsilonAdjustPointersOopClosure : public BasicOopIterateClosure { 573 private: 574 template <class T> 575 void do_oop_work(T* p) { 576 // p is the pointer to memory location where oop is, load the value 577 // from it, unpack the compressed reference, if needed: 578 T o = RawAccess<>::oop_load(p); 579 if (!CompressedOops::is_null(o)) { 580 oop obj = CompressedOops::decode_not_null(o); 581 582 // Rewrite the current pointer to the object with its forwardee. 583 // Skip the write if update is not needed. 584 if (obj->is_forwarded()) { 585 oop fwd = obj->forwardee(); 586 assert(fwd != NULL, "just checking"); 587 RawAccess<>::oop_store(p, fwd); 588 } 589 } 590 } 591 592 public: 593 virtual void do_oop(oop* p) { do_oop_work(p); } 594 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 595 }; 596 597 class EpsilonAdjustPointersObjectClosure : public ObjectClosure { 598 private: 599 EpsilonAdjustPointersOopClosure _cl; 600 public: 601 void do_object(oop obj) { 602 // Apply the updates to all references reachable from current object: 603 obj->oop_iterate(&_cl); 604 } 605 }; 606 607 class EpsilonMoveObjectsObjectClosure : public ObjectClosure { 608 private: 609 size_t _moved; 610 public: 611 EpsilonMoveObjectsObjectClosure() : ObjectClosure(), _moved(0) {} 612 613 void do_object(oop obj) { 614 // Copy the object to its new location, if needed. This is final step, 615 // so we have to re-initialize its new mark word, dropping the forwardee 616 // data from it. 617 if (obj->is_forwarded()) { 618 oop fwd = obj->forwardee(); 619 assert(fwd != NULL, "just checking"); 620 Copy::aligned_conjoint_words((HeapWord*)obj, (HeapWord*)fwd, obj->size()); 621 fwd->init_mark_raw(); 622 _moved++; 623 } 624 } 625 626 size_t moved() { 627 return _moved; 628 } 629 }; 630 631 class EpsilonVerifyOopClosure : public BasicOopIterateClosure { 632 private: 633 EpsilonHeap* const _heap; 634 EpsilonMarkStack* const _stack; 635 MarkBitMap* const _bitmap; 636 637 template <class T> 638 void do_oop_work(T* p) { 639 T o = RawAccess<>::oop_load(p); 640 if (!CompressedOops::is_null(o)) { 641 oop obj = CompressedOops::decode_not_null(o); 642 if (!_bitmap->is_marked(obj)) { 643 _bitmap->mark((HeapWord*)obj); 644 645 guarantee(_heap->is_in(obj), "Is in heap: " PTR_FORMAT, p2i(obj)); 646 guarantee(oopDesc::is_oop(obj), "Is an object: " PTR_FORMAT, p2i(obj)); 647 guarantee(!obj->mark().is_marked(), "Mark is gone: " PTR_FORMAT, p2i(obj)); 648 649 _stack->push(obj); 650 } 651 } 652 } 653 654 public: 655 EpsilonVerifyOopClosure(EpsilonMarkStack* stack, MarkBitMap* bitmap) : 656 _heap(EpsilonHeap::heap()), _stack(stack), _bitmap(bitmap) {} 657 virtual void do_oop(oop* p) { do_oop_work(p); } 658 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 659 }; 660 661 void EpsilonHeap::entry_collect(GCCause::Cause cause) { 662 GCIdMark mark; 663 GCTraceTime(Info, gc) time("Lisp2-style Mark-Compact", NULL, cause, true); 664 665 // Some statistics, for fun and profit: 666 size_t stat_reachable_roots = 0; 667 size_t stat_reachable_heap = 0; 668 size_t stat_moved = 0; 669 size_t stat_preserved_marks = 0; 670 671 { 672 GCTraceTime(Info, gc) time("Step 0: Prologue", NULL); 673 674 // Commit marking bitmap memory. There are several upsides of doing this 675 // before the cycle: no memory is taken if GC is not happening, the memory 676 // is "cleared" on first touch, and untouched parts of bitmap are mapped 677 // to zero page, boosting performance on sparse heaps. 678 if (!os::commit_memory((char*)_bitmap_region.start(), _bitmap_region.byte_size(), false)) { 679 log_warning(gc)("Could not commit native memory for marking bitmap, GC failed"); 680 return; 681 } 682 683 // We do not need parsable heap for this algorithm to work, but we want 684 // threads to give up their TLABs. 685 ensure_parsability(true); 686 687 // Tell various parts of runtime we are doing GC. 688 BiasedLocking::preserve_marks(); 689 690 // Derived pointers would be re-discovered during the mark. 691 // Clear and activate the table for them. 692 DerivedPointerTable::clear(); 693 } 694 695 { 696 GCTraceTime(Info, gc) time("Step 1: Mark", NULL); 697 698 // Marking stack and the closure that does most of the work. The closure 699 // would scan the outgoing references, mark them, and push newly-marked 700 // objects to stack for further processing. 701 EpsilonMarkStack stack; 702 EpsilonScanOopClosure cl(&stack, &_bitmap); 703 704 // Seed the marking with roots. 705 process_roots(&cl); 706 stat_reachable_roots = stack.size(); 707 708 // Scan the rest of the heap until we run out of objects. Termination is 709 // guaranteed, because all reachable objects would be marked eventually. 710 while (!stack.is_empty()) { 711 oop obj = stack.pop(); 712 obj->oop_iterate(&cl); 713 stat_reachable_heap++; 714 } 715 716 // No more derived pointers discovered after marking is done. 717 DerivedPointerTable::set_active(false); 718 } 719 720 // We are going to store forwarding information (where the new copy resides) 721 // in mark words. Some of those mark words need to be carefully preserved. 722 // This is an utility that maintains the list of those special mark words. 723 PreservedMarks preserved_marks; 724 725 // New top of the allocated space. 726 HeapWord* new_top; 727 728 { 729 GCTraceTime(Info, gc) time("Step 2: Calculate new locations", NULL); 730 731 // Walk all alive objects, compute their new addresses and store those 732 // addresses in mark words. Optionally preserve some marks. 733 EpsilonCalcNewLocationObjectClosure cl(_space->bottom(), &preserved_marks); 734 walk_bitmap(&cl); 735 736 // After addresses are calculated, we know the new top for the allocated 737 // space. We cannot set it just yet, because some asserts check that objects 738 // are "in heap" based on current "top". 739 new_top = cl.compact_point(); 740 741 stat_preserved_marks = preserved_marks.size(); 742 } 743 744 { 745 GCTraceTime(Info, gc) time("Step 3: Adjust pointers", NULL); 746 747 // Walk all alive objects _and their reference fields_, and put "new 748 // addresses" there. We know the new addresses from the forwarding data 749 // in mark words. Take care of the heap objects first. 750 EpsilonAdjustPointersObjectClosure cl; 751 walk_bitmap(&cl); 752 753 // Now do the same, but for all VM roots, which reference the objects on 754 // their own: their references should also be updated. 755 EpsilonAdjustPointersOopClosure cli; 756 process_roots(&cli); 757 758 // Finally, make sure preserved marks know the objects are about to move. 759 preserved_marks.adjust_during_full_gc(); 760 } 761 762 { 763 GCTraceTime(Info, gc) time("Step 4: Move objects", NULL); 764 765 // Move all alive objects to their new locations. All the references are 766 // already adjusted at previous step. 767 EpsilonMoveObjectsObjectClosure cl; 768 walk_bitmap(&cl); 769 stat_moved = cl.moved(); 770 771 // Now we moved all objects to their relevant locations, we can retract 772 // the "top" of the allocation space to the end of the compacted prefix. 773 _space->set_top(new_top); 774 } 775 776 { 777 GCTraceTime(Info, gc) time("Step 5: Epilogue", NULL); 778 779 // Restore all special mark words. 780 preserved_marks.restore(); 781 782 // Tell the rest of runtime we have finished the GC. 783 DerivedPointerTable::update_pointers(); 784 BiasedLocking::restore_marks(); 785 786 // Verification code walks entire heap and verifies nothing is broken. 787 if (EpsilonVerify) { 788 // The basic implementation turns heap into entirely parsable one with 789 // only alive objects, which mean we could just walked the heap object 790 // by object and verify it. But, it would be inconvenient for verification 791 // to assume heap has only alive objects. Any future change that leaves 792 // at least one dead object with dead outgoing references would fail the 793 // verification. Therefore, it makes more sense to mark through the heap 794 // again, not assuming objects are all alive. 795 EpsilonMarkStack stack; 796 EpsilonVerifyOopClosure cl(&stack, &_bitmap); 797 798 _bitmap.clear(); 799 800 // Verify all roots are correct, and that we have the same number of 801 // object reachable from roots. 802 process_all_roots(&cl); 803 804 size_t verified_roots = stack.size(); 805 guarantee(verified_roots == stat_reachable_roots, 806 "Verification discovered " SIZE_FORMAT " roots out of " SIZE_FORMAT, 807 verified_roots, stat_reachable_roots); 808 809 // Verify the rest of the heap is correct, and that we have the same 810 // number of objects reachable from heap. 811 size_t verified_heap = 0; 812 while (!stack.is_empty()) { 813 oop obj = stack.pop(); 814 obj->oop_iterate(&cl); 815 verified_heap++; 816 } 817 818 guarantee(verified_heap == stat_reachable_heap, 819 "Verification discovered " SIZE_FORMAT " heap objects out of " SIZE_FORMAT, 820 verified_heap, stat_reachable_heap); 821 822 // Ask parts of runtime to verify themselves too 823 Universe::verify(VerifyOption_Default, ""); 824 } 825 826 // Marking bitmap is not needed anymore 827 if (!os::uncommit_memory((char*)_bitmap_region.start(), _bitmap_region.byte_size())) { 828 log_warning(gc)("Could not uncommit native memory for marking bitmap"); 829 } 830 831 // Return all memory back if so requested. On large heaps, this would 832 // take a while. 833 if (EpsilonUncommit) { 834 _virtual_space.shrink_by((_space->end() - new_top) * HeapWordSize); 835 _space->set_end((HeapWord*)_virtual_space.high()); 836 } 837 } 838 839 size_t stat_reachable = stat_reachable_roots + stat_reachable_heap; 840 log_info(gc)("GC Stats: " SIZE_FORMAT " (%.2f%%) reachable from roots, " SIZE_FORMAT " (%.2f%%) reachable from heap, " 841 SIZE_FORMAT " (%.2f%%) moved, " SIZE_FORMAT " (%.2f%%) markwords preserved", 842 stat_reachable_roots, 100.0 * stat_reachable_roots / stat_reachable, 843 stat_reachable_heap, 100.0 * stat_reachable_heap / stat_reachable, 844 stat_moved, 100.0 * stat_moved / stat_reachable, 845 stat_preserved_marks, 100.0 * stat_preserved_marks / stat_reachable); 846 847 print_heap_info(used()); 848 print_metaspace_info(); 849 }