diff --git a/src/runtime/mgcscavenge.go b/src/runtime/mgcscavenge.go index 4dacfa0a5c..b74da1057a 100644 --- a/src/runtime/mgcscavenge.go +++ b/src/runtime/mgcscavenge.go @@ -452,8 +452,8 @@ func (s *pageAlloc) scavengeStartGen() { s.inUse.cloneInto(&s.scav.inUse) // Pick the new starting address for the scavenger cycle. - var startAddr uintptr - if s.scav.scavLWM < s.scav.freeHWM { + var startAddr offAddr + if s.scav.scavLWM.lessThan(s.scav.freeHWM) { // The "free" high watermark exceeds the "scavenged" low watermark, // so there are free scavengable pages in parts of the address space // that the scavenger already searched, the high watermark being the @@ -467,7 +467,7 @@ func (s *pageAlloc) scavengeStartGen() { // scavenging from where we were. startAddr = s.scav.scavLWM } - s.scav.inUse.removeGreaterEqual(startAddr) + s.scav.inUse.removeGreaterEqual(startAddr.addr()) // reservationBytes may be zero if s.inUse.totalBytes is small, or if // scavengeReservationShards is large. This case is fine as the scavenger @@ -478,8 +478,8 @@ func (s *pageAlloc) scavengeStartGen() { s.scav.reservationBytes = alignUp(s.inUse.totalBytes, pallocChunkBytes) / scavengeReservationShards s.scav.gen++ s.scav.released = 0 - s.scav.freeHWM = 0 - s.scav.scavLWM = maxSearchAddr + s.scav.freeHWM = minOffAddr + s.scav.scavLWM = maxOffAddr } // scavengeReserve reserves a contiguous range of the address space @@ -698,8 +698,8 @@ func (s *pageAlloc) scavengeRangeLocked(ci chunkIdx, base, npages uint) uintptr addr := chunkBase(ci) + uintptr(base)*pageSize // Update the scavenge low watermark. - if addr < s.scav.scavLWM { - s.scav.scavLWM = addr + if oAddr := (offAddr{addr}); oAddr.lessThan(s.scav.scavLWM) { + s.scav.scavLWM = oAddr } // Only perform the actual scavenging if we're not in a test. diff --git a/src/runtime/mpagealloc.go b/src/runtime/mpagealloc.go index 5078738b60..a28dd26cb5 100644 --- a/src/runtime/mpagealloc.go +++ b/src/runtime/mpagealloc.go @@ -81,15 +81,14 @@ const ( // there should this change. pallocChunksL2Bits = heapAddrBits - logPallocChunkBytes - pallocChunksL1Bits pallocChunksL1Shift = pallocChunksL2Bits - - // Maximum searchAddr value, which indicates that the heap has no free space. - // - // We subtract arenaBaseOffset because we want this to represent the maximum - // value in the shifted address space, but searchAddr is stored as a regular - // memory address. See arenaBaseOffset for details. - maxSearchAddr = ^uintptr(0) - arenaBaseOffset ) +// Maximum searchAddr value, which indicates that the heap has no free space. +// +// We alias maxOffAddr just to make it clear that this is the maximum address +// for the page allocator's search space. See maxOffAddr for details. +var maxSearchAddr = maxOffAddr + // Global chunk index. // // Represents an index into the leaf level of the radix tree. @@ -134,6 +133,18 @@ func (i chunkIdx) l2() uint { } } +// offAddrToLevelIndex converts an address in the offset address space +// to the index into summary[level] containing addr. +func offAddrToLevelIndex(level int, addr offAddr) int { + return int((addr.a + arenaBaseOffset) >> levelShift[level]) +} + +// levelIndexToOffAddr converts an index into summary[level] into +// the corresponding address in the offset address space. +func levelIndexToOffAddr(level, idx int) offAddr { + return offAddr{(uintptr(idx) << levelShift[level]) - arenaBaseOffset} +} + // addrsToSummaryRange converts base and limit pointers into a range // of entries for the given summary level. // @@ -232,7 +243,7 @@ type pageAlloc struct { // Note that adding in arenaBaseOffset transforms addresses // to a new address space with a linear view of the full address // space on architectures with segmented address spaces. - searchAddr uintptr + searchAddr offAddr // start and end represent the chunk indices // which pageAlloc knows about. It assumes @@ -271,13 +282,13 @@ type pageAlloc struct { // released is the amount of memory released this generation. released uintptr - // scavLWM is the lowest address that the scavenger reached this + // scavLWM is the lowest (offset) address that the scavenger reached this // scavenge generation. - scavLWM uintptr + scavLWM offAddr - // freeHWM is the highest address of a page that was freed to + // freeHWM is the highest (offset) address of a page that was freed to // the page allocator this scavenge generation. - freeHWM uintptr + freeHWM offAddr } // mheap_.lock. This level of indirection makes it possible @@ -319,29 +330,6 @@ func (s *pageAlloc) init(mheapLock *mutex, sysStat *uint64) { s.scav.scavLWM = maxSearchAddr } -// compareSearchAddrTo compares an address against s.searchAddr in a linearized -// view of the address space on systems with discontinuous process address spaces. -// This linearized view is the same one generated by chunkIndex and arenaIndex, -// done by adding arenaBaseOffset. -// -// On systems without a discontinuous address space, it's just a normal comparison. -// -// Returns < 0 if addr is less than s.searchAddr in the linearized address space. -// Returns > 0 if addr is greater than s.searchAddr in the linearized address space. -// Returns 0 if addr and s.searchAddr are equal. -func (s *pageAlloc) compareSearchAddrTo(addr uintptr) int { - // Compare with arenaBaseOffset added because it gives us a linear, contiguous view - // of the heap on architectures with signed address spaces. - lAddr := addr + arenaBaseOffset - lSearchAddr := s.searchAddr + arenaBaseOffset - if lAddr < lSearchAddr { - return -1 - } else if lAddr > lSearchAddr { - return 1 - } - return 0 -} - // chunkOf returns the chunk at the given chunk index. func (s *pageAlloc) chunkOf(ci chunkIdx) *pallocData { return &s.chunks[ci.l1()][ci.l2()] @@ -378,10 +366,10 @@ func (s *pageAlloc) grow(base, size uintptr) { s.inUse.add(makeAddrRange(base, limit)) // A grow operation is a lot like a free operation, so if our - // chunk ends up below the (linearized) s.searchAddr, update - // s.searchAddr to the new address, just like in free. - if s.compareSearchAddrTo(base) < 0 { - s.searchAddr = base + // chunk ends up below s.searchAddr, update s.searchAddr to the + // new address, just like in free. + if b := (offAddr{base}); b.lessThan(s.searchAddr) { + s.searchAddr = b } // Add entries into chunks, which is sparse, if needed. Then, @@ -545,7 +533,7 @@ func (s *pageAlloc) allocRange(base, npages uintptr) uintptr { // searchAddr returned is invalid and must be ignored. // // s.mheapLock must be held. -func (s *pageAlloc) find(npages uintptr) (uintptr, uintptr) { +func (s *pageAlloc) find(npages uintptr) (uintptr, offAddr) { // Search algorithm. // // This algorithm walks each level l of the radix tree from the root level @@ -585,13 +573,13 @@ func (s *pageAlloc) find(npages uintptr) (uintptr, uintptr) { // firstFree is updated by calling foundFree each time free space in the // heap is discovered. // - // At the end of the search, base-arenaBaseOffset is the best new + // At the end of the search, base.addr() is the best new // searchAddr we could deduce in this search. firstFree := struct { - base, bound uintptr + base, bound offAddr }{ - base: 0, - bound: (1< firstFree.bound) { + firstFree.bound = addr.add(size - 1) + } else if !(addr.add(size-1).lessThan(firstFree.base) || firstFree.bound.lessThan(addr)) { // This range only partially overlaps with the firstFree range, // so throw. - print("runtime: addr = ", hex(addr), ", size = ", size, "\n") - print("runtime: base = ", hex(firstFree.base), ", bound = ", hex(firstFree.bound), "\n") + print("runtime: addr = ", hex(addr.addr()), ", size = ", size, "\n") + print("runtime: base = ", hex(firstFree.base.addr()), ", bound = ", hex(firstFree.bound.addr()), "\n") throw("range partially overlaps") } } @@ -642,7 +630,7 @@ nextLevel: // searchAddr on the previous level or we're on the root leve, in which // case the searchAddr should be the same as i after levelShift. j0 := 0 - if searchIdx := int((s.searchAddr + arenaBaseOffset) >> levelShift[l]); searchIdx&^(entriesPerBlock-1) == i { + if searchIdx := offAddrToLevelIndex(l, s.searchAddr); searchIdx&^(entriesPerBlock-1) == i { j0 = searchIdx & (entriesPerBlock - 1) } @@ -668,7 +656,7 @@ nextLevel: // We've encountered a non-zero summary which means // free memory, so update firstFree. - foundFree(uintptr((i+j)<= uint(npages) { @@ -706,8 +694,8 @@ nextLevel: if size >= uint(npages) { // We found a sufficiently large run of free pages straddling // some boundary, so compute the address and return it. - addr := uintptr(i<= s.end { + if chunkIndex(s.searchAddr.addr()) >= s.end { return 0, 0 } // If npages has a chance of fitting in the chunk where the searchAddr is, // search it directly. - searchAddr := uintptr(0) - if pallocChunkPages-chunkPageIndex(s.searchAddr) >= uint(npages) { + searchAddr := minOffAddr + if pallocChunkPages-chunkPageIndex(s.searchAddr.addr()) >= uint(npages) { // npages is guaranteed to be no greater than pallocChunkPages here. - i := chunkIndex(s.searchAddr) + i := chunkIndex(s.searchAddr.addr()) if max := s.summary[len(s.summary)-1][i].max(); max >= uint(npages) { - j, searchIdx := s.chunkOf(i).find(npages, chunkPageIndex(s.searchAddr)) + j, searchIdx := s.chunkOf(i).find(npages, chunkPageIndex(s.searchAddr.addr())) if j == ^uint(0) { print("runtime: max = ", max, ", npages = ", npages, "\n") - print("runtime: searchIdx = ", chunkPageIndex(s.searchAddr), ", s.searchAddr = ", hex(s.searchAddr), "\n") + print("runtime: searchIdx = ", chunkPageIndex(s.searchAddr.addr()), ", s.searchAddr = ", hex(s.searchAddr.addr()), "\n") throw("bad summary data") } addr = chunkBase(i) + uintptr(j)*pageSize - searchAddr = chunkBase(i) + uintptr(searchIdx)*pageSize + searchAddr = offAddr{chunkBase(i) + uintptr(searchIdx)*pageSize} goto Found } } @@ -807,10 +795,10 @@ Found: // Go ahead and actually mark the bits now that we have an address. scav = s.allocRange(addr, npages) - // If we found a higher (linearized) searchAddr, we know that all the - // heap memory before that searchAddr in a linear address space is + // If we found a higher searchAddr, we know that all the + // heap memory before that searchAddr in an offset address space is // allocated, so bump s.searchAddr up to the new one. - if s.compareSearchAddrTo(searchAddr) > 0 { + if s.searchAddr.lessThan(searchAddr) { s.searchAddr = searchAddr } return addr, scav @@ -820,14 +808,14 @@ Found: // // s.mheapLock must be held. func (s *pageAlloc) free(base, npages uintptr) { - // If we're freeing pages below the (linearized) s.searchAddr, update searchAddr. - if s.compareSearchAddrTo(base) < 0 { - s.searchAddr = base + // If we're freeing pages below the s.searchAddr, update searchAddr. + if b := (offAddr{base}); b.lessThan(s.searchAddr) { + s.searchAddr = b } // Update the free high watermark for the scavenger. limit := base + npages*pageSize - 1 - if s.scav.freeHWM < limit { - s.scav.freeHWM = limit + if offLimit := (offAddr{limit}); s.scav.freeHWM.lessThan(offLimit) { + s.scav.freeHWM = offLimit } if npages == 1 { // Fast path: we're clearing a single bit, and we know exactly diff --git a/src/runtime/mpagecache.go b/src/runtime/mpagecache.go index fae54d7cdd..683a997136 100644 --- a/src/runtime/mpagecache.go +++ b/src/runtime/mpagecache.go @@ -91,8 +91,8 @@ func (c *pageCache) flush(s *pageAlloc) { } // Since this is a lot like a free, we need to make sure // we update the searchAddr just like free does. - if s.compareSearchAddrTo(c.base) < 0 { - s.searchAddr = c.base + if b := (offAddr{c.base}); b.lessThan(s.searchAddr) { + s.searchAddr = b } s.update(c.base, pageCachePages, false, false) *c = pageCache{} @@ -106,15 +106,15 @@ func (c *pageCache) flush(s *pageAlloc) { func (s *pageAlloc) allocToCache() pageCache { // If the searchAddr refers to a region which has a higher address than // any known chunk, then we know we're out of memory. - if chunkIndex(s.searchAddr) >= s.end { + if chunkIndex(s.searchAddr.addr()) >= s.end { return pageCache{} } c := pageCache{} - ci := chunkIndex(s.searchAddr) // chunk index + ci := chunkIndex(s.searchAddr.addr()) // chunk index if s.summary[len(s.summary)-1][ci] != 0 { // Fast path: there's free pages at or near the searchAddr address. chunk := s.chunkOf(ci) - j, _ := chunk.find(1, chunkPageIndex(s.searchAddr)) + j, _ := chunk.find(1, chunkPageIndex(s.searchAddr.addr())) if j == ^uint(0) { throw("bad summary data") } @@ -156,6 +156,6 @@ func (s *pageAlloc) allocToCache() pageCache { // However, s.searchAddr is not allowed to point into unmapped heap memory // unless it is maxSearchAddr, so make it the last page as opposed to // the page after. - s.searchAddr = c.base + pageSize*(pageCachePages-1) + s.searchAddr = offAddr{c.base + pageSize*(pageCachePages-1)} return c } diff --git a/src/runtime/mranges.go b/src/runtime/mranges.go index 468a73057b..e574c2f518 100644 --- a/src/runtime/mranges.go +++ b/src/runtime/mranges.go @@ -69,6 +69,31 @@ func (a addrRange) subtract(b addrRange) addrRange { return a } +var ( + // minOffAddr is the minimum address in the offset space, and + // it corresponds to the virtual address -arenaBaseOffset. + // + // We don't initialize this with offAddrFromRaw because allocation + // may happen during bootstrapping, and we rely on this value + // being initialized. + // + // As a result, creating this value in Go is tricky because of + // overflow not being allowed in constants. In order to get + // the value we want, we take arenaBaseOffset and do a manual + // two's complement negation, then mask that into what can fit + // into a uintptr. + minOffAddr = offAddr{((^arenaBaseOffset) + 1) & uintptrMask} + + // maxOffAddr is the maximum address in the offset address + // space, and it corresponds to the virtual address + // ^uintptr(0) - arenaBaseOffset. + // + // We don't initialize this with offAddrFromRaw because allocation + // may happen during bootstrapping, and we rely on this value + // being initialized. + maxOffAddr = offAddr{^uintptr(0) - arenaBaseOffset} +) + // offAddr represents an address in a contiguous view // of the address space on systems where the address space is // segmented. On other systems, it's just a normal address. @@ -268,7 +293,7 @@ func (a *addrRanges) removeGreaterEqual(addr uintptr) { } if r := a.ranges[pivot-1]; r.contains(addr) { removed += r.size() - r = r.subtract(makeAddrRange(addr, maxSearchAddr)) + r = r.subtract(makeAddrRange(addr, maxOffAddr.addr())) if r.size() == 0 { pivot-- } else {