[Rd] Ncell GC trigger reaches SIZE_MAX after grow/shrink/regrow in AdjustHeapSize

[Ivan Krylov]

В Mon, 5 Oct 2026 01:43:31 -0400
Tyler <tylermw using gmail.com> пишет:

> I found a reproducible node-heap sizing bug in R 4.6.0 on arm64 macOS.
> After allocating and releasing a list, shrinking the heap with gc(),
> and allocating a smaller list, the Ncell GC trigger becomes
> approximately 1.844674e19, although fewer than one million Ncells are
> live.

Thank you for the report! I can also reproduce the R_NSize overflow on
Linux/amd64 when R is compiled using Clang, but not GCC. As you
mention, R builds with -fsanitize=undefined using both GCC and Clang
show a runtime error when the double -> size_t conversion overflows,
even if R_NSize stays fine.

R_size_t next_in_use = R_NodesInUse + (R_NodesInUse - last_in_use);
(gdb) p R_NodesInUse                   
$6 = 992417                            
(gdb) p last_in_use                    
$7 = 2332966                           
(gdb) p /x (R_NodesInUse - last_in_use)
$8 = 0xffffffffffeb8b7b                

R_size_t next_nsize = (R_size_t) (next_in_use / R_NGrowFrac);
// ../../../src/main/memory.c:1204:52: runtime error: 2.63525e+19 is
// outside the range of representable values of type 'long unsigned int'

GCC compiles this as:
                                R_NGrowFrac -> xmm5
                        (double)PTRDIFF_MAX -> xmm1
; convert next_in_use to double accounting for the sign flag set
mov      %rdx,%rdi     
and      $0x1,%edx     
pxor     %xmm0,%xmm0
shr      %rdi          
or       %rdx,%rdi     
cvtsi2sd %rdi,%xmm0  
addsd    %xmm0,%xmm0          ; (double)next_in_use -> xmm0
divsd    %xmm5,%xmm0          ; xmm0 / xmm5 -> xmm0
(gdb) p $xmm0.v2_double[0]
$35 = 2.6352491533870297e+19
comisd   %xmm1,%xmm0          ; result exceeds PTRDIFF_MAX? (yes)
...
subsd    %xmm1,%xmm0          ; xmm0 - xmm1 -> xmm0
cvttsd2si %xmm0,%rdi          ; failed conversion stores (1<<63) in rdi
btc      $0x3f,%rdi           ; flips bit 63 in rdi, leaving 0
                              ; rdi -> next_nsize
(gdb) p /x next_nsize
$51 = 0x0

Clang compiles the same code as:
                                R_NGrowFrac -> xmm3
                                next_in_use -> r9
movq      %r9,%xmm1           ; (double)r9 -> xmm1
punpckldq 0x101164(%rip),%xmm1
subpd     0x10116c(%rip),%xmm1
movapd    %xmm1,%xmm2         ; xmm1 -> xmm2
unpckhpd  %xmm1,%xmm2
addsd     %xmm1,%xmm2
divsd     %xmm3,%xmm2         ; xmm2 / xmm3 -> xmm2
(gdb) p $xmm2.v2_double[0]
$32 = 2.6352491533870293e+19 
cvttsd2si %xmm2,%rsi          ; failed conversion stores (1<<63) in rsi
mov       %rsi,%r9
sar       $0x3f,%r9           ; account for the sign bit?
subsd     0x1011f0(%rip),%xmm2
cvttsd2si %xmm2,%r10          ; stores PTRDIFF_MAX in r10
and       %r9,%r10
or        %rsi,%r10           ; r10 -> next_nsize
(gdb) p /x next_nsize   
$46 = 0x8000000000000000

Thus the branch

    if (next_nsize > R_NSize + change)
        change = next_nsize - R_NSize;

is taken on Clang but not on GCC.

I think that your patch is sufficient to avoid the next_in_use overflow
and the resulting invalid double -> size_t conversion.

-- 
Best regards,
Ivan