Question

I’m writing an assembly program that calculates Fibonacci numbers, but I need to find a way to detect for overflow when the numbers get too large. My current code is:

.file   "fib.c"
.text
.globl fib
    .type   fib, @function
fib:
    pushl %ebp
    movl %esp, %ebp
    movl 8(%ebp), %eax
    cmpl $0, %eax
    je .end
    cmpl $1, %eax
    je .end

    pushl %edx

    subl $1, %eax
    push %eax
    call fib
    popl %ebx
    movl %eax, %edx

    movl 8(%ebp), %eax
    subl $2, %eax
    push %eax
    call fib
    popl %ebx
    addl %edx, %eax
    jo .overflow
    popl %edx

.end:
    movl %ebp, %esp
    popl %ebp
    ret
.overflow:
    movl $-1, %eax
    ret

I thought I would be able to just use the jo test to see if there was an overflow (Around line 25), but I get a segmentation fault when I enter a number that should overflow.

Any thoughts how I can do this correctly? (BTW, running on a 32 bit machine if that matters)

Thanks,
Mike

EDIT:
For anyone who is interested, here is the working version. I wasn’t clearing the stack and I wasn’t checking for overflow on the movl after the first computation.

.file "fib.c"
.text
.globl fib
    .type   fib, @function
fib:
    pushl %ebp
    movl %esp, %ebp
    movl 8(%ebp), %eax
    cmpl $0, %eax
    je .end
    cmpl $1, %eax
    je .end

    pushl %edx  

    subl $1, %eax
    pushl %eax
    call fib
    popl %ebx
    movl %eax, %edx
    jo .overflow

    movl 8(%ebp), %eax
    subl $2, %eax
    pushl %eax
    call fib
    popl %ebx
    addl %edx, %eax
    jo .overflow
    popl %edx

.end:
    movl %ebp, %esp
    popl %ebp
    ret
.overflow:
    movl $-1, %eax
    movl $-1, %edx
    jmp .end

Answer 1

since you’re using recursion, I’d suspect the stack is what is overflowing, causing your segfault.

---

on second thought, when you hit numeric overflow, you ‘ret’ without cleaning up your stack first. jump to ‘.end’ instead:

.overflow:
    movl $-1, %eax
    jmp .end