A couple of problems.

You should only loop up to i < size - 1 since otherwise 1u<<(i+1) will create 0u on the last iteration.

You should also start counter at 1, since there will always be at least 1 segment.

Code:

#include <stdio.h>
#include <limits.h>
int countSegments(unsigned n)
{
const unsigned size = sizeof(n) * CHAR_BIT;
int counter = 1;
for (unsigned i = 0; i < size - 1; i++)
{
unsigned mask1 = n & (1u << i);
unsigned mask2 = n & (1u << (i + 1));
if ((mask1 << 1) != mask2)
++counter;
}
return counter;
}
void bits(unsigned n)
{
unsigned mask = 1u << (sizeof(n) * CHAR_BIT - 1);
for ( ; mask; mask >>= 1) putchar('0' + !!(n & mask));
}
int main()
{
unsigned nums[] = {
0, // 0000
1, // 0001
2, // 0010
3, // 0011
4, // 0100
5, // 0101
0xFFFFFFFF,
0x80000000,
0xAB, // 1010 1011 (8)
0x68B9 // 0110 1000 1011 1001 (10)
};
for (unsigned i = 0; i < sizeof nums/sizeof *nums; ++i)
{
// Note that you should use %u to print an unsigned int.
printf("%10u (0x%08X) ", nums[i], nums[i]);
bits(nums[i]);
printf(" : %2d\n", countSegments(nums[i]));
}
return 0;
}