A very straightforward way of doing it:

Code:

#include <stdio.h>
#include <stdlib.h>
#define LIMIT 10000000000000000000u // ten quintillion
int main() {
unsigned long a[20];
unsigned n = 0, size = sizeof a / sizeof *a;
unsigned long tri = 1, tri_add = 2, tri_delta = 1,
pent = 1, pent_add = 4, pent_delta = 3;
unsigned tri_ind = 1, pent_ind = 1;
printf("triangular and pentagonal:\n");
while (pent < LIMIT) {
while (tri < pent) {
tri += tri_add; tri_add += tri_delta; ++tri_ind;
}
if (tri == pent) {
printf("%19lu (%10u, %10u)\n", tri, tri_ind, pent_ind);
if (n >= size) { printf("array overflow\n"); exit(1); }
a[n++] = tri;
tri += tri_add; tri_add += tri_delta; ++tri_ind;
}
while (pent < tri) {
pent += pent_add; pent_add += pent_delta; ++pent_ind;
}
if (tri == pent) {
printf("%19lu (%10u, %10u)\n", tri, tri_ind, pent_ind);
if (n >= size) { printf("array overflow\n"); exit(1); }
a[n++] = tri;
pent += pent_add; pent_add += pent_delta; ++pent_ind;
}
}
printf("\nalso hexagonal:\n");
unsigned long hex = 1, hex_add = 5, hex_delta = 4;
unsigned hex_ind = 1;
for (unsigned i = 0; i < n; ) {
while (hex < a[i]) {
hex += hex_add; hex_add += hex_delta; ++hex_ind;
}
if (hex == a[i]) printf("%19lu (%10u)\n", hex, hex_ind);
++i;
hex += hex_add; hex_add += hex_delta; ++hex_ind;
}
return 0;
}
Output:
triangular and pentagonal:
1 ( 1, 1)
210 ( 20, 12)
40755 ( 285, 165)
7906276 ( 3976, 2296)
1533776805 ( 55385, 31977)
297544793910 ( 771420, 445380)
57722156241751 ( 10744501, 6203341)
11197800766105800 ( 149651600, 86401392)
2172315626468283465 (2084377905, 1203416145)
also hexagonal:
1 ( 1)
40755 ( 143)
1533776805 ( 27693)
57722156241751 ( 5372251)
2172315626468283465 (1042188953)

So every second number that is both triangular and pentagonal is also hexagonal.