Let's try something like this, with debugging printfs left in.
Code:

/* calculation description
Y[0] = x(0) * x(0) + x(1) * x(1) + x(2) * x(2) + x(3) * x(3)
Y[1] = x(0) * x(1) + x(1) * x(2) + x(2) * x(3)
Y[2] = x(0) * x(2) + x(1) * x(3)
Y[3] = x(0) * x(3)
*/
#include <stdio.h>
#define ARRAYSIZE(x) (sizeof(x)/sizeof(*(x)))
int main(void)
{
int x[] = {1,2,3,4}, y [ ARRAYSIZE(x) ];
size_t i,j;
for(j = 0; j < ARRAYSIZE(y); ++j)
{
y[j] = 0;
printf("y[%d] =", (int)j);
for(i = j; i < ARRAYSIZE(x); ++i)
{
printf("%s x[%d] * x[%d]", i != j ? " +" : "", (int)(i - j), (int)i);
y[j] += x[i - j] * x[i];
}
printf(" = %d\n", y[j]);
}
return 0;
}
/* my output
y[0] = x[0] * x[0] + x[1] * x[1] + x[2] * x[2] + x[3] * x[3] = 30
y[1] = x[0] * x[1] + x[1] * x[2] + x[2] * x[3] = 20
y[2] = x[0] * x[2] + x[1] * x[3] = 11
y[3] = x[0] * x[3] = 4
*/

Is this in the ballpark?