To be honest, I am not sure what is the best way to go about doing this. One option is to make use of covariant return types with a helper function:

Code:

class Matrix
{
public:
Matrix(int m, int n) {}
virtual ~Matrix() {}
virtual Matrix* transpose() const
{
Matrix* temp = new Matrix(*this);
temp->transposeSelf();
return temp;
}
protected:
void transposeSelf()
{
// change contents of object
}
};
class Matrix3D : public Matrix
{
public:
Matrix3D() : Matrix(3, 3) {}
virtual Matrix3D* transpose() const
{
Matrix3D* temp = new Matrix3D(*this);
temp->transposeSelf();
return temp;
}
};
int main()
{
Matrix3D M;
Matrix3D* N = M.transpose();
delete N;
}

Another option is to hide the base class' transpose() while using a helper function:

Code:

class Matrix
{
public:
Matrix(int m, int n) {}
virtual ~Matrix() {}
Matrix transpose() const
{
Matrix temp(*this);
temp.transposeSelf();
return temp;
}
protected:
void transposeSelf()
{
// change contents of object
}
};
class Matrix3D : public Matrix
{
public:
Matrix3D() : Matrix(3, 3) {}
Matrix3D transpose() const
{
Matrix3D temp(*this);
temp.transposeSelf();
return temp;
}
// Matrix::transpose() does not exist in this scope.
};
int main()
{
Matrix3D M;
Matrix3D N = M.transpose();
}

Note that for the latter transpose() is not polymorphic.

As for whether these two options I propose are good solutions at all, I do not really know.