Google 3D picking. Essentially you cast a ray from your cursor into the 3D world and then make a list of what it hit and normally return the closest object to the camera or provide the player with a list of things to select.
Code:
IGameObject *Picking::Pick()
{
POINT s;
GetCursorPos(&s);
ScreenToClient(g_pApp->GetWindowHandle(),&s);
//ClientToScreen(g_pApp->GetWindowHandle(),&s);
D3DVIEWPORT9 vp;
g_pApp->GetDevice()->GetViewport(&vp);
float w = vp.Width;
float h = vp.Height;
D3DXMATRIX matProj = g_pApp->getCamera()->GetProjMatrix();
float x = ( ( ( 2.0f * s.x ) / w ) - 1.0f ) / matProj._11;
float y = -( ( ( 2.0f * s.y ) / h ) - 1.0f ) / matProj._22;
float z = 1.0f;
//float x = (2.0f * s.x / w - 1.0f) / matProj(0,0);
//float y = (-2.0f * s.y / h + 1.01f) / matProj(1,1);
D3DXVECTOR3 origin(0.0f,0.0f,0.0f);
D3DXVECTOR3 dir(x,y,1.0f);
D3DXMATRIX matInvView;
g_pApp->getCamera()->CalcViewMatrix();
D3DXMATRIX matView = g_pApp->getCamera()->GetViewMatrix();
D3DXMatrixInverse(&matInvView,0,&matView);
D3DXVec3TransformCoord(&m_vecWorldOrigin,&origin,&matInvView);
D3DXVec3TransformNormal(&m_vecWorldDir,&dir,&matInvView);
D3DXVec3Normalize(&m_vecWorldDir,&m_vecWorldDir);
/*m_Line[0] = PickVertex(m_vecWorldOrigin.x,m_vecWorldOrigin.y,m_vecWorldOrigin.z,D3DCOLOR_ARGB(255,255,0,0));
D3DXVECTOR3 vecEnd = m_vecWorldOrigin + m_vecWorldDir * 200.0f;
m_Line[1] = PickVertex(vecEnd.x,vecEnd.y,vecEnd.z,D3DCOLOR_ARGB(255,255,255,255));*/
return testObjects();
}
IGameObject *Picking::testObjects()
{
std::vector<IGameObject *> visible = g_pApp->getScene()->getVisible();
IGameObject *result = 0;
util::AABB temp;
D3DXMATRIX matView = g_pApp->getCamera()->GetViewMatrix();
D3DXMATRIX matProj = g_pApp->getCamera()->GetProjMatrix();
D3DXMATRIX matWorldViewProj;
D3DXMATRIX matTrans;
std::vector<IGameObject *> hitList;
for (size_t i = 0;i < visible.size(); ++i)
{
util::AABB box = visible[i]->GetAABB();
D3DXVECTOR3 pos = visible[i]->GetPosition();
D3DXMatrixTranslation(&matTrans,pos.x,pos.y,pos.z);
matWorldViewProj = matTrans; // * matProj;
box.XForm(matWorldViewProj,box);
BOOL hit = D3DXBoxBoundProbe(&box.GetMin(),
&box.GetMax(),
&m_vecWorldOrigin,
&m_vecWorldDir);
if (hit == TRUE)
{
hitList.push_back(visible[i]);
}
}
D3DXVECTOR3 camPos = g_pApp->getCamera()->GetPosition();
float minDist = FLT_MAX;
for (size_t i = 0;i < hitList.size(); ++i)
{
IGameObject *temp = hitList[i];
D3DXVECTOR3 toObject = hitList[i]->GetPosition() - camPos;
float dist = D3DXVec3Length(&toObject);
if (dist < minDist)
{
minDist = dist;
result = hitList[i];
}
}
hitList.clear();
if (result)
{
g_pApp->getCamera()->LookAt(result->GetPosition());
}
return result;
}
This is for D3D so you will have to adjust the z direction of the ray and transpose all the matrices. However the math for computing the ray should be the same (on a transposed matrix).