How to tell what you're selecting?

Say I have an 3d opengl game.....
You can walk around and there are objects and things on the ground....
If I wanted people to be able to click on these objects and have something happen, how do I tell if the person has his mouse over it.

Basically, how do I use mouse position and character position to determine where and what the mouse is selecting ingame?

OpenGL has inbuilt selection and feedback support. You can use them to do whatever you want that have to do with selection and it's very efficient!

Btw, i would REALLY suggest nehe tutorials if you are doing opengl. You can find anything that you are looking for there.

nehe opengl tutorials

Google 3D picking. Essentially you cast a ray from your cursor into the 3D world and then make a list of what it hit and normally return the closest object to the camera or provide the player with a list of things to select.

Code:

IGameObject *Picking::Pick()
{
    POINT s;
    GetCursorPos(&s);

    ScreenToClient(g_pApp->GetWindowHandle(),&s);
    //ClientToScreen(g_pApp->GetWindowHandle(),&s);
    
    D3DVIEWPORT9 vp;

    g_pApp->GetDevice()->GetViewport(&vp);

    float w = vp.Width;
    float h = vp.Height;

    D3DXMATRIX matProj = g_pApp->getCamera()->GetProjMatrix();

    float x =  ( ( ( 2.0f * s.x ) / w ) - 1.0f ) / matProj._11;
    float y = -( ( ( 2.0f * s.y ) / h ) - 1.0f ) / matProj._22;
    float z =  1.0f;
    
    //float x = (2.0f * s.x / w - 1.0f) / matProj(0,0);
    //float y = (-2.0f * s.y / h + 1.01f) / matProj(1,1);

    D3DXVECTOR3 origin(0.0f,0.0f,0.0f);
    D3DXVECTOR3 dir(x,y,1.0f);

    D3DXMATRIX matInvView;

    g_pApp->getCamera()->CalcViewMatrix();
    D3DXMATRIX matView = g_pApp->getCamera()->GetViewMatrix();

    D3DXMatrixInverse(&matInvView,0,&matView);

    D3DXVec3TransformCoord(&m_vecWorldOrigin,&origin,&matInvView);
    D3DXVec3TransformNormal(&m_vecWorldDir,&dir,&matInvView);
    D3DXVec3Normalize(&m_vecWorldDir,&m_vecWorldDir);
    
    /*m_Line[0] = PickVertex(m_vecWorldOrigin.x,m_vecWorldOrigin.y,m_vecWorldOrigin.z,D3DCOLOR_ARGB(255,255,0,0));
    
    D3DXVECTOR3 vecEnd = m_vecWorldOrigin + m_vecWorldDir * 200.0f;
    m_Line[1] = PickVertex(vecEnd.x,vecEnd.y,vecEnd.z,D3DCOLOR_ARGB(255,255,255,255));*/

    
    return testObjects();
}

IGameObject *Picking::testObjects()
{
    std::vector<IGameObject *> visible = g_pApp->getScene()->getVisible();

    IGameObject *result = 0;
    util::AABB temp;
    
    D3DXMATRIX matView = g_pApp->getCamera()->GetViewMatrix();
    D3DXMATRIX matProj = g_pApp->getCamera()->GetProjMatrix();
    D3DXMATRIX matWorldViewProj;
    D3DXMATRIX matTrans;

    std::vector<IGameObject *> hitList;

    for (size_t i = 0;i < visible.size(); ++i)
    {
        util::AABB box = visible[i]->GetAABB();
        D3DXVECTOR3 pos = visible[i]->GetPosition();
        
        D3DXMatrixTranslation(&matTrans,pos.x,pos.y,pos.z);

        matWorldViewProj = matTrans; // * matProj;
        box.XForm(matWorldViewProj,box);

        BOOL hit = D3DXBoxBoundProbe(&box.GetMin(),
                                     &box.GetMax(),
                                     &m_vecWorldOrigin,
                                     &m_vecWorldDir);

        if (hit == TRUE)
        {
            hitList.push_back(visible[i]);  
        }
    }

    D3DXVECTOR3 camPos = g_pApp->getCamera()->GetPosition();
    float minDist = FLT_MAX;

    for (size_t i = 0;i < hitList.size(); ++i)
    {
        IGameObject *temp = hitList[i];
        D3DXVECTOR3 toObject = hitList[i]->GetPosition() - camPos;
        float dist = D3DXVec3Length(&toObject);

        if (dist < minDist)
        {
            minDist = dist;
            result = hitList[i];
        }
    }

    hitList.clear();
    
    if (result)
    {
        g_pApp->getCamera()->LookAt(result->GetPosition());
    }
    return result;
}

This is for D3D so you will have to adjust the z direction of the ray and transpose all the matrices. However the math for computing the ray should be the same (on a transposed matrix).

Thanks, interesting way to do it wouldn't have though of that.