I need help with D3D picking with mouse

If the model is not set to the Identity , what space do i pick in ?

this is its translation and rendering

Code:

// Build castle's world matrix.
D3DXMatrixRotationY(&Ry, D3DX_PI);
D3DXMatrixTranslation(&T, 8.0f, 35.0f, -80.0f);
mCastleWorld = Ry*T;

Code:

void PropsDemo::drawObject(Object3D& obj, const D3DXMATRIX& toWorld)
{
	// Transform AABB into the world space.
	
	AABB box;
	obj.box.xform(toWorld, box);

	// Only draw if AABB is visible.
	if( gCamera->isVisible( box ) )
	{
		HR(mFX->SetMatrix(mhWVP, &(toWorld*gCamera->viewProj())));
		
		D3DXMatrixInverse(&worldInvTrans, 0, &toWorld); //objectspace
		D3DXMatrixTranspose(&worldInvTrans, &worldInvTrans);
		HR(mFX->SetMatrix(mhWorldInvTrans, &worldInvTrans));
		HR(mFX->SetMatrix(mhWorld, &toWorld));

		for(UINT j = 0; j < obj.mtrls.size(); ++j)
		{
			HR(mFX->SetValue(mhMtrl, &obj.mtrls[j], sizeof(Mtrl)));
		
			// If there is a texture, then use.
			if(obj.textures[j] != 0)
			{
				HR(mFX->SetTexture(mhTex, obj.textures[j]));
			}

			// But if not, then set a pure white texture.  When the texture color
			// is multiplied by the color from lighting, it is like multiplying by
			// 1 and won't change the color from lighting.
			else
			{
				HR(mFX->SetTexture(mhTex, mWhiteTex));
			}
		
			HR(mFX->CommitChanges());
			HR(obj.mesh->DrawSubset(j));
		}
	}
}

Code:

drawObject(mCastle, mCastleWorld);

You want to know how to pick an object using the mouse? For example, if you clicked over a castle, you want to know whether or not you selected it?


First, get the mouse position, they will be in screen space and in pixels... like

(450, 600) Thats a valid mouse position.... Now you need to translate that to something you can use, right?

Here is some old code that I have for picking objects in a 3d world... You should be able to copy and paste, and replace the variables with yours...

Code:

D3DXMATRIX P = GetCamera().proj();
	float vx = (+2.0f*g_MousePos_V.x/g_WindowDimensions_V.x  - 1.0f)/P(0,0);// Compute picking ray in view space.
	float vy = (-2.0f*g_MousePos_V.y/g_WindowDimensions_V.y + 1.0f)/P(1,1);
	D3DXMATRIX V = GetCamera().view();
	D3DXMATRIX inverseV;
	D3DXMatrixInverse(&inverseV, 0, &V);// Tranform to world space.
	D3DXVECTOR3 rayOrigin(0.0f, 0.0f, 0.0f);
	D3DXVECTOR3 rayDir(vx, vy, 1.0f);
	D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseV);// this function actually translated the point
	D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseV);// this function does no translate the vector, it only changes the direction into the new coordinate system
	D3DXVECTOR3 srayOrigin(rayOrigin), srayDir(rayDir);// save these because I change them in the for loop. This way I dont need to recalc every iteration
	// Transform to the mesh's local space for all meshes!!
	// add tests for multiple intersections......
	vector<cTargetInfo> targets;
	vector<float> distances;
	string empty;
	for(size_t i(0); i< Mesh.size(); i++){
		rayOrigin=srayOrigin;// restore original values
		rayDir=srayDir;// same
		D3DXMATRIX inverseW, wor(Mesh[i]->Scaling*Mesh[i]->Rotation* Mesh[i]->Translation);
		D3DXMatrixInverse(&inverseW, 0, &wor);
		D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseW);// transform the origin into the objects space
		D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseW);// trasform ray into object space
		rayDir*=FarPlane;// scale it to make sure it goes through the entire scene, when it is translated from view space it is a short dog... make it a big dog :)
		// first, do a AABB intersection, if we hit, then do an extensive triangle intersection test to be sure
		if(Mesh[i]->IntersectAABB(rayOrigin, rayDir)){// if ret true, we hit the AABB, now we can check triangles to be sure
			rayDir/=FarPlane;// undo the scaling
			ID3DX10Mesh* d3dxmesh = Mesh[i]->MeshData;
			uint32_t picked, hitcount;
			float u,v,t;
			ID3D10Blob* allHits;
			d3dxmesh->Intersect(&rayOrigin, &rayDir, &hitcount, &picked, &u, &v, &t, &allHits);
			distances.push_back(t);
			if(hitcount>0){
				distances.push_back(t);// add the distance
			} else {
				distances.push_back(FLT_MAX);// otherwise, we missed add a huge distance
			}
			ReleaseCOM(allHits);
		} else {
			distances.push_back(FLT_MAX);// otherwise, we missed add a huge distance
		}// else, we missed the AABB
	}
	// figure out which target we really hit
	float cdist(FLT_MAX);
	for(size_t i(0); i< distances.size(); i++){
		if(distances[i] < cdist){
			cdist=distances[i];
		}
	}

but the castle in is in objectspace
D3DXMATRIX V = GetCamera().view(); ,is only for D3DXMatrixIdentity based models,etc set straight in to worldspace...

Not sure what you mean....


The code I posted is for selecting an object in a 3d world, using mouse coords.

i mean there is no SetTransform on the castle , so its still in localspace but using a shader...

3d objects are always in local space. You translate them to world space when you apply a rotation, scale or translation, which you do in your code.

So, your castle is in world space.

never mind ive figured it out ,the pick space is
D3DXMatrixInverse(&MeshLocalWorld,NULL,&(Rot * matTrans * gCamera->view()));
its only gCamera->view() if model is set to the Identity Matrix.

You don't have to pick in local space or object space. But whatever space you choose to pick in just make sure your model and your pick ray are in the same space.