Well, written explanations are usually harder to grasp than a visual demonstration so I fixed your program to a working state and put out some comments and I hope it will help.
Code:
#include <iostream>
#include <cstdlib>
#include <ctime>
#include <ctype>
using namespace std;
bool win (void); //Function prototype.
int main () //Remember to use good variable names if applicable!
{
int number; //Originally "a". Also a local variable this time, don't use globals.
int guess; //Originally "b".
int guesscount; //Originally "x".
srand(time(0)); //Give rand() a seed so we actually get a random number.
do { //The main loop.
guesscount = 1;
number = rand()%999 + 1; //rand()%999 gives a number between 0 and 998 instead of 1 and 999.
cout << "See if you can crack the code!" << endl;
do { //The other loop.
cout << "Guess #" << guesscount++ << ": " << flush; //Notice the ++ after guesscount.
cin >> guess;
cin.ignore();
if (guess > number)
cout << "Too high, try again..." << endl;
else if (guess < number)
cout << "Too low, try again..." << endl;
} while (guess != number);
} while (win()); //If the user wants to continue win() returns true and the loop continues.
}
bool win (void)
{
char answer; //Originally "y".
cout << "Correct!" << endl //I am only making one call to cout here.
<< "Try to break another code? Y or N: " << flush; //If I'm not ending a cout with endl then I end it with flush.
cin >> answer;
if (tolower(answer) == 'y')
return true;
//"else"
return false;
}