help !! : problem with binary file, and fread into structs

Hi Guys

I'm a PERL programmer usually but I have to write something in a C by tommorow morning and I'm having a real problem doing it (and whilst I can write fair PERL I am more or less a C n00b).

I have a binary file with a known format. I am trying to read the file a record at a time using fread. If I use this code :

Code:

---

#include <stdio.h>
#include <stdint.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <inttypes.h>


struct utpDquoteMessage
{
    char messageType;
    char ProtoOrVer;
    uint16_t MsgLen;     
    uint32_t MsgSeqNum;
    uint64_t ClOrdID;
    char OnBehalfOfCompID[11];
    char Rule80A;
    char Account[12];
    char TechnicalOrdType;
    char ClearingFirm[8];
    char ClientID[8];
    char FreeText[18];
    char OpenClose;
    char ClearingHandlingType;
    char Filler1;
    uint16_t NoQuoteEntries;
};


int main(void)
{
    char utpFile[64];
    FILE *utpF;
    int c,i,z, ret;
    char rest[1024];
   
    struct utpDquoteMessage qMsg;

    if((utpF = fopen("../data/enxtUtpDirectBeta.dump", "rb")) == NULL)
    {
        printf("Can't open file for reading");
        return(1);
    }
   
    while(!feof(utpF))
    { 
        printf("reading %u bytes\n", sizeof(struct utpDquoteMessage));
        int read=fread(&qMsg, sizeof(qMsg),1,utpF);
        printf("fread returned %d\n", read);
        printf(".\n");
        printf("messagetype : %c\n", qMsg.messageType);
        printf("ProtoOrVer : %c\n", qMsg.ProtoOrVer);
        printf("MsgLen : %hu\n", qMsg.MsgLen);
        printf("MsgSeqNum : %u\n", qMsg.MsgSeqNum);
        printf("ClOrdID : %u\n", qMsg.ClOrdID);
        printf("OnBehalfOfCompID : %s\n", qMsg.OnBehalfOfCompID);
        printf("Rule80A : %c\n", qMsg.Rule80A);
        printf("Account : %s\n", qMsg.Account);
        printf("TechnicalOrdType : %c\n", qMsg.TechnicalOrdType);
        printf("ClearingFirm : %s\n", qMsg.ClearingFirm);
        printf("ClientID : %s\n", qMsg.ClientID);
        printf("FreeText : %s\n",qMsg.FreeText);
        printf("OpenClose : %c\n", qMsg.OpenClose);
        printf("ClearingHandlingType : %c\n", qMsg.ClearingHandlingType);
        printf("NoQuoteEntries : %u\n", qMsg.NoQuoteEntries);       
        fseek(utpF, (qMsg.MsgLen - sizeof(struct utpDquoteMessage) + 1), SEEK_CUR);

       
        printf(".\n.\n");
    }
 
   
    fclose(utpF);
   
    return(0);
}

---

then it works :

Code:

---

[mddev@mdsaln01 /app/devel/mdscripts/PerfAgent/bin] $ gcc -o enxtUtpDirectDecoder foo.c && ./enxtUtpDirectDecoder|less
reading 80 bytes
fread returned 1
.
messagetype : B
ProtoOrVer : 2
MsgLen : 980
MsgSeqNum : 32557
ClOrdID : 543376761
OnBehalfOfCompID : 00029002
Rule80A : 6
Account : 2
TechnicalOrdType : ^@
ClearingFirm :
ClientID :
FreeText :
OpenClose : O
ClearingHandlingType : 0
NoQuoteEntries : 28

---

(I know this is the right output because it's been verified)


Problem is - I want to split the header and message body up, since the header always contains the same fields but the message body changes. So, I've written this :

Code:

---

#include <stdio.h>
#include <stdint.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <inttypes.h>


struct utpDmessageHeader
{
    char messageType;
    char ProtoOrVer;
    uint16_t MsgLen;   
};

struct utpDquoteMessageBody
{     
    uint32_t MsgSeqNum;
    uint64_t ClOrdID;
    char OnBehalfOfCompID[11];
    char Rule80A;
    char Account[12];
    char TechnicalOrdType;
    char ClearingFirm[8];
    char ClientID[8];
    char FreeText[18];
    char OpenClose;
    char ClearingHandlingType;
    char Filler1;
    uint16_t NoQuoteEntries;
};


int main(void)
{
    char utpFile[64];
    FILE *utpF;
    int c,i,z, ret;
    char rest[1024];
   
    struct utpDmessageHeader msgHeader;
    struct utpDquoteMessageBody qMsgBody;
   
    if((utpF = fopen("../data/enxtUtpDirectBeta.dump", "rb")) == NULL)
    {
        printf("Can't open file for reading");
        return(1);
    }
   
        printf("reading %u bytes\n", sizeof(struct utpDmessageHeader));
        fread(&msgHeader, sizeof(struct utpDmessageHeader), 1, utpF);
        printf("done.\n");
        printf("messagetype : %c\n", msgHeader.messageType);
        printf("ProtoOrVer : %c\n", msgHeader.ProtoOrVer);
        printf("MsgLen : %hu\n", msgHeader.MsgLen);
        filePosition+=sizeof(msgHeader);
        printf("filePosition == %u\n", filePosition);
        printf("ftell says %u\n", ftell(utpF));
       
        printf("reading %u bytes\n", sizeof(struct utpDquoteMessageBody));
        int read=fread(&qMsgBody, sizeof(struct utpDquoteMessageBody),1,utpF);
        printf("fread returned %d\n", read);
        printf(".\n");
        printf("MsgSeqNum : %u\n", qMsgBody.MsgSeqNum);
        printf("ClOrdID : %u\n", qMsgBody.ClOrdID);
        printf("OnBehalfOfCompID : %s\n", qMsgBody.OnBehalfOfCompID);
        printf("Rule80A : %c\n", qMsgBody.Rule80A);
        printf("Account : %s\n", qMsgBody.Account);
        printf("TechnicalOrdType : %c\n", qMsgBody.TechnicalOrdType);
        printf("ClearingFirm : %s\n", qMsgBody.ClearingFirm);
        printf("ClientID : %s\n", qMsgBody.ClientID);
        printf("FreeText : %s\n",qMsgBody.FreeText);
        printf("OpenClose : %c\n", qMsgBody.OpenClose);
        printf("ClearingHandlingType : %c\n", qMsgBody.ClearingHandlingType);
        printf("NoQuoteEntries : %u\n", qMsgBody.NoQuoteEntries);

 
       
        printf(".\n.\n");
   
 
   
    fclose(utpF);
   
    return(0);
}

---

...but when I run it the contents of the second struct are messed up after the first field :

Code:

---

[mddev@mdsaln01 /app/devel/mdscripts/PerfAgent/bin] $ gcc -o enxtUtpDirectDecoder enxtUtpDirectDecoder.c && ./enxtUtpDirectDecoder|less
.
.
reading 4 bytes
done.
messagetype : B
ProtoOrVer : 2
MsgLen : 980
filePosition == 4
ftell says 4
reading 80 bytes
fread returned 1
.
MsgSeqNum : 32557
ClOrdID : 1569433187
OnBehalfOfCompID : 9002
Rule80A : ^@
Account :
TechnicalOrdType : ^@
ClearingFirm :
ClientID :
FreeText :
OpenClose : ^\
ClearingHandlingType : N
NoQuoteEntries : 12336

---

Maybe I'm missing something but how come it's correctly writing the first field(MsgSeqNum) but then the rest is garbage ??.

I'm tearing my hair out and not getting anywhere : ( Any help would be greatly appreciated !!!

but I'm already reading MsgLen in the first struct "utpDmessageHeader" ...

Did you notice how the size of your 2nd structure was still 80?

> uint64_t ClOrdID;
Your original structure aligned this on an 8-byte boundary.

Unfortunately for you, by removing the header, this is no longer the case. So the compiler inserts padding to preserve the most efficient way of accessing the struct.

You can either
- add the header (either as members, or as your struct).
- resort to various compiler "pack" options
Structure-Packing Pragmas - Using the GNU Compiler Collection (GCC)
Variable Attributes - Using the GNU Compiler Collection (GCC)

I tried this

Code:

---

#include <stdio.h>

typedef unsigned short uint16_t;
typedef unsigned int uint32_t;
typedef unsigned long long uint64_t;

typedef struct utpDquoteMessage
{
    char messageType;
    char ProtoOrVer;
    uint16_t MsgLen;     
    uint32_t MsgSeqNum;
    uint64_t ClOrdID;
    char OnBehalfOfCompID[11];
    char Rule80A;
    char Account[12];
    char TechnicalOrdType;
    char ClearingFirm[8];
    char ClientID[8];
    char FreeText[18];
    char OpenClose;
    char ClearingHandlingType;
    char Filler1;
    uint16_t NoQuoteEntries;
} total;

typedef struct utpDmessageHeader
{
    char messageType;
    char ProtoOrVer;
    uint16_t MsgLen;   
} hdr;

typedef struct utpDquoteMessageBody
{     
    uint32_t MsgSeqNum;
    uint64_t ClOrdID;
    char OnBehalfOfCompID[11];
    char Rule80A;
    char Account[12];
    char TechnicalOrdType;
    char ClearingFirm[8];
    char ClientID[8];
    char FreeText[18];
    char OpenClose;
    char ClearingHandlingType;
    char Filler1;
    uint16_t NoQuoteEntries;
} body;

int main() {
    printf("%d %d %d\n", sizeof(total), sizeof(hdr), sizeof(body));
}

---

and it gives me

Code:

---

80 4 80

---

seems to be a packing problem. ( I don't know if the typedefs are correct. )

Kurt

>>So the compiler inserts padding to preserve the most efficient way of accessing the struct.

that's nice of it !!

>>- add the header (either as members, or as your struct).


Thanks very much for your assistance guys, at least I know what's happening now and can move forward. I'll read the info on packing then and try to figure out how best to proceed.

: )

Hi Guys

I have now used the packing pragmas around my structs :

Code:

---

#pragma pack(1)
struct utpDmessageHeader
{
    char messageType;
    char ProtoOrVer;
    uint16_t MsgLen;   
};

struct utpDquoteMessageBody
{       
    uint32_t MsgSeqNum;
    uint64_t ClOrdID;
    char OnBehalfOfCompID[11];
    char Rule80A;
    char Account[12];
    char TechnicalOrdType;
    char ClearingFirm[8];
    char ClientID[8];
    char FreeText[18];
    char OpenClose;
    char ClearingHandlingType;
    char Filler1;
    uint16_t NoQuoteEntries;
};
#pragma pack()

---

...and it works a treat :

Code:

---

[mddev@mdsaln01:/app/devel/mdscripts/PerfAgent/bin] $ gcc -o enxtUtpDirectDecoder enxtUtpDirectDecoder.c && ./enxtUtpDirectDecoder|less
reading 4 bytes
done.
messagetype : B
ProtoOrVer : 2
MsgLen : 980
quote.
reading 76 bytes
fread returned 1
.
MsgSeqNum : 32557
ClOrdID : 543376761
OnBehalfOfCompID : 00029002
Rule80A : 6
Account : 2
TechnicalOrdType : ^@
ClearingFirm :
ClientID :
FreeText :
OpenClose : O
ClearingHandlingType : 0
NoQuoteEntries : 28
.
.
[mddev@mdsaln01:/app/devel/mdscripts/PerfAgent/bin] $

---

you guys are the best, thanks a million !!!