help !! : problem with binary file, and fread into structs

This is a discussion on help !! : problem with binary file, and fread into structs within the C Programming forums, part of the General Programming Boards category; Hi Guys I'm a PERL programmer usually but I have to write something in a C by tommorow morning and ...

  1. #1
    Registered User
    Join Date
    Feb 2010
    Posts
    8

    Unhappy help !! : problem with binary file, and fread into structs

    Hi Guys

    I'm a PERL programmer usually but I have to write something in a C by tommorow morning and I'm having a real problem doing it (and whilst I can write fair PERL I am more or less a C n00b).

    I have a binary file with a known format. I am trying to read the file a record at a time using fread. If I use this code :

    Code:
    #include <stdio.h>
    #include <stdint.h>
    #include <sys/types.h>
    #include <netinet/in.h>
    #include <inttypes.h>
    
    
    struct utpDquoteMessage
    {
        char messageType;
        char ProtoOrVer;
        uint16_t MsgLen;      
        uint32_t MsgSeqNum;
        uint64_t ClOrdID;
        char OnBehalfOfCompID[11];
        char Rule80A;
        char Account[12];
        char TechnicalOrdType;
        char ClearingFirm[8];
        char ClientID[8];
        char FreeText[18];
        char OpenClose;
        char ClearingHandlingType;
        char Filler1;
        uint16_t NoQuoteEntries;
    };
    
    
    int main(void)
    {
        char utpFile[64];
        FILE *utpF;
        int c,i,z, ret;
        char rest[1024];
        
        struct utpDquoteMessage qMsg;
    
        if((utpF = fopen("../data/enxtUtpDirectBeta.dump", "rb")) == NULL)
        {
            printf("Can't open file for reading");
            return(1);
        }
        
        while(!feof(utpF))
        {   
            printf("reading %u bytes\n", sizeof(struct utpDquoteMessage));
            int read=fread(&qMsg, sizeof(qMsg),1,utpF);
            printf("fread returned %d\n", read);
            printf(".\n");
            printf("messagetype : %c\n", qMsg.messageType);
            printf("ProtoOrVer : %c\n", qMsg.ProtoOrVer);
            printf("MsgLen : %hu\n", qMsg.MsgLen);
            printf("MsgSeqNum : %u\n", qMsg.MsgSeqNum);
            printf("ClOrdID : %u\n", qMsg.ClOrdID);
            printf("OnBehalfOfCompID : %s\n", qMsg.OnBehalfOfCompID);
            printf("Rule80A : %c\n", qMsg.Rule80A);
            printf("Account : %s\n", qMsg.Account);
            printf("TechnicalOrdType : %c\n", qMsg.TechnicalOrdType);
            printf("ClearingFirm : %s\n", qMsg.ClearingFirm);
            printf("ClientID : %s\n", qMsg.ClientID);
            printf("FreeText : %s\n",qMsg.FreeText);
            printf("OpenClose : %c\n", qMsg.OpenClose);
            printf("ClearingHandlingType : %c\n", qMsg.ClearingHandlingType);
            printf("NoQuoteEntries : %u\n", qMsg.NoQuoteEntries);        
            fseek(utpF, (qMsg.MsgLen - sizeof(struct utpDquoteMessage) + 1), SEEK_CUR);
    
            
            printf(".\n.\n");
        }
     
        
        fclose(utpF);
        
        return(0);
    }
    then it works :

    Code:
    [mddev@mdsaln01 /app/devel/mdscripts/PerfAgent/bin] $ gcc -o enxtUtpDirectDecoder foo.c && ./enxtUtpDirectDecoder|less
    reading 80 bytes
    fread returned 1
    .
    messagetype : B
    ProtoOrVer : 2
    MsgLen : 980
    MsgSeqNum : 32557
    ClOrdID : 543376761
    OnBehalfOfCompID : 00029002
    Rule80A : 6
    Account : 2
    TechnicalOrdType : ^@
    ClearingFirm : 
    ClientID : 
    FreeText : 
    OpenClose : O
    ClearingHandlingType : 0
    NoQuoteEntries : 28
    (I know this is the right output because it's been verified)


    Problem is - I want to split the header and message body up, since the header always contains the same fields but the message body changes. So, I've written this :

    Code:
    #include <stdio.h>
    #include <stdint.h>
    #include <sys/types.h>
    #include <netinet/in.h>
    #include <inttypes.h>
    
    
    struct utpDmessageHeader
    {
        char messageType;
        char ProtoOrVer;
        uint16_t MsgLen;    
    };
    
    struct utpDquoteMessageBody
    {      
        uint32_t MsgSeqNum;
        uint64_t ClOrdID;
        char OnBehalfOfCompID[11];
        char Rule80A;
        char Account[12];
        char TechnicalOrdType;
        char ClearingFirm[8];
        char ClientID[8];
        char FreeText[18];
        char OpenClose;
        char ClearingHandlingType;
        char Filler1;
        uint16_t NoQuoteEntries;
    };
    
    
    int main(void)
    {
        char utpFile[64];
        FILE *utpF;
        int c,i,z, ret;
        char rest[1024];
        
        struct utpDmessageHeader msgHeader;
        struct utpDquoteMessageBody qMsgBody;
        
        if((utpF = fopen("../data/enxtUtpDirectBeta.dump", "rb")) == NULL)
        {
            printf("Can't open file for reading");
            return(1);
        }
        
            printf("reading %u bytes\n", sizeof(struct utpDmessageHeader));
            fread(&msgHeader, sizeof(struct utpDmessageHeader), 1, utpF);
            printf("done.\n");
            printf("messagetype : %c\n", msgHeader.messageType);
            printf("ProtoOrVer : %c\n", msgHeader.ProtoOrVer);
            printf("MsgLen : %hu\n", msgHeader.MsgLen);
            filePosition+=sizeof(msgHeader);
            printf("filePosition == %u\n", filePosition);
            printf("ftell says %u\n", ftell(utpF));
            
            printf("reading %u bytes\n", sizeof(struct utpDquoteMessageBody));
            int read=fread(&qMsgBody, sizeof(struct utpDquoteMessageBody),1,utpF);
            printf("fread returned %d\n", read);
            printf(".\n");
            printf("MsgSeqNum : %u\n", qMsgBody.MsgSeqNum);
            printf("ClOrdID : %u\n", qMsgBody.ClOrdID);
            printf("OnBehalfOfCompID : %s\n", qMsgBody.OnBehalfOfCompID);
            printf("Rule80A : %c\n", qMsgBody.Rule80A);
            printf("Account : %s\n", qMsgBody.Account);
            printf("TechnicalOrdType : %c\n", qMsgBody.TechnicalOrdType);
            printf("ClearingFirm : %s\n", qMsgBody.ClearingFirm);
            printf("ClientID : %s\n", qMsgBody.ClientID);
            printf("FreeText : %s\n",qMsgBody.FreeText);
            printf("OpenClose : %c\n", qMsgBody.OpenClose);
            printf("ClearingHandlingType : %c\n", qMsgBody.ClearingHandlingType);
            printf("NoQuoteEntries : %u\n", qMsgBody.NoQuoteEntries);
    
     
            
            printf(".\n.\n");
        
     
        
        fclose(utpF);
        
        return(0);
    }
    ...but when I run it the contents of the second struct are messed up after the first field :

    Code:
    [mddev@mdsaln01 /app/devel/mdscripts/PerfAgent/bin] $ gcc -o enxtUtpDirectDecoder enxtUtpDirectDecoder.c && ./enxtUtpDirectDecoder|less
    .
    .
    reading 4 bytes
    done.
    messagetype : B
    ProtoOrVer : 2
    MsgLen : 980
    filePosition == 4
    ftell says 4
    reading 80 bytes
    fread returned 1
    .
    MsgSeqNum : 32557
    ClOrdID : 1569433187
    OnBehalfOfCompID : 9002
    Rule80A : ^@
    Account : 
    TechnicalOrdType : ^@
    ClearingFirm : 
    ClientID : 
    FreeText : 
    OpenClose : ^\
    ClearingHandlingType : N
    NoQuoteEntries : 12336
    Maybe I'm missing something but how come it's correctly writing the first field(MsgSeqNum) but then the rest is garbage ??.

    I'm tearing my hair out and not getting anywhere : ( Any help would be greatly appreciated !!!

  2. #2
    Registered User
    Join Date
    Feb 2010
    Posts
    8
    but I'm already reading MsgLen in the first struct "utpDmessageHeader" ...

  3. #3
    and the hat of wrongness Salem's Avatar
    Join Date
    Aug 2001
    Location
    The edge of the known universe
    Posts
    32,554
    Did you notice how the size of your 2nd structure was still 80?

    > uint64_t ClOrdID;
    Your original structure aligned this on an 8-byte boundary.

    Unfortunately for you, by removing the header, this is no longer the case. So the compiler inserts padding to preserve the most efficient way of accessing the struct.

    You can either
    - add the header (either as members, or as your struct).
    - resort to various compiler "pack" options
    Structure-Packing Pragmas - Using the GNU Compiler Collection (GCC)
    Variable Attributes - Using the GNU Compiler Collection (GCC)
    If you dance barefoot on the broken glass of undefined behaviour, you've got to expect the occasional cut.
    If at first you don't succeed, try writing your phone number on the exam paper.
    I support http://www.ukip.org/ as the first necessary step to a free Europe.

  4. #4
    ZuK
    ZuK is offline
    Registered User
    Join Date
    Aug 2005
    Location
    Austria
    Posts
    1,990
    I tried this
    Code:
    #include <stdio.h>
    
    typedef unsigned short uint16_t;
    typedef unsigned int uint32_t;
    typedef unsigned long long uint64_t;
    
    typedef struct utpDquoteMessage
    {
        char messageType;
        char ProtoOrVer;
        uint16_t MsgLen;      
        uint32_t MsgSeqNum;
        uint64_t ClOrdID;
        char OnBehalfOfCompID[11];
        char Rule80A;
        char Account[12];
        char TechnicalOrdType;
        char ClearingFirm[8];
        char ClientID[8];
        char FreeText[18];
        char OpenClose;
        char ClearingHandlingType;
        char Filler1;
        uint16_t NoQuoteEntries;
    } total;
    
    typedef struct utpDmessageHeader
    {
        char messageType;
        char ProtoOrVer;
        uint16_t MsgLen;    
    } hdr;
    
    typedef struct utpDquoteMessageBody
    {      
        uint32_t MsgSeqNum;
        uint64_t ClOrdID;
        char OnBehalfOfCompID[11];
        char Rule80A;
        char Account[12];
        char TechnicalOrdType;
        char ClearingFirm[8];
        char ClientID[8];
        char FreeText[18];
        char OpenClose;
        char ClearingHandlingType;
        char Filler1;
        uint16_t NoQuoteEntries;
    } body;
    
    int main() {
        printf("%d %d %d\n", sizeof(total), sizeof(hdr), sizeof(body));
    }
    and it gives me
    Code:
    80 4 80
    seems to be a packing problem. ( I don't know if the typedefs are correct. )

    Kurt

  5. #5
    Registered User
    Join Date
    Feb 2010
    Posts
    8
    >>So the compiler inserts padding to preserve the most efficient way of accessing the struct.

    that's nice of it !!

    >>- add the header (either as members, or as your struct).


    Thanks very much for your assistance guys, at least I know what's happening now and can move forward. I'll read the info on packing then and try to figure out how best to proceed.

    : )

  6. #6
    Registered User
    Join Date
    Feb 2010
    Posts
    8

    Smile help !! : problem with binary file, and fread into structs : resolved

    Hi Guys

    I have now used the packing pragmas around my structs :

    Code:
    #pragma pack(1)
    struct utpDmessageHeader
    {
        char messageType;
        char ProtoOrVer;
        uint16_t MsgLen;    
    };
    
    struct utpDquoteMessageBody
    {         
        uint32_t MsgSeqNum;
        uint64_t ClOrdID;
        char OnBehalfOfCompID[11];
        char Rule80A;
        char Account[12];
        char TechnicalOrdType;
        char ClearingFirm[8];
        char ClientID[8];
        char FreeText[18];
        char OpenClose;
        char ClearingHandlingType;
        char Filler1;
        uint16_t NoQuoteEntries;
    };
    #pragma pack()

    ...and it works a treat :

    Code:
    [mddev@mdsaln01:/app/devel/mdscripts/PerfAgent/bin] $ gcc -o enxtUtpDirectDecoder enxtUtpDirectDecoder.c && ./enxtUtpDirectDecoder|less
    reading 4 bytes
    done.
    messagetype : B
    ProtoOrVer : 2
    MsgLen : 980
    quote.
    reading 76 bytes
    fread returned 1
    .
    MsgSeqNum : 32557
    ClOrdID : 543376761
    OnBehalfOfCompID : 00029002
    Rule80A : 6
    Account : 2
    TechnicalOrdType : ^@
    ClearingFirm : 
    ClientID : 
    FreeText : 
    OpenClose : O
    ClearingHandlingType : 0
    NoQuoteEntries : 28
    .
    .
    [mddev@mdsaln01:/app/devel/mdscripts/PerfAgent/bin] $

    you guys are the best, thanks a million !!!

Popular pages Recent additions subscribe to a feed

Similar Threads

  1. Read a file (binary) into an array of structs
    By Separ in forum C Programming
    Replies: 3
    Last Post: 04-14-2009, 09:09 PM
  2. File Writing Problem
    By polskash in forum C Programming
    Replies: 3
    Last Post: 02-13-2009, 09:47 AM
  3. Reading from binary file; fread problems?
    By pmgeahan in forum C Programming
    Replies: 3
    Last Post: 01-15-2009, 04:07 PM
  4. Problem with Binary Files I/O (fread function)
    By g33koo in forum C Programming
    Replies: 21
    Last Post: 09-30-2008, 04:56 AM
  5. problem with fread
    By GaPe in forum C Programming
    Replies: 3
    Last Post: 02-23-2003, 02:54 AM

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21