Thread: Mutlithreaded file handling

We have a client server application that is writing files to a mapped storage system. Sometimes renaming or removing a file failes, because despite fclose'ing it, the OS shows it as opened by that process.

I wrote a test tool that mimics our servers multithreaded filehandling behaviour. However, I probably have some stupid bug in it, because it fails even on my local harddisc frequently with either a "bad file descriptor" or "permission denied". I don't know how my own disc can find out I don't have permissions after the fact that I already opened and wrote almost half the file ( flushing after each record included ).

Anyway, here's the code. It's as ANSI as possible, threading is windows specific. I compiled it on VC6 and will try on VC7 today.

Run it with C:\Test or something as parameter ( make sure the folder exists ). Depending on the number of threads it will crash after about 10-50, 100-200 ( 3,2 threads ). With only one thread it did not yet fail, so maybe it's MT related.

PHP Code:

#include <stdio.h>
#include <windows.h>

volatile int        g_nThreadsRunning    = 0;
const char*            g_lpszPath            = NULL;
CRITICAL_SECTION    g_CriticalSection;    

#define RECORD_COUNT            (1024 * 10)
#define RECORD_SIZE                (1024)
#define PATH_SIZE                (1024 + 1)
#define CONCURRENT_THREADS        3

// thread:
// open file a
// write to a
// close file a
// open a for reading
// open b for writing
// write a to b
// close a
// close b
// remove a
// rename b to a
// ( remove a to clean up )
DWORD WINAPI ThreadFunction( void* pThreadnumber )
{
    char szFileInput[ PATH_SIZE ];
    char szFileOutput[ PATH_SIZE ];
    char szBuffer[ RECORD_SIZE ];
    int  nBlock;

    ZeroMemory( szBuffer, RECORD_SIZE );

    // zero data to be written

    sprintf( szFileInput,  "%s\\%d.a", g_lpszPath, (int)pThreadnumber );
    sprintf( szFileOutput, "%s\\%d.b", g_lpszPath, (int)pThreadnumber );

    printf( "%d: %s -> %s.\n", (int)pThreadnumber, szFileInput, szFileOutput );

    FILE* fpInput  = NULL;
    FILE* fpOutput = NULL;

    fpInput = fopen( szFileInput, "w" );

    if( fpInput == NULL )
    {
        printf( "Thread %d: error opening input file for initial writing: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
        exit( 0 );
    }

    for( nBlock = 0 ; nBlock < RECORD_COUNT ; nBlock++ )
    {
        if( fwrite( szBuffer, RECORD_SIZE, 1, fpInput ) != 1 )
        {
            printf( "Thread %d: error writing initial input file: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
            exit( 0 );
        }

        if( fflush( fpInput ) != 0 )
        {
            printf( "Thread %d: error flushing input file: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
            exit( 0 );
        }
    }

    fclose( fpInput );

    fpInput = fopen( szFileInput, "r" );

    if( fpInput == NULL )
    {
        printf( "Thread %d: error opening input file for reading: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
        exit( 0 );
    }

    fpOutput = fopen( szFileOutput, "w" );

    if( fpOutput == NULL )
    {
        printf( "Thread %d: error opening output file for writing: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
        exit( 0 );
    }

    for( nBlock = 0 ; nBlock < RECORD_COUNT ; nBlock++ )
    {
        if( fread( szBuffer, RECORD_SIZE, 1, fpInput ) != 1 )
        {
            printf( "Thread %d: error reading input file: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
            exit( 0 );
        }

        if( fwrite( szBuffer, RECORD_SIZE, 1, fpOutput ) != 1 )
        {
            printf( "Thread %d: error writing output file: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
            exit( 0 );
        }

        if( fflush( fpOutput ) != 0 )
        {
            printf( "Thread %d: error flushing output file: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
            exit( 0 );
        }
    }

    fclose( fpInput );
    fclose( fpOutput );

    if( remove( szFileInput ) != 0 )
    {
        printf( "Thread %d: error removing input file: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
        exit( 0 );
    }

    if( rename( szFileOutput, szFileInput ) != 0 )
    {
        printf( "Thread %d: error renaming files: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
        exit( 0 );
    }

    if( remove( szFileInput ) != 0 )
    {
        printf( "Thread %d: error removing renamed file: \n%s\n%s", (int)pThreadnumber, strerror( errno ),strerror( _doserrno ) );
        exit( 0 );
    }
            
    EnterCriticalSection( &g_CriticalSection );

    g_nThreadsRunning--;

    LeaveCriticalSection( &g_CriticalSection );

    return 0;
}

int main( int argc, char* argv[] )
{
    // Initialize the critical section one time only.
    InitializeCriticalSection( &g_CriticalSection ); 
    
    if( argc != 2 )
    {
        printf( "Usage: %s Path", argv[0] );
        return 0;
    }

    g_lpszPath = argv[1];

    int nCount = 0;

    while( true )
    {
        EnterCriticalSection( &g_CriticalSection );

        if( g_nThreadsRunning < CONCURRENT_THREADS )
        {
            DWORD dwThreadID;
            
            g_nThreadsRunning++;
            nCount++;
            
            CreateThread( NULL, 0, ThreadFunction, (void*)(nCount), 0, &dwThreadID );
        }

        LeaveCriticalSection( &g_CriticalSection );

        Sleep( 100 );
    }

    // Release resources used by the critical section object.
    DeleteCriticalSection( &g_CriticalSection );
    
    return 0;
} 


Would it fail if thread1 removes the input file just before thread 2 tries to open it? You used fopen with "r" so the file must exist or the fopen call fails. Or is it not getting that far?

The threads don't share any data or files. Each thread has two files of it's own ( in this case threadnumber.a and threadnumber.b ).

It does run for a certain time, sometimes it crashes earlier, sometimes later. It can run a whole thread or even 5 or ten or fifty without crashing. Sometimes. Sometimes not.

Except for bugs I may have put it, the number of threads should not matter as they only share a single common variable, the number of threads running.

Each thread does something like this:

// create the source from scratch
create a file.
write to it.
close it.

// copy it once
open source
open a new file as destination
copy file
close both

// remove one, put the other in it's place
remove first
rename second to first

// final cleanup ( not in original server application )
remove first

yeah should have read the sprintf more closely. As you are using c runtime funcs you should be using _beginthread rather than CreateThread. Try changing the thread spinning. Does it still fail? It really looks fine to me apart from that.

well i have tested this now using CreateThread and _beginthread linking to MTd on xp pro sp2. It runs fine and as expected in all cases on MSVC.net and MSVC.net 7.1. Maybe the problem is MSVC6 specific or OS specific? I let it run until 500-600 files had been moved.I even increases concurrent threads to 10 and it still performed as expected in all cases. Tested on as suggested c:\test with default folder permissions. Ran as an administrative user.

I changed it to use _beginthreadex and it's at 400 and counting. I guess that did the trick. I'm on XPSP2 with administrative rights btw. I did not yet test it on VC7 because the original application is compiled with VC6.

Anyway, I think that's it, thanks a lot

Uhm... no clue, where do I look that up ?
I was not using ( and not linking ) the MFC btw.

I left it running when I left yesterday evening and it's at 50K+ right now Will try on the storage system later, where errors are to be expected

You can link to MTd thats static debug multithreaded library
You can link to MT thats the release multithreaded library
You can link to MLd thats the multithreaded debug dynamic lib.
You can link to ML and thats the multithreaded release dynamic lib.

I tested your code as I said statically linking to the debug. i.e. /MTd

I had compiler errors until I linked with /MT(d) so I guess I forgot to link with /MT earlier. Maybe that had been the only problem.

Why I had compiler errors from a linker configuration when using _beginthread(ex) is beyond me though.