> Im trying to speed up the reading process so it can read ~200Mb files in onle a couple of seconds.
Unless you've got the latest kit, I doubt you'll get anywhere near that.
> Sine the buffer on most machines is quite small, around 512Kb
Says who?
And which buffer?
You can't know in general how many buffers there are, let alone how big they are.
What are you intending to do with the data once you're read it?
If its something simple like
Code:
for ( i = 0 ; i < len ; i++ ) {
sum += buff[i];
}
Then its a waste of time and resource to allocate memory for the whole file.
A couple of ideas for you
Code:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#define BSIZE BUFSIZ*100
void calcs ( unsigned char *buff, size_t len ) {
int sum = 0;
size_t i;
for ( i = 0 ; i < len ; i++ ) {
sum += buff[i];
}
printf( "Sum=%d\n", sum );
}
void exp1 ( char *filename ) {
FILE *fp;
fp = fopen( filename, "rb" );
if ( fp ) {
long len;
unsigned char *mem;
fseek( fp, 0, SEEK_END );
len = ftell(fp);
fseek( fp, 0, SEEK_SET );
printf( "Len=%ld\n", len );
mem = malloc( len );
if ( mem ) {
int n;
unsigned char *p = mem;
while ( (n=fread(p,1,BSIZE,fp)) > 0 ) {
p += n;
}
calcs( mem, len );
free(mem);
}
fclose(fp);
}
}
void exp2 ( char *filename ) {
int fd;
fd = open( filename, O_RDONLY );
if ( fd != -1 ) {
off_t len;
unsigned char *mem;
len = lseek( fd, 0, SEEK_END );
mem = mmap( 0, len, PROT_READ, MAP_SHARED, fd, 0 ); /* map the whole file */
if ( mem != NULL ) {
calcs( mem, len );
munmap( mem, len );
} else {
perror("oops");
}
}
}
int main ( int argc, char *argv[] ) {
printf( "%d\n", clock() );
exp1( argv[1] );
printf( "%d\n", clock() );
exp2( argv[1] );
printf( "%d\n", clock() );
return 0;
}
For me at least, the mmap() solution is twice as quick as fread()ing large blocks.
EDIT
So you are just touching each byte once. It's a waste of time reading the whole file into a huge buffer. Use a small buffer about BUFSIZ in size and keep freading into that and performing next block of your CRC