Skip to contentSkip to conversation

having probs with files > 2G

DISCUSSION Thomas Ackermann 2 participants 2 messages May 24, 2002

lore
Share & tools
lore
#0 lore raw

hi!
i hope i'm not off topic with this, but, i'having problems with some split-like program i'm writing at the moment.

when opening files > 2GB the program freezes when data is read from the file the first time. had it already working before with a buffer size of (char)1 before but that went really slow..

source code is attached, please help..

thx, thomas

#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>

void usage (char *pn)

{
	printf("\
%s - split INPUT file into a piece of\n\
SIZE1 bytes and remaining pieces of SIZE2\n\n\
Usage: %s [SIZE1 SIZE2 INPUT]\n"
			,pn,pn);
	exit(0);
}

int main (int argc, char *argv[] )

{
	char *progname;

	int fdin, fdout;
	char *buf;

	char *file;
	char *fileout;

	int ac, bc;
	int fc = 0;

	int bsize, bread;
	int btotal = 0;

	struct stat stat_buf;
	struct stat64 *stat_buf64;

	progname = argv[0];
	if (argc < 4)
		usage(progname);

	file = argv[3];
	ac = atoi(argv[1]);
	bc = atoi(argv[2]);

	fileout = malloc (sizeof(char) * (strlen(file) + 24));
	sprintf(fileout, "%s.slice-%d", file, fc);

	// open input file
	fdin = open64(file, O_RDONLY);
	if (fdin == -1)
	{
		printf("error: %s\n", strerror(errno));
		exit(0);
	}

	// adjust read buffer to an optimal size
	if (fstat(fdin, &stat_buf) < 0)
	{
		if (errno == 75)
		{
			if (fstat64(fdin, stat_buf64) < 0)
			{
				printf("error 0: %s\n", strerror(errno));
				exit(0);
			} else 
				bsize = (int) stat_buf64->st_blksize;
		}
		else
		{
			printf("error 1: %s\n", strerror(errno));
			exit(0);
		}
	} else 
		bsize = (int) stat_buf.st_blksize;

	buf = (char *) malloc (bsize + 1);
	
	// read first bytes
	bread = read(fdin, buf, bsize);

	printf("read:%d\n", bread);
	if (bread == -1)
	{
		printf("error 2: %s\n", strerror(errno));
		close(fdin);
		exit(0);
	}
	
	// if successfilly read, open output file
	fdout = open64(fileout, O_WRONLY|O_CREAT|O_TRUNC, 0666);

	// while data can be read from file
	while (bread > 0)
	{
		btotal += bread;
		write(fdout, buf, bread);

		// cut off first slice
		if ((fc > 0 && btotal >= bc) || (fc == 0 && btotal >= ac ))
		{
			fc++;
			sprintf(fileout, "%s.slice-%d", file, fc);
			close(fdout);
			fdout = open64(fileout, O_CREAT|O_WRONLY|O_TRUNC, 0666);
			btotal = 0;
		}

		bread = read(fdin, buf, bsize);
	}
	
	close(fdout);
	close(fdin);
}
#1 lore raw
Quoted 11 lines · Thomas Ackermann

hi!
i hope i'm not off topic with this, but, i'having problems with some split-like program i'm writing at the moment.

when opening files > 2GB the program freezes when data is read from the file the first time. had it already working before with a buffer size of (char)1 before but that went really slow..

source code is attached, please help..

thx, thomas

You should use
unsigned long long int var;

and

the %llu modifier in your printf's and stuffs like that...

This illustrates your problem... ------------------------[ cut here ]--------------------------- #include <stdio.h>

int main() {

unsigned long long int x = 0;
unsigned int y = 0;
x--;
y--;

  printf("New Limit:    [%llu]\nActual Limit: [%u]\n", x,y/2);

return(0);
}
------------------------[ cut here ]---------------------------

Of course you MUST NOT rely only in the %ull limit as you also have a limit in the filesystem but if you know how to deal with it, you will get everything working fine. =]

William

Signature · 2 lines
Perl combines all of the worst aspects of BASIC, C and line noise.
                -- Keith Packard