Cleanup: 64 bit file IO on windows.

Unlike Linux where fseek/tell will be either 32 or 64 bit
depending on the target platform, it will always be 32 bit
on windows.

We had some macro magic in BLI_winstuff.h that substituted
them for 64 bit versions, but that is upsetting the system
headers if they get included after BLI_winstuff.h which
is problematic for D6811.

This diff adds proper functions in blenlib and updates
all calls that were using the BLI_winstuff.h header to
gain 64 bit file IO.

note: Anything that was using the 32 bit versions (ie not
including BLI_winstuff.h) will still be using the 32 bit
versions, which is perhaps a good code quality Friday project.

Differential Revision: https://developer.blender.org/D7160

Reviewers: brecht dfelinto
This commit is contained in:
Ray molenkamp 2020-03-18 12:13:03 -06:00
parent ac74a843d2
commit 9a116c7c2d
9 changed files with 101 additions and 76 deletions

View File

@ -2385,7 +2385,7 @@ static int ptcache_file_header_begin_read(PTCacheFile *pf)
/* if there was an error set file as it was */
if (error) {
fseek(pf->fp, 0, SEEK_SET);
BLI_fseek(pf->fp, 0, SEEK_SET);
}
return !error;

View File

@ -70,6 +70,10 @@ typedef struct stat BLI_stat_t;
int BLI_fstat(int fd, BLI_stat_t *buffer) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
int BLI_stat(const char *path, BLI_stat_t *buffer) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
int64_t BLI_ftell(FILE *stream) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
int BLI_fseek(FILE *stream, int64_t offset, int whence);
int64_t BLI_lseek(int fd, int64_t offset, int whence);
#ifdef WIN32
int BLI_wstat(const wchar_t *path, BLI_stat_t *buffer);
#endif

View File

@ -80,13 +80,6 @@ extern "C" {
typedef unsigned int mode_t;
/* use functions that take a 64 bit offset for files larger than 4GB */
#include <stdio.h>
#define fseek(stream, offset, origin) _fseeki64(stream, offset, origin)
#define ftell(stream) _ftelli64(stream)
#define lseek(fd, offset, origin) _lseeki64(fd, offset, origin)
#define tell(fd) _telli64(fd)
#ifndef _SSIZE_T_
# define _SSIZE_T_
/* python uses HAVE_SSIZE_T */

View File

@ -175,6 +175,33 @@ double BLI_dir_free_space(const char *dir)
#endif
}
int64_t BLI_ftell(FILE *stream)
{
#ifdef WIN32
return _ftelli64(stream);
#else
return ftell(stream);
#endif
}
int BLI_fseek(FILE *stream, int64_t offset, int whence)
{
#ifdef WIN32
return _fseeki64(stream, offset, whence);
#else
return fseek(stream, offset, whence);
#endif
}
int64_t BLI_lseek(int fd, int64_t offset, int whence)
{
#ifdef WIN32
return _lseeki64(fd, offset, whence);
#else
return lseek(fd, offset, whence);
#endif
}
/**
* Returns the file size of an opened file descriptor.
*/
@ -383,15 +410,15 @@ static void *file_read_data_as_mem_impl(FILE *fp,
if (S_ISDIR(st.st_mode)) {
return NULL;
}
if (fseek(fp, 0L, SEEK_END) == -1) {
if (BLI_fseek(fp, 0L, SEEK_END) == -1) {
return NULL;
}
/* Don't use the 'st_size' because it may be the symlink. */
const long int filelen = ftell(fp);
const long int filelen = BLI_ftell(fp);
if (filelen == -1) {
return NULL;
}
if (fseek(fp, 0L, SEEK_SET) == -1) {
if (BLI_fseek(fp, 0L, SEEK_SET) == -1) {
return NULL;
}
@ -519,9 +546,9 @@ LinkNode *BLI_file_read_as_lines(const char *name)
return NULL;
}
fseek(fp, 0, SEEK_END);
size = (size_t)ftell(fp);
fseek(fp, 0, SEEK_SET);
BLI_fseek(fp, 0, SEEK_END);
size = (size_t)BLI_ftell(fp);
BLI_fseek(fp, 0, SEEK_SET);
if (UNLIKELY(size == (size_t)-1)) {
fclose(fp);

View File

@ -1171,7 +1171,7 @@ static int fd_read_data_from_file(FileData *filedata,
static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence)
{
filedata->file_offset = lseek(filedata->filedes, offset, whence);
filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence);
return filedata->file_offset;
}
@ -1349,7 +1349,7 @@ static FileData *blo_filedata_from_file_descriptor(const char *filepath,
return NULL;
}
else {
lseek(file, 0, SEEK_SET);
BLI_lseek(file, 0, SEEK_SET);
}
/* Regular file. */

View File

@ -249,7 +249,7 @@ bool AVI_is_avi(const char *name)
movie.header->Reserved[2] = GET_FCC(movie.fp);
movie.header->Reserved[3] = GET_FCC(movie.fp);
fseek(movie.fp, movie.header->size - 14 * 4, SEEK_CUR);
BLI_fseek(movie.fp, movie.header->size - 14 * 4, SEEK_CUR);
/* Limit number of streams to some reasonable amount to prevent
* buffer overflow vulnerabilities. */
@ -312,7 +312,7 @@ bool AVI_is_avi(const char *name)
movie.streams[temp].sh.right = GET_TCC(movie.fp);
movie.streams[temp].sh.bottom = GET_TCC(movie.fp);
fseek(movie.fp, movie.streams[temp].sh.size - 14 * 4, SEEK_CUR);
BLI_fseek(movie.fp, movie.streams[temp].sh.size - 14 * 4, SEEK_CUR);
if (GET_FCC(movie.fp) != FCC("strf")) {
DEBUG_PRINT("no stream format information\n");
@ -362,27 +362,27 @@ bool AVI_is_avi(const char *name)
}
}
if (j > 0) {
fseek(movie.fp, j, SEEK_CUR);
BLI_fseek(movie.fp, j, SEEK_CUR);
}
}
else {
fseek(movie.fp, movie.streams[temp].sf_size, SEEK_CUR);
BLI_fseek(movie.fp, movie.streams[temp].sf_size, SEEK_CUR);
}
/* Walk to the next LIST */
while (GET_FCC(movie.fp) != FCC("LIST")) {
temp = GET_FCC(movie.fp);
if (temp < 0 || ftell(movie.fp) > movie.size) {
if (temp < 0 || BLI_ftell(movie.fp) > movie.size) {
DEBUG_PRINT("incorrect size in header or error in AVI\n");
MEM_freeN(movie.streams);
fclose(movie.fp);
return 0;
}
fseek(movie.fp, temp, SEEK_CUR);
BLI_fseek(movie.fp, temp, SEEK_CUR);
}
fseek(movie.fp, -4L, SEEK_CUR);
BLI_fseek(movie.fp, -4L, SEEK_CUR);
}
MEM_freeN(movie.streams);
@ -437,7 +437,7 @@ AviError AVI_open_movie(const char *name, AviMovie *movie)
movie->header->Reserved[2] = GET_FCC(movie->fp);
movie->header->Reserved[3] = GET_FCC(movie->fp);
fseek(movie->fp, movie->header->size - 14 * 4, SEEK_CUR);
BLI_fseek(movie->fp, movie->header->size - 14 * 4, SEEK_CUR);
/* Limit number of streams to some reasonable amount to prevent
* buffer overflow vulnerabilities. */
@ -493,7 +493,7 @@ AviError AVI_open_movie(const char *name, AviMovie *movie)
movie->streams[temp].sh.right = GET_TCC(movie->fp);
movie->streams[temp].sh.bottom = GET_TCC(movie->fp);
fseek(movie->fp, movie->streams[temp].sh.size - 14 * 4, SEEK_CUR);
BLI_fseek(movie->fp, movie->streams[temp].sh.size - 14 * 4, SEEK_CUR);
if (GET_FCC(movie->fp) != FCC("strf")) {
DEBUG_PRINT("no stream format information\n");
@ -540,24 +540,24 @@ AviError AVI_open_movie(const char *name, AviMovie *movie)
}
}
if (j > 0) {
fseek(movie->fp, j, SEEK_CUR);
BLI_fseek(movie->fp, j, SEEK_CUR);
}
}
else {
fseek(movie->fp, movie->streams[temp].sf_size, SEEK_CUR);
BLI_fseek(movie->fp, movie->streams[temp].sf_size, SEEK_CUR);
}
/* Walk to the next LIST */
while (GET_FCC(movie->fp) != FCC("LIST")) {
temp = GET_FCC(movie->fp);
if (temp < 0 || ftell(movie->fp) > movie->size) {
if (temp < 0 || BLI_ftell(movie->fp) > movie->size) {
DEBUG_PRINT("incorrect size in header or error in AVI\n");
return AVI_ERROR_FORMAT;
}
fseek(movie->fp, temp, SEEK_CUR);
BLI_fseek(movie->fp, temp, SEEK_CUR);
}
fseek(movie->fp, -4L, SEEK_CUR);
BLI_fseek(movie->fp, -4L, SEEK_CUR);
}
while (1) {
@ -573,24 +573,24 @@ AviError AVI_open_movie(const char *name, AviMovie *movie)
break;
}
else {
fseek(movie->fp, size - 4, SEEK_CUR);
BLI_fseek(movie->fp, size - 4, SEEK_CUR);
}
}
else {
fseek(movie->fp, size, SEEK_CUR);
BLI_fseek(movie->fp, size, SEEK_CUR);
}
if (ftell(movie->fp) > movie->size) {
if (BLI_ftell(movie->fp) > movie->size) {
DEBUG_PRINT("incorrect size in header or error in AVI\n");
return AVI_ERROR_FORMAT;
}
}
movie->movi_offset = ftell(movie->fp);
movie->movi_offset = BLI_ftell(movie->fp);
movie->read_offset = movie->movi_offset;
/* Read in the index if the file has one, otherwise create one */
if (movie->header->Flags & AVIF_HASINDEX) {
fseek(movie->fp, size - 4, SEEK_CUR);
BLI_fseek(movie->fp, size - 4, SEEK_CUR);
if (GET_FCC(movie->fp) != FCC("idx1")) {
DEBUG_PRINT("bad index informatio\n");
@ -669,7 +669,7 @@ void *AVI_read_frame(AviMovie *movie, AviFormat format, int frame, int stream)
return NULL;
}
fseek(movie->fp, movie->read_offset + movie->entries[i - 1].Offset, SEEK_SET);
BLI_fseek(movie->fp, movie->read_offset + movie->entries[i - 1].Offset, SEEK_SET);
size_t size = GET_FCC(movie->fp);
buffer = MEM_mallocN(size, "readbuffer");
@ -834,9 +834,9 @@ AviError AVI_open_compress(char *name, AviMovie *movie, int streams, ...)
awrite(movie, &list, 1, sizeof(AviList), movie->fp, AVI_LIST);
header_pos1 = ftell(movie->fp);
header_pos1 = BLI_ftell(movie->fp);
movie->offset_table[0] = ftell(movie->fp);
movie->offset_table[0] = BLI_ftell(movie->fp);
awrite(movie, movie->header, 1, sizeof(AviMainHeader), movie->fp, AVI_MAINH);
@ -847,24 +847,24 @@ AviError AVI_open_compress(char *name, AviMovie *movie, int streams, ...)
awrite(movie, &list, 1, sizeof(AviList), movie->fp, AVI_LIST);
stream_pos1 = ftell(movie->fp);
stream_pos1 = BLI_ftell(movie->fp);
movie->offset_table[1 + i * 2] = ftell(movie->fp);
movie->offset_table[1 + i * 2] = BLI_ftell(movie->fp);
awrite(movie, &movie->streams[i].sh, 1, sizeof(AviStreamHeader), movie->fp, AVI_STREAMH);
movie->offset_table[1 + i * 2 + 1] = ftell(movie->fp);
movie->offset_table[1 + i * 2 + 1] = BLI_ftell(movie->fp);
awrite(movie, movie->streams[i].sf, 1, movie->streams[i].sf_size, movie->fp, AVI_BITMAPH);
stream_pos2 = ftell(movie->fp);
stream_pos2 = BLI_ftell(movie->fp);
fseek(movie->fp, stream_pos1 - 8, SEEK_SET);
BLI_fseek(movie->fp, stream_pos1 - 8, SEEK_SET);
PUT_FCCN((stream_pos2 - stream_pos1 + 4L), movie->fp);
fseek(movie->fp, stream_pos2, SEEK_SET);
BLI_fseek(movie->fp, stream_pos2, SEEK_SET);
}
junk_pos = ftell(movie->fp);
junk_pos = BLI_ftell(movie->fp);
if (junk_pos < 2024 - 8) {
chunk.fcc = FCC("JUNK");
@ -877,7 +877,7 @@ AviError AVI_open_compress(char *name, AviMovie *movie, int streams, ...)
}
}
header_pos2 = ftell(movie->fp);
header_pos2 = BLI_ftell(movie->fp);
list.fcc = FCC("LIST");
list.size = 0;
@ -885,9 +885,9 @@ AviError AVI_open_compress(char *name, AviMovie *movie, int streams, ...)
awrite(movie, &list, 1, sizeof(AviList), movie->fp, AVI_LIST);
movie->movi_offset = ftell(movie->fp) - 8L;
movie->movi_offset = BLI_ftell(movie->fp) - 8L;
fseek(movie->fp, AVI_HDRL_SOFF, SEEK_SET);
BLI_fseek(movie->fp, AVI_HDRL_SOFF, SEEK_SET);
PUT_FCCN((header_pos2 - header_pos1 + 4L), movie->fp);
@ -920,7 +920,7 @@ AviError AVI_write_frame(AviMovie *movie, int frame_num, ...)
/* Slap a new record entry onto the end of the file */
fseek(movie->fp, 0L, SEEK_END);
BLI_fseek(movie->fp, 0L, SEEK_END);
list.fcc = FCC("LIST");
list.size = 0;
@ -928,7 +928,7 @@ AviError AVI_write_frame(AviMovie *movie, int frame_num, ...)
awrite(movie, &list, 1, sizeof(AviList), movie->fp, AVI_LIST);
rec_off = ftell(movie->fp) - 8L;
rec_off = BLI_ftell(movie->fp) - 8L;
/* Write a frame for every stream */
@ -947,7 +947,7 @@ AviError AVI_write_frame(AviMovie *movie, int frame_num, ...)
/* Write the header info for this data chunk */
fseek(movie->fp, 0L, SEEK_END);
BLI_fseek(movie->fp, 0L, SEEK_END);
chunk.fcc = avi_get_data_id(format, stream);
chunk.size = size;
@ -963,7 +963,7 @@ AviError AVI_write_frame(AviMovie *movie, int frame_num, ...)
movie->entries[frame_num * (movie->header->Streams + 1) + stream + 1].ChunkId = chunk.fcc;
movie->entries[frame_num * (movie->header->Streams + 1) + stream + 1].Flags = AVIIF_KEYFRAME;
movie->entries[frame_num * (movie->header->Streams + 1) + stream + 1].Offset =
(int)(ftell(movie->fp) - 12L - movie->movi_offset);
(int)(BLI_ftell(movie->fp) - 12L - movie->movi_offset);
movie->entries[frame_num * (movie->header->Streams + 1) + stream + 1].Size = chunk.size;
/* Write the chunk */
@ -976,29 +976,29 @@ AviError AVI_write_frame(AviMovie *movie, int frame_num, ...)
/* Update the stream headers length field */
movie->streams[stream].sh.Length++;
fseek(movie->fp, movie->offset_table[1 + stream * 2], SEEK_SET);
BLI_fseek(movie->fp, movie->offset_table[1 + stream * 2], SEEK_SET);
awrite(movie, &movie->streams[stream].sh, 1, sizeof(AviStreamHeader), movie->fp, AVI_STREAMH);
}
va_end(ap);
/* Record the entry for the new record */
fseek(movie->fp, 0L, SEEK_END);
BLI_fseek(movie->fp, 0L, SEEK_END);
movie->entries[frame_num * (movie->header->Streams + 1)].ChunkId = FCC("rec ");
movie->entries[frame_num * (movie->header->Streams + 1)].Flags = AVIIF_LIST;
movie->entries[frame_num * (movie->header->Streams + 1)].Offset = (int)(rec_off - 8L -
movie->movi_offset);
movie->entries[frame_num * (movie->header->Streams + 1)].Size = (int)(ftell(movie->fp) -
movie->entries[frame_num * (movie->header->Streams + 1)].Size = (int)(BLI_ftell(movie->fp) -
(rec_off + 4L));
/* Update the record size */
fseek(movie->fp, rec_off, SEEK_SET);
BLI_fseek(movie->fp, rec_off, SEEK_SET);
PUT_FCCN(movie->entries[frame_num * (movie->header->Streams + 1)].Size, movie->fp);
/* Update the main header information in the file */
movie->header->TotalFrames++;
fseek(movie->fp, movie->offset_table[0], SEEK_SET);
BLI_fseek(movie->fp, movie->offset_table[0], SEEK_SET);
awrite(movie, movie->header, 1, sizeof(AviMainHeader), movie->fp, AVI_MAINH);
return AVI_ERROR_NONE;
@ -1013,8 +1013,8 @@ AviError AVI_close_compress(AviMovie *movie)
return AVI_ERROR_FOUND;
}
fseek(movie->fp, 0L, SEEK_END);
movi_size = (int)ftell(movie->fp);
BLI_fseek(movie->fp, 0L, SEEK_END);
movi_size = (int)BLI_ftell(movie->fp);
PUT_FCC("idx1", movie->fp);
PUT_FCCN((movie->index_entries * (movie->header->Streams + 1) * 16), movie->fp);
@ -1023,13 +1023,13 @@ AviError AVI_close_compress(AviMovie *movie)
awrite(movie, &movie->entries[temp], 1, sizeof(AviIndexEntry), movie->fp, AVI_INDEXE);
}
temp = (int)ftell(movie->fp);
temp = (int)BLI_ftell(movie->fp);
fseek(movie->fp, AVI_RIFF_SOFF, SEEK_SET);
BLI_fseek(movie->fp, AVI_RIFF_SOFF, SEEK_SET);
PUT_FCCN((temp - 8L), movie->fp);
fseek(movie->fp, movie->movi_offset, SEEK_SET);
BLI_fseek(movie->fp, movie->movi_offset, SEEK_SET);
PUT_FCCN((movi_size - (movie->movi_offset + 4L)), movie->fp);

View File

@ -27,6 +27,7 @@
#include "AVI_avi.h"
#include "avi_intern.h"
#include "avi_endian.h"
#include "BLI_fileops.h"
#ifdef WIN32
# include "BLI_winstuff.h"
@ -61,7 +62,7 @@ AviError AVI_set_compress_option(
movie->streams[i].sh.right = *((int *)opt_data);
((AviBitmapInfoHeader *)movie->streams[i].sf)->SizeImage =
movie->header->SuggestedBufferSize;
fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
BLI_fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
awrite(movie,
movie->streams[i].sf,
1,
@ -84,7 +85,7 @@ AviError AVI_set_compress_option(
movie->streams[i].sh.bottom = *((int *)opt_data);
((AviBitmapInfoHeader *)movie->streams[i].sf)->SizeImage =
movie->header->SuggestedBufferSize;
fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
BLI_fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
awrite(movie,
movie->streams[i].sf,
1,
@ -100,7 +101,7 @@ AviError AVI_set_compress_option(
for (i = 0; i < movie->header->Streams; i++) {
if (avi_get_format_type(movie->streams[i].format) == FCC("vids")) {
movie->streams[i].sh.Quality = (*((int *)opt_data)) * 100;
fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
BLI_fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
awrite(movie,
movie->streams[i].sf,
1,
@ -120,7 +121,7 @@ AviError AVI_set_compress_option(
for (i = 0; i < movie->header->Streams; i++) {
if (avi_get_format_type(movie->streams[i].format) == FCC("vids")) {
movie->streams[i].sh.Scale = movie->header->MicroSecPerFrame;
fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
BLI_fseek(movie->fp, movie->offset_table[1 + i * 2 + 1], SEEK_SET);
awrite(movie,
movie->streams[i].sf,
1,
@ -132,7 +133,7 @@ AviError AVI_set_compress_option(
break;
}
fseek(movie->fp, movie->offset_table[0], SEEK_SET);
BLI_fseek(movie->fp, movie->offset_table[0], SEEK_SET);
awrite(movie, movie->header, 1, sizeof(AviMainHeader), movie->fp, AVI_MAINH);
break;

View File

@ -155,12 +155,12 @@ bool MOD_meshcache_read_mdd_index(FILE *fp,
return false;
}
if (fseek(fp, mdd_head.frame_tot * sizeof(int), SEEK_CUR) != 0) {
if (BLI_fseek(fp, mdd_head.frame_tot * sizeof(int), SEEK_CUR) != 0) {
*err_str = "Header seek failed";
return false;
}
if (fseek(fp, sizeof(float) * 3 * index * mdd_head.verts_tot, SEEK_CUR) != 0) {
if (BLI_fseek(fp, sizeof(float) * 3 * index * mdd_head.verts_tot, SEEK_CUR) != 0) {
*err_str = "Failed to seek frame";
return false;
}
@ -234,7 +234,7 @@ bool MOD_meshcache_read_mdd_frame(FILE *fp,
if (index_range[0] == index_range[1]) {
/* read single */
if ((fseek(fp, 0, SEEK_SET) == 0) &&
if ((BLI_fseek(fp, 0, SEEK_SET) == 0) &&
MOD_meshcache_read_mdd_index(fp, vertexCos, verts_tot, index_range[0], 1.0f, err_str)) {
return true;
}
@ -244,9 +244,9 @@ bool MOD_meshcache_read_mdd_frame(FILE *fp,
}
else {
/* read both and interpolate */
if ((fseek(fp, 0, SEEK_SET) == 0) &&
if ((BLI_fseek(fp, 0, SEEK_SET) == 0) &&
MOD_meshcache_read_mdd_index(fp, vertexCos, verts_tot, index_range[0], 1.0f, err_str) &&
(fseek(fp, 0, SEEK_SET) == 0) &&
(BLI_fseek(fp, 0, SEEK_SET) == 0) &&
MOD_meshcache_read_mdd_index(fp, vertexCos, verts_tot, index_range[1], factor, err_str)) {
return true;
}

View File

@ -146,7 +146,7 @@ bool MOD_meshcache_read_pc2_index(FILE *fp,
return false;
}
if (fseek(fp, sizeof(float) * 3 * index * pc2_head.verts_tot, SEEK_CUR) != 0) {
if (BLI_fseek(fp, sizeof(float) * 3 * index * pc2_head.verts_tot, SEEK_CUR) != 0) {
*err_str = "Failed to seek frame";
return false;
}
@ -209,7 +209,7 @@ bool MOD_meshcache_read_pc2_frame(FILE *fp,
if (index_range[0] == index_range[1]) {
/* read single */
if ((fseek(fp, 0, SEEK_SET) == 0) &&
if ((BLI_fseek(fp, 0, SEEK_SET) == 0) &&
MOD_meshcache_read_pc2_index(fp, vertexCos, verts_tot, index_range[0], 1.0f, err_str)) {
return true;
}
@ -219,9 +219,9 @@ bool MOD_meshcache_read_pc2_frame(FILE *fp,
}
else {
/* read both and interpolate */
if ((fseek(fp, 0, SEEK_SET) == 0) &&
if ((BLI_fseek(fp, 0, SEEK_SET) == 0) &&
MOD_meshcache_read_pc2_index(fp, vertexCos, verts_tot, index_range[0], 1.0f, err_str) &&
(fseek(fp, 0, SEEK_SET) == 0) &&
(BLI_fseek(fp, 0, SEEK_SET) == 0) &&
MOD_meshcache_read_pc2_index(fp, vertexCos, verts_tot, index_range[1], factor, err_str)) {
return true;
}