Refactor low-level blendfile reading into separate files

Instead of handling mmap, compression etc. all directly in readfile.c, refactor
the code to use a generic FileReader.
This makes it easier to add new compression methods or similar, and allows to
reuse the logic in other places (e.g. thumbnail reading).

Reviewed By: campbellbarton, brecht, mont29

Differential Revision: https://developer.blender.org/D5799
This commit is contained in:
Lukas Stockner 2021-08-19 23:57:00 +02:00
parent 34a05f39be
commit 2b170f16d6
15 changed files with 673 additions and 450 deletions

View File

@ -166,6 +166,8 @@ size_t BLI_gzip_mem_to_file_at_pos(void *buf,
int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
bool BLI_file_magic_is_gzip(const char header[4]);
size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();

View File

@ -0,0 +1,79 @@
/*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
* The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
* All rights reserved.
*/
/** \file
* \ingroup bli
* \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...).
*/
#pragma once
#ifdef WIN32
# include "BLI_winstuff.h"
#else
# include <sys/types.h>
#endif
#include "BLI_compiler_attrs.h"
#include "BLI_utildefines.h"
#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
typedef int64_t off64_t;
#endif
#ifdef __cplusplus
extern "C" {
#endif
struct FileReader;
typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size);
typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence);
typedef void (*FileReaderCloseFn)(struct FileReader *reader);
/* General structure for all FileReaders, implementations add custom fields at the end. */
typedef struct FileReader {
FileReaderReadFn read;
FileReaderSeekFn seek;
FileReaderCloseFn close;
off64_t offset;
} FileReader;
/* Functions for opening the various types of FileReader.
* They either succeed and return a valid FileReader, or fail and return NULL.
*
* If a FileReader is created, it has to be cleaned up and freed by calling
* its close() function unless another FileReader has taken ownership - for example,
* Gzip takes over the base FileReader and will clean it up when their clean() is called.
*/
/* Create FileReader from raw file descriptor. */
FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT;
/* Create FileReader from raw file descriptor using memory-mapped IO. */
FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
/* Create FileReader from a region of memory. */
FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
ATTR_NONNULL();
/* Create FileReader from applying Gzip decompression on an underlying file. */
FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
#ifdef __cplusplus
}
#endif

View File

@ -75,6 +75,9 @@ set(SRC
intern/endian_switch.c
intern/expr_pylike_eval.c
intern/fileops.c
intern/filereader_file.c
intern/filereader_gzip.c
intern/filereader_memory.c
intern/fnmatch.c
intern/freetypefont.c
intern/gsqueue.c
@ -194,6 +197,7 @@ set(SRC
BLI_enumerable_thread_specific.hh
BLI_expr_pylike_eval.h
BLI_fileops.h
BLI_filereader.h
BLI_fileops_types.h
BLI_float2.hh
BLI_float3.hh

View File

@ -255,6 +255,13 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
#undef CHUNK
bool BLI_file_magic_is_gzip(const char header[4])
{
/* GZIP itself starts with the magic bytes 0x1f 0x8b.
* The third byte indicates the compression method, which is 0x08 for DEFLATE. */
return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
}
/**
* Returns true if the file with the specified name can be written.
* This implementation uses access(2), which makes the check according

View File

@ -0,0 +1,80 @@
/*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
* All rights reserved.
*/
/** \file
* \ingroup bli
*/
#ifndef WIN32
# include <unistd.h> /* for read close */
#else
# include "BLI_winstuff.h"
# include "winsock2.h"
# include <io.h> /* for open close read */
#endif
#include "BLI_blenlib.h"
#include "BLI_filereader.h"
#include "MEM_guardedalloc.h"
typedef struct {
FileReader reader;
int filedes;
} RawFileReader;
static ssize_t file_read(FileReader *reader, void *buffer, size_t size)
{
RawFileReader *rawfile = (RawFileReader *)reader;
ssize_t readsize = read(rawfile->filedes, buffer, size);
if (readsize >= 0) {
rawfile->reader.offset += readsize;
}
return readsize;
}
static off64_t file_seek(FileReader *reader, off64_t offset, int whence)
{
RawFileReader *rawfile = (RawFileReader *)reader;
rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence);
return rawfile->reader.offset;
}
static void file_close(FileReader *reader)
{
RawFileReader *rawfile = (RawFileReader *)reader;
close(rawfile->filedes);
MEM_freeN(rawfile);
}
FileReader *BLI_filereader_new_file(int filedes)
{
RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__);
rawfile->filedes = filedes;
rawfile->reader.read = file_read;
rawfile->reader.seek = file_seek;
rawfile->reader.close = file_close;
return (FileReader *)rawfile;
}

View File

@ -0,0 +1,108 @@
/*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
* All rights reserved.
*/
/** \file
* \ingroup bli
*/
#include <zlib.h>
#include "BLI_blenlib.h"
#include "BLI_filereader.h"
#include "MEM_guardedalloc.h"
typedef struct {
FileReader reader;
FileReader *base;
z_stream strm;
void *in_buf;
size_t in_size;
} GzipReader;
static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size)
{
GzipReader *gzip = (GzipReader *)reader;
gzip->strm.avail_out = size;
gzip->strm.next_out = buffer;
while (gzip->strm.avail_out > 0) {
if (gzip->strm.avail_in == 0) {
/* Ran out of buffered input data, read some more. */
size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size);
if (readsize > 0) {
/* We got some data, so mark the buffer as refilled. */
gzip->strm.avail_in = readsize;
gzip->strm.next_in = gzip->in_buf;
}
else {
/* The underlying file is EOF, so return as much as we can. */
break;
}
}
int ret = inflate(&gzip->strm, Z_NO_FLUSH);
if (ret != Z_OK && ret != Z_BUF_ERROR) {
break;
}
}
ssize_t read_len = size - gzip->strm.avail_out;
gzip->reader.offset += read_len;
return read_len;
}
static void gzip_close(FileReader *reader)
{
GzipReader *gzip = (GzipReader *)reader;
if (inflateEnd(&gzip->strm) != Z_OK) {
printf("close gzip stream error\n");
}
MEM_freeN((void *)gzip->in_buf);
gzip->base->close(gzip->base);
MEM_freeN(gzip);
}
FileReader *BLI_filereader_new_gzip(FileReader *base)
{
GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__);
gzip->base = base;
if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) {
MEM_freeN(gzip);
return NULL;
}
gzip->in_size = 256 * 2014;
gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf");
gzip->reader.read = gzip_read;
gzip->reader.seek = NULL;
gzip->reader.close = gzip_close;
return (FileReader *)gzip;
}

View File

@ -0,0 +1,145 @@
/*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
* All rights reserved.
*/
/** \file
* \ingroup bli
*/
#include <string.h>
#include "BLI_blenlib.h"
#include "BLI_filereader.h"
#include "BLI_mmap.h"
#include "MEM_guardedalloc.h"
/* This file implements both memory-backed and memory-mapped-file-backed reading. */
typedef struct {
FileReader reader;
const char *data;
BLI_mmap_file *mmap;
size_t length;
} MemoryReader;
static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size)
{
MemoryReader *mem = (MemoryReader *)reader;
/* Don't read more bytes than there are available in the buffer. */
size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
memcpy(buffer, mem->data + mem->reader.offset, readsize);
mem->reader.offset += readsize;
return readsize;
}
static off64_t memory_seek(FileReader *reader, off64_t offset, int whence)
{
MemoryReader *mem = (MemoryReader *)reader;
off64_t new_pos;
if (whence == SEEK_CUR) {
new_pos = mem->reader.offset + offset;
}
else if (whence == SEEK_SET) {
new_pos = offset;
}
else if (whence == SEEK_END) {
new_pos = mem->length + offset;
}
else {
return -1;
}
if (new_pos < 0 || new_pos > mem->length) {
return -1;
}
mem->reader.offset = new_pos;
return mem->reader.offset;
}
static void memory_close_raw(FileReader *reader)
{
MEM_freeN(reader);
}
FileReader *BLI_filereader_new_memory(const void *data, size_t len)
{
MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
mem->data = (const char *)data;
mem->length = len;
mem->reader.read = memory_read_raw;
mem->reader.seek = memory_seek;
mem->reader.close = memory_close_raw;
return (FileReader *)mem;
}
/* Memory-mapped file reading.
* By using `mmap()`, we can map a file so that it can be treated like normal memory,
* meaning that we can just read from it with `memcpy()` etc.
* This avoids system call overhead and can significantly speed up file loading.
*/
static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size)
{
MemoryReader *mem = (MemoryReader *)reader;
/* Don't read more bytes than there are available in the buffer. */
size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) {
return 0;
}
mem->reader.offset += readsize;
return readsize;
}
static void memory_close_mmap(FileReader *reader)
{
MemoryReader *mem = (MemoryReader *)reader;
BLI_mmap_free(mem->mmap);
MEM_freeN(mem);
}
FileReader *BLI_filereader_new_mmap(int filedes)
{
BLI_mmap_file *mmap = BLI_mmap_open(filedes);
if (mmap == NULL) {
return NULL;
}
MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
mem->mmap = mmap;
mem->length = BLI_lseek(filedes, 0, SEEK_END);
mem->reader.read = memory_read_mmap;
mem->reader.seek = memory_seek;
mem->reader.close = memory_close_mmap;
return (FileReader *)mem;
}

View File

@ -24,6 +24,8 @@
* \ingroup blenloader
*/
#include "BLI_filereader.h"
struct GHash;
struct Scene;
@ -65,6 +67,16 @@ typedef struct MemFileUndoData {
size_t undo_size;
} MemFileUndoData;
/* FileReader-compatible wrapper for reading MemFiles */
typedef struct {
FileReader reader;
MemFile *memfile;
int undo_direction;
bool memchunk_identical;
} UndoReader;
/* actually only used writefile.c */
void BLO_memfile_write_init(MemFileWriteData *mem_data,
@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile,
struct Main *bmain,
struct Scene **r_scene);
extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename);
FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction);

View File

@ -21,8 +21,6 @@
* \ingroup blenloader
*/
#include "zlib.h"
#include <ctype.h> /* for isdigit. */
#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
#include <limits.h>
@ -71,7 +69,6 @@
#include "BLI_math.h"
#include "BLI_memarena.h"
#include "BLI_mempool.h"
#include "BLI_mmap.h"
#include "BLI_threads.h"
#include "PIL_time.h"
@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd)
*/
if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) {
bhead4.code = DATA;
readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL);
readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4));
if (readsize == sizeof(bhead4) || bhead4.code == ENDB) {
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd)
}
else {
bhead8.code = DATA;
readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL);
readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8));
if (readsize == sizeof(bhead8) || bhead8.code == ENDB) {
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd)
/* pass */
}
#ifdef USE_BHEAD_READ_ON_DEMAND
else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
/* Delay reading bhead content. */
new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead");
if (new_bhead) {
new_bhead->next = new_bhead->prev = NULL;
new_bhead->file_offset = fd->file_offset;
new_bhead->file_offset = fd->file->offset;
new_bhead->has_data = false;
new_bhead->is_memchunk_identical = false;
new_bhead->bhead = bhead;
off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR);
off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR);
if (seek_new == -1) {
fd->is_eof = true;
MEM_freeN(new_bhead);
new_bhead = NULL;
}
BLI_assert(fd->file_offset == seek_new);
BLI_assert(fd->file->offset == seek_new);
}
else {
fd->is_eof = true;
@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd)
new_bhead->is_memchunk_identical = false;
new_bhead->bhead = bhead;
readsize = fd->read(
fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical);
readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len);
if (readsize != (ssize_t)bhead.len) {
if (readsize != bhead.len) {
fd->is_eof = true;
MEM_freeN(new_bhead);
new_bhead = NULL;
}
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
}
}
else {
fd->is_eof = true;
@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
bool success = true;
BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock);
BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0);
off64_t offset_backup = fd->file_offset;
if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) {
off64_t offset_backup = fd->file->offset;
if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) {
success = false;
}
else {
if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
(ssize_t)new_bhead->bhead.len) {
if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) {
success = false;
}
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
}
}
if (fd->seek(fd, offset_backup, SEEK_SET) == -1) {
if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) {
success = false;
}
return success;
@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd)
ssize_t readsize;
/* read in the header data */
readsize = fd->read(fd, header, sizeof(header), NULL);
readsize = fd->file->read(fd->file, header, sizeof(header));
if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') &&
ELEM(header[8], 'v', 'V') &&
@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd)
/** \} */
/* -------------------------------------------------------------------- */
/** \name File Data API
* \{ */
/* Regular file reading. */
static ssize_t fd_read_data_from_file(FileData *filedata,
void *buffer,
size_t size,
bool *UNUSED(r_is_memchunck_identical))
{
ssize_t readsize = read(filedata->filedes, buffer, size);
if (readsize < 0) {
readsize = EOF;
}
else {
filedata->file_offset += readsize;
}
return readsize;
}
static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence)
{
filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence);
return filedata->file_offset;
}
/* GZip file reading. */
static ssize_t fd_read_gzip_from_file(FileData *filedata,
void *buffer,
size_t size,
bool *UNUSED(r_is_memchunck_identical))
{
BLI_assert(size <= INT_MAX);
ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size);
if (readsize < 0) {
readsize = EOF;
}
else {
filedata->file_offset += readsize;
}
return readsize;
}
/* Memory reading. */
static ssize_t fd_read_from_memory(FileData *filedata,
void *buffer,
size_t size,
bool *UNUSED(r_is_memchunck_identical))
{
/* don't read more bytes than there are available in the buffer */
ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset);
memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize);
filedata->file_offset += readsize;
return readsize;
}
/* Memory-mapped file reading.
* By using mmap(), we can map a file so that it can be treated like normal memory,
* meaning that we can just read from it with memcpy() etc.
* This avoids system call overhead and can significantly speed up file loading.
*/
static ssize_t fd_read_from_mmap(FileData *filedata,
void *buffer,
size_t size,
bool *UNUSED(r_is_memchunck_identical))
{
/* don't read more bytes than there are available in the buffer */
size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset));
if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) {
return 0;
}
filedata->file_offset += readsize;
return readsize;
}
static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence)
{
off64_t new_pos;
if (whence == SEEK_CUR) {
new_pos = filedata->file_offset + offset;
}
else if (whence == SEEK_SET) {
new_pos = offset;
}
else if (whence == SEEK_END) {
new_pos = filedata->buffersize + offset;
}
else {
return -1;
}
if (new_pos < 0 || new_pos > filedata->buffersize) {
return -1;
}
filedata->file_offset = new_pos;
return filedata->file_offset;
}
/* MemFile reading. */
static ssize_t fd_read_from_memfile(FileData *filedata,
void *buffer,
size_t size,
bool *r_is_memchunck_identical)
{
static size_t seek = SIZE_MAX; /* the current position */
static size_t offset = 0; /* size of previous chunks */
static MemFileChunk *chunk = NULL;
size_t chunkoffset, readsize, totread;
if (r_is_memchunck_identical != NULL) {
*r_is_memchunck_identical = true;
}
if (size == 0) {
return 0;
}
if (seek != (size_t)filedata->file_offset) {
chunk = filedata->memfile->chunks.first;
seek = 0;
while (chunk) {
if (seek + chunk->size > (size_t)filedata->file_offset) {
break;
}
seek += chunk->size;
chunk = chunk->next;
}
offset = seek;
seek = (size_t)filedata->file_offset;
}
if (chunk) {
totread = 0;
do {
/* first check if it's on the end if current chunk */
if (seek - offset == chunk->size) {
offset += chunk->size;
chunk = chunk->next;
}
/* debug, should never happen */
if (chunk == NULL) {
CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk");
return 0;
}
chunkoffset = seek - offset;
readsize = size - totread;
/* data can be spread over multiple chunks, so clamp size
* to within this chunk, and then it will read further in
* the next chunk */
if (chunkoffset + readsize > chunk->size) {
readsize = chunk->size - chunkoffset;
}
memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
totread += readsize;
filedata->file_offset += readsize;
seek += readsize;
if (r_is_memchunck_identical != NULL) {
/* `is_identical` of current chunk represents whether it changed compared to previous undo
* step. this is fine in redo case, but not in undo case, where we need an extra flag
* defined when saving the next (future) step after the one we want to restore, as we are
* supposed to 'come from' that future undo step, and not the one before current one. */
*r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ?
chunk->is_identical :
chunk->is_identical_future;
}
} while (totread < size);
return (ssize_t)totread;
}
return 0;
}
static FileData *filedata_new(BlendFileReadReport *reports)
{
BLI_assert(reports != NULL);
FileData *fd = MEM_callocN(sizeof(FileData), "FileData");
fd->filedes = -1;
fd->gzfiledes = NULL;
fd->memsdna = DNA_sdna_current_get();
fd->datamap = oldnewmap_new();
@ -1387,78 +1191,60 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports)
static FileData *blo_filedata_from_file_descriptor(const char *filepath,
BlendFileReadReport *reports,
int file)
int filedes)
{
FileDataReadFn *read_fn = NULL;
FileDataSeekFn *seek_fn = NULL; /* Optional. */
size_t buffersize = 0;
BLI_mmap_file *mmap_file = NULL;
gzFile gzfile = (gzFile)Z_NULL;
char header[7];
FileReader *rawfile = BLI_filereader_new_file(filedes);
FileReader *file = NULL;
/* Regular file. */
errno = 0;
if (read(file, header, sizeof(header)) != sizeof(header)) {
/* If opening the file failed or we can't read the header, give up. */
if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
BKE_reportf(reports->reports,
RPT_WARNING,
"Unable to read '%s': %s",
filepath,
errno ? strerror(errno) : TIP_("insufficient content"));
if (rawfile) {
rawfile->close(rawfile);
}
else {
close(filedes);
}
return NULL;
}
/* Regular file. */
/* Rewind the file after reading the header. */
rawfile->seek(rawfile, 0, SEEK_SET);
/* Check if we have a regular file. */
if (memcmp(header, "BLENDER", sizeof(header)) == 0) {
read_fn = fd_read_data_from_file;
seek_fn = fd_seek_data_from_file;
mmap_file = BLI_mmap_open(file);
if (mmap_file != NULL) {
read_fn = fd_read_from_mmap;
seek_fn = fd_seek_from_mmap;
buffersize = BLI_lseek(file, 0, SEEK_END);
/* Try opening the file with memory-mapped IO. */
file = BLI_filereader_new_mmap(filedes);
if (file == NULL) {
/* mmap failed, so just keep using rawfile. */
file = rawfile;
rawfile = NULL;
}
}
else if (BLI_file_magic_is_gzip(header)) {
file = BLI_filereader_new_gzip(rawfile);
if (file != NULL) {
rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */
}
}
BLI_lseek(file, 0, SEEK_SET);
/* Gzip file. */
errno = 0;
if ((read_fn == NULL) &&
/* Check header magic. */
(header[0] == 0x1f && header[1] == 0x8b)) {
gzfile = BLI_gzopen(filepath, "rb");
if (gzfile == (gzFile)Z_NULL) {
BKE_reportf(reports->reports,
RPT_WARNING,
"Unable to open '%s': %s",
filepath,
errno ? strerror(errno) : TIP_("unknown error reading file"));
return NULL;
}
/* 'seek_fn' is too slow for gzip, don't set it. */
read_fn = fd_read_gzip_from_file;
/* Caller must close. */
file = -1;
/* Clean up `rawfile` if it wasn't taken over. */
if (rawfile != NULL) {
rawfile->close(rawfile);
}
if (read_fn == NULL) {
if (file == NULL) {
BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath);
return NULL;
}
FileData *fd = filedata_new(reports);
fd->filedes = file;
fd->gzfiledes = gzfile;
fd->read = read_fn;
fd->seek = seek_fn;
fd->mmap_file = mmap_file;
fd->buffersize = buffersize;
fd->file = file;
return fd;
}
@ -1475,11 +1261,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead
errno ? strerror(errno) : TIP_("unknown error reading file"));
return NULL;
}
FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file);
if ((fd == NULL) || (fd->filedes == -1)) {
close(file);
}
return fd;
return blo_filedata_from_file_descriptor(filepath, reports, file);
}
/* cannot be called with relative paths anymore! */
@ -1513,50 +1295,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath)
return NULL;
}
static ssize_t fd_read_gzip_from_memory(FileData *filedata,
void *buffer,
size_t size,
bool *UNUSED(r_is_memchunck_identical))
{
int err;
filedata->strm.next_out = (Bytef *)buffer;
filedata->strm.avail_out = (uint)size;
/* Inflate another chunk. */
err = inflate(&filedata->strm, Z_SYNC_FLUSH);
if (err == Z_STREAM_END) {
return 0;
}
if (err != Z_OK) {
CLOG_ERROR(&LOG, "ZLib error (code %d)", err);
return 0;
}
filedata->file_offset += size;
return (ssize_t)size;
}
static int fd_read_gzip_from_memory_init(FileData *fd)
{
fd->strm.next_in = (Bytef *)fd->buffer;
fd->strm.avail_in = fd->buffersize;
fd->strm.total_out = 0;
fd->strm.zalloc = Z_NULL;
fd->strm.zfree = Z_NULL;
if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) {
return 0;
}
fd->read = fd_read_gzip_from_memory;
return 1;
}
FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports)
{
if (!mem || memsize < SIZEOFBLENDERHEADER) {
@ -1565,24 +1303,21 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe
return NULL;
}
FileReader *mem_file = BLI_filereader_new_memory(mem, memsize);
FileReader *file = mem_file;
if (BLI_file_magic_is_gzip(mem)) {
file = BLI_filereader_new_gzip(mem_file);
}
if (file == NULL) {
/* Compression initialization failed. */
mem_file->close(mem_file);
return NULL;
}
FileData *fd = filedata_new(reports);
const char *cp = mem;
fd->buffer = mem;
fd->buffersize = memsize;
/* test if gzip */
if (cp[0] == 0x1f && cp[1] == 0x8b) {
if (0 == fd_read_gzip_from_memory_init(fd)) {
blo_filedata_free(fd);
return NULL;
}
}
else {
fd->read = fd_read_from_memory;
}
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
fd->file = file;
return blo_decode_and_check(fd, reports->reports);
}
@ -1597,11 +1332,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
}
FileData *fd = filedata_new(reports);
fd->memfile = memfile;
fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction);
fd->undo_direction = params->undo_direction;
fd->read = fd_read_from_memfile;
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
fd->flags |= FD_FLAGS_IS_MEMFILE;
return blo_decode_and_check(fd, reports->reports);
}
@ -1609,30 +1342,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
void blo_filedata_free(FileData *fd)
{
if (fd) {
if (fd->filedes != -1) {
close(fd->filedes);
}
if (fd->gzfiledes != NULL) {
gzclose(fd->gzfiledes);
}
if (fd->strm.next_in) {
int err = inflateEnd(&fd->strm);
if (err != Z_OK) {
CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err);
}
}
if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) {
MEM_freeN((void *)fd->buffer);
fd->buffer = NULL;
}
if (fd->mmap_file) {
BLI_mmap_free(fd->mmap_file);
fd->mmap_file = NULL;
}
fd->file->close(fd->file);
/* Free all BHeadN data blocks */
#ifndef NDEBUG
@ -1640,7 +1350,7 @@ void blo_filedata_free(FileData *fd)
#else
/* Sanity check we're not keeping memory we don't need. */
LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) {
if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
BLI_assert(new_bhead->has_data == 0);
}
MEM_freeN(new_bhead);
@ -2096,7 +1806,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id),
void blo_cache_storage_init(FileData *fd, Main *bmain)
{
if (fd->memfile != NULL) {
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
BLI_assert(fd->cache_storage == NULL);
fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__);
fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
@ -2261,7 +1971,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
* undo since DNA must match. */
static const void *peek_struct_undo(FileData *fd, BHead *bhead)
{
BLI_assert(fd->memfile != NULL);
BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE);
UNUSED_VARS_NDEBUG(fd);
return (bhead->len) ? (const void *)(bhead + 1) : NULL;
}
@ -3679,7 +3389,7 @@ static BHead *read_libblock(FileData *fd,
* When datablocks are changed but still exist, we restore them at the old
* address and inherit recalc flags for the dependency graph. */
ID *id_old = NULL;
if (fd->memfile != NULL) {
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) {
if (r_id) {
*r_id = id_old;
@ -3980,13 +3690,14 @@ static void lib_link_all(FileData *fd, Main *bmain)
continue;
}
if (fd->memfile != NULL && GS(id->name) == ID_WM) {
if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) {
/* No load UI for undo memfiles.
* Only WM currently, SCR needs it still (see below), and so does WS? */
continue;
}
if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo &&
(id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
/* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across
* current undo step, and old IDs re-use their old memory address, we do not need to liblink
* it at all. */
@ -4165,7 +3876,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
BlendFileData *bfd;
ListBase mainlist = {NULL, NULL};
if (fd->memfile != NULL) {
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step");
}
@ -4256,7 +3967,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
}
/* do before read_libraries, but skip undo case */
if (fd->memfile == NULL) {
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) {
do_versions(fd, NULL, bfd->main);
}
@ -4278,7 +3989,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries;
/* Skip in undo case. */
if (fd->memfile == NULL) {
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
/* Note that we can't recompute user-counts at this point in undo case, we play too much with
* IDs from different memory realms, and Main database is not in a fully valid state yet.
*/
@ -4311,7 +4022,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
/* Now that all our data-blocks are loaded,
* we can re-generate overrides from their references. */
if (fd->memfile == NULL) {
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
/* Do not apply in undo case! */
fd->reports->duration.lib_overrides = PIL_check_seconds_timer();
@ -4391,7 +4102,7 @@ static void sort_bhead_old_map(FileData *fd)
static BHead *find_previous_lib(FileData *fd, BHead *bhead)
{
/* Skip library data-blocks in undo, see comment in read_libblock. */
if (fd->memfile) {
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
return NULL;
}
@ -5850,7 +5561,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p)
bool BLO_read_data_is_undo(BlendDataReader *reader)
{
return reader->fd->memfile != NULL;
return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
}
void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr)
@ -5870,7 +5581,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader)
bool BLO_read_lib_is_undo(BlendLibReader *reader)
{
return reader->fd->memfile != NULL;
return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
}
Main *BLO_read_lib_get_main(BlendLibReader *reader)

View File

@ -28,10 +28,10 @@
# include "BLI_winstuff.h"
#endif
#include "BLI_filereader.h"
#include "DNA_sdna_types.h"
#include "DNA_space_types.h"
#include "DNA_windowmanager_types.h" /* for ReportType */
#include "zlib.h"
struct BLI_mmap_file;
struct BLOCacheStorage;
@ -50,7 +50,7 @@ enum eFileDataFlag {
FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1,
FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2,
FD_FLAGS_FILE_OK = 1 << 3,
FD_FLAGS_NOT_MY_BUFFER = 1 << 4,
FD_FLAGS_IS_MEMFILE = 1 << 4,
/* XXX Unused in practice (checked once but never set). */
FD_FLAGS_NOT_MY_LIBMAP = 1 << 5,
};
@ -60,44 +60,18 @@ enum eFileDataFlag {
# pragma GCC poison off_t
#endif
#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
typedef int64_t off64_t;
#endif
typedef ssize_t(FileDataReadFn)(struct FileData *filedata,
void *buffer,
size_t size,
bool *r_is_memchunk_identical);
typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence);
typedef struct FileData {
/** Linked list of BHeadN's. */
ListBase bhead_list;
enum eFileDataFlag flags;
bool is_eof;
size_t buffersize;
off64_t file_offset;
FileDataReadFn *read;
FileDataSeekFn *seek;
FileReader *file;
/** Regular file reading. */
int filedes;
/** Variables needed for reading from memory / stream / memory-mapped files. */
const char *buffer;
struct BLI_mmap_file *mmap_file;
/** Variables needed for reading from memfile (undo). */
struct MemFile *memfile;
/** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use
* to detect unchanged data from memfile. */
int undo_direction; /* eUndoStepDir */
/** Variables needed for reading from file. */
gzFile gzfiledes;
/** Gzip stream for memory decompression. */
z_stream strm;
/** Now only in use for library appending. */
char relabase[FILE_MAX];

View File

@ -48,6 +48,7 @@
#include "BKE_lib_id.h"
#include "BKE_main.h"
#include "BKE_undo_system.h"
/* keep last */
#include "BLI_strict_flags.h"
@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename)
}
return true;
}
static ssize_t undo_read(FileReader *reader, void *buffer, size_t size)
{
UndoReader *undo = (UndoReader *)reader;
static size_t seek = SIZE_MAX; /* The current position. */
static size_t offset = 0; /* Size of previous chunks. */
static MemFileChunk *chunk = NULL;
size_t chunkoffset, readsize, totread;
undo->memchunk_identical = true;
if (size == 0) {
return 0;
}
if (seek != (size_t)undo->reader.offset) {
chunk = undo->memfile->chunks.first;
seek = 0;
while (chunk) {
if (seek + chunk->size > (size_t)undo->reader.offset) {
break;
}
seek += chunk->size;
chunk = chunk->next;
}
offset = seek;
seek = (size_t)undo->reader.offset;
}
if (chunk) {
totread = 0;
do {
/* First check if it's on the end if current chunk. */
if (seek - offset == chunk->size) {
offset += chunk->size;
chunk = chunk->next;
}
/* Debug, should never happen. */
if (chunk == NULL) {
printf("illegal read, chunk zero\n");
return 0;
}
chunkoffset = seek - offset;
readsize = size - totread;
/* Data can be spread over multiple chunks, so clamp size
* to within this chunk, and then it will read further in
* the next chunk. */
if (chunkoffset + readsize > chunk->size) {
readsize = chunk->size - chunkoffset;
}
memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
totread += readsize;
undo->reader.offset += (off64_t)readsize;
seek += readsize;
/* `is_identical` of current chunk represents whether it changed compared to previous undo
* step. this is fine in redo case, but not in undo case, where we need an extra flag
* defined when saving the next (future) step after the one we want to restore, as we are
* supposed to 'come from' that future undo step, and not the one before current one. */
undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical :
chunk->is_identical_future;
} while (totread < size);
return (ssize_t)totread;
}
return 0;
}
static void undo_close(FileReader *reader)
{
MEM_freeN(reader);
}
FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction)
{
UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__);
undo->memfile = memfile;
undo->undo_direction = undo_direction;
undo->reader.read = undo_read;
undo->reader.seek = NULL;
undo->reader.close = undo_close;
return (FileReader *)undo;
}

View File

@ -23,8 +23,7 @@
#else
# include "BLI_winstuff.h"
# include "winsock2.h"
# include <io.h> /* for open close read */
# include <zlib.h> /* odd include order-issue */
# include <io.h> /* for open close read */
#endif
/* allow readfile to use deprecated functionality */

View File

@ -28,8 +28,7 @@
#else
# include "BLI_winstuff.h"
# include "winsock2.h"
# include <io.h> /* for open close read */
# include <zlib.h> /* odd include order-issue */
# include <io.h> /* for open close read */
#endif
/* allow readfile to use deprecated functionality */

View File

@ -78,12 +78,12 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <zlib.h>
#ifdef WIN32
# include "BLI_winstuff.h"
# include "winsock2.h"
# include <io.h>
# include <zlib.h> /* odd include order-issue */
#else
# include <unistd.h> /* FreeBSD, for write() and close(). */
#endif

View File

@ -28,11 +28,10 @@
* winsock stuff.
*/
#include <errno.h>
#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
#include <stddef.h>
#include <string.h>
#include "zlib.h" /* wm_read_exotic() */
#ifdef WIN32
/* Need to include windows.h so _WIN32_IE is defined. */
# include <windows.h>
@ -51,6 +50,7 @@
#include "BLI_blenlib.h"
#include "BLI_fileops_types.h"
#include "BLI_filereader.h"
#include "BLI_linklist.h"
#include "BLI_math.h"
#include "BLI_system.h"
@ -481,53 +481,59 @@ static void wm_init_userdef(Main *bmain)
/* intended to check for non-blender formats but for now it only reads blends */
static int wm_read_exotic(const char *name)
{
int len;
gzFile gzfile;
char header[7];
int retval;
/* make sure we're not trying to read a directory.... */
len = strlen(name);
if (len > 0 && ELEM(name[len - 1], '/', '\\')) {
retval = BKE_READ_EXOTIC_FAIL_PATH;
int namelen = strlen(name);
if (namelen > 0 && ELEM(name[namelen - 1], '/', '\\')) {
return BKE_READ_EXOTIC_FAIL_PATH;
}
/* open the file. */
const int filedes = BLI_open(name, O_BINARY | O_RDONLY, 0);
if (filedes == -1) {
return BKE_READ_EXOTIC_FAIL_OPEN;
}
FileReader *rawfile = BLI_filereader_new_file(filedes);
if (rawfile == NULL) {
return BKE_READ_EXOTIC_FAIL_OPEN;
}
/* read the header (7 bytes are enough to identify all known types). */
char header[7];
if (rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
rawfile->close(rawfile);
return BKE_READ_EXOTIC_FAIL_FORMAT;
}
rawfile->seek(rawfile, 0, SEEK_SET);
/* check for uncompressed .blend */
if (STREQLEN(header, "BLENDER", 7)) {
rawfile->close(rawfile);
return BKE_READ_EXOTIC_OK_BLEND;
}
/* check for compressed .blend */
FileReader *compressed_file = NULL;
if (BLI_file_magic_is_gzip(header)) {
compressed_file = BLI_filereader_new_gzip(rawfile);
}
/* If a compression signature matches, try decompressing the start and check if it's a .blend */
if (compressed_file != NULL) {
size_t len = compressed_file->read(compressed_file, header, sizeof(header));
compressed_file->close(compressed_file);
if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
return BKE_READ_EXOTIC_OK_BLEND;
}
}
else {
gzfile = BLI_gzopen(name, "rb");
if (gzfile == NULL) {
retval = BKE_READ_EXOTIC_FAIL_OPEN;
}
else {
len = gzread(gzfile, header, sizeof(header));
gzclose(gzfile);
if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
retval = BKE_READ_EXOTIC_OK_BLEND;
}
else {
/* We may want to support loading other file formats
* from their header bytes or file extension.
* This used to be supported in the code below and may be added
* back at some point. */
#if 0
WM_cursor_wait(true);
if (is_foo_format(name)) {
read_foo(name);
retval = BKE_READ_EXOTIC_OK_OTHER;
}
else
#endif
{
retval = BKE_READ_EXOTIC_FAIL_FORMAT;
}
#if 0
WM_cursor_wait(false);
#endif
}
}
rawfile->close(rawfile);
}
return retval;
/* Add check for future file formats here. */
return BKE_READ_EXOTIC_FAIL_FORMAT;
}
/** \} */