Refactor low-level blendfile reading into separate files
Instead of handling mmap, compression etc. all directly in readfile.c, refactor the code to use a generic FileReader. This makes it easier to add new compression methods or similar, and allows to reuse the logic in other places (e.g. thumbnail reading). Reviewed By: campbellbarton, brecht, mont29 Differential Revision: https://developer.blender.org/D5799
This commit is contained in:
parent
34a05f39be
commit
2b170f16d6
|
@ -166,6 +166,8 @@ size_t BLI_gzip_mem_to_file_at_pos(void *buf,
|
|||
int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
|
||||
ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
bool BLI_file_magic_is_gzip(const char header[4]);
|
||||
|
||||
size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
|
||||
size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
|
||||
|
|
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
* \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...).
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef WIN32
|
||||
# include "BLI_winstuff.h"
|
||||
#else
|
||||
# include <sys/types.h>
|
||||
#endif
|
||||
|
||||
#include "BLI_compiler_attrs.h"
|
||||
#include "BLI_utildefines.h"
|
||||
|
||||
#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
|
||||
typedef int64_t off64_t;
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct FileReader;
|
||||
|
||||
typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size);
|
||||
typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence);
|
||||
typedef void (*FileReaderCloseFn)(struct FileReader *reader);
|
||||
|
||||
/* General structure for all FileReaders, implementations add custom fields at the end. */
|
||||
typedef struct FileReader {
|
||||
FileReaderReadFn read;
|
||||
FileReaderSeekFn seek;
|
||||
FileReaderCloseFn close;
|
||||
|
||||
off64_t offset;
|
||||
} FileReader;
|
||||
|
||||
/* Functions for opening the various types of FileReader.
|
||||
* They either succeed and return a valid FileReader, or fail and return NULL.
|
||||
*
|
||||
* If a FileReader is created, it has to be cleaned up and freed by calling
|
||||
* its close() function unless another FileReader has taken ownership - for example,
|
||||
* Gzip takes over the base FileReader and will clean it up when their clean() is called.
|
||||
*/
|
||||
|
||||
/* Create FileReader from raw file descriptor. */
|
||||
FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT;
|
||||
/* Create FileReader from raw file descriptor using memory-mapped IO. */
|
||||
FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
|
||||
/* Create FileReader from a region of memory. */
|
||||
FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
|
||||
ATTR_NONNULL();
|
||||
/* Create FileReader from applying Gzip decompression on an underlying file. */
|
||||
FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
|
@ -75,6 +75,9 @@ set(SRC
|
|||
intern/endian_switch.c
|
||||
intern/expr_pylike_eval.c
|
||||
intern/fileops.c
|
||||
intern/filereader_file.c
|
||||
intern/filereader_gzip.c
|
||||
intern/filereader_memory.c
|
||||
intern/fnmatch.c
|
||||
intern/freetypefont.c
|
||||
intern/gsqueue.c
|
||||
|
@ -194,6 +197,7 @@ set(SRC
|
|||
BLI_enumerable_thread_specific.hh
|
||||
BLI_expr_pylike_eval.h
|
||||
BLI_fileops.h
|
||||
BLI_filereader.h
|
||||
BLI_fileops_types.h
|
||||
BLI_float2.hh
|
||||
BLI_float3.hh
|
||||
|
|
|
@ -255,6 +255,13 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
|
|||
|
||||
#undef CHUNK
|
||||
|
||||
bool BLI_file_magic_is_gzip(const char header[4])
|
||||
{
|
||||
/* GZIP itself starts with the magic bytes 0x1f 0x8b.
|
||||
* The third byte indicates the compression method, which is 0x08 for DEFLATE. */
|
||||
return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the file with the specified name can be written.
|
||||
* This implementation uses access(2), which makes the check according
|
||||
|
|
|
@ -0,0 +1,80 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
*/
|
||||
|
||||
#ifndef WIN32
|
||||
# include <unistd.h> /* for read close */
|
||||
#else
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h> /* for open close read */
|
||||
#endif
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_filereader.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
int filedes;
|
||||
} RawFileReader;
|
||||
|
||||
static ssize_t file_read(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
RawFileReader *rawfile = (RawFileReader *)reader;
|
||||
ssize_t readsize = read(rawfile->filedes, buffer, size);
|
||||
|
||||
if (readsize >= 0) {
|
||||
rawfile->reader.offset += readsize;
|
||||
}
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t file_seek(FileReader *reader, off64_t offset, int whence)
|
||||
{
|
||||
RawFileReader *rawfile = (RawFileReader *)reader;
|
||||
rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence);
|
||||
return rawfile->reader.offset;
|
||||
}
|
||||
|
||||
static void file_close(FileReader *reader)
|
||||
{
|
||||
RawFileReader *rawfile = (RawFileReader *)reader;
|
||||
close(rawfile->filedes);
|
||||
MEM_freeN(rawfile);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_file(int filedes)
|
||||
{
|
||||
RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__);
|
||||
|
||||
rawfile->filedes = filedes;
|
||||
|
||||
rawfile->reader.read = file_read;
|
||||
rawfile->reader.seek = file_seek;
|
||||
rawfile->reader.close = file_close;
|
||||
|
||||
return (FileReader *)rawfile;
|
||||
}
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
*/
|
||||
|
||||
#include <zlib.h>
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_filereader.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
FileReader *base;
|
||||
|
||||
z_stream strm;
|
||||
|
||||
void *in_buf;
|
||||
size_t in_size;
|
||||
} GzipReader;
|
||||
|
||||
static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
GzipReader *gzip = (GzipReader *)reader;
|
||||
|
||||
gzip->strm.avail_out = size;
|
||||
gzip->strm.next_out = buffer;
|
||||
|
||||
while (gzip->strm.avail_out > 0) {
|
||||
if (gzip->strm.avail_in == 0) {
|
||||
/* Ran out of buffered input data, read some more. */
|
||||
size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size);
|
||||
|
||||
if (readsize > 0) {
|
||||
/* We got some data, so mark the buffer as refilled. */
|
||||
gzip->strm.avail_in = readsize;
|
||||
gzip->strm.next_in = gzip->in_buf;
|
||||
}
|
||||
else {
|
||||
/* The underlying file is EOF, so return as much as we can. */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int ret = inflate(&gzip->strm, Z_NO_FLUSH);
|
||||
|
||||
if (ret != Z_OK && ret != Z_BUF_ERROR) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ssize_t read_len = size - gzip->strm.avail_out;
|
||||
gzip->reader.offset += read_len;
|
||||
return read_len;
|
||||
}
|
||||
|
||||
static void gzip_close(FileReader *reader)
|
||||
{
|
||||
GzipReader *gzip = (GzipReader *)reader;
|
||||
|
||||
if (inflateEnd(&gzip->strm) != Z_OK) {
|
||||
printf("close gzip stream error\n");
|
||||
}
|
||||
MEM_freeN((void *)gzip->in_buf);
|
||||
|
||||
gzip->base->close(gzip->base);
|
||||
MEM_freeN(gzip);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_gzip(FileReader *base)
|
||||
{
|
||||
GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__);
|
||||
gzip->base = base;
|
||||
|
||||
if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) {
|
||||
MEM_freeN(gzip);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
gzip->in_size = 256 * 2014;
|
||||
gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf");
|
||||
|
||||
gzip->reader.read = gzip_read;
|
||||
gzip->reader.seek = NULL;
|
||||
gzip->reader.close = gzip_close;
|
||||
|
||||
return (FileReader *)gzip;
|
||||
}
|
|
@ -0,0 +1,145 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_filereader.h"
|
||||
#include "BLI_mmap.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
/* This file implements both memory-backed and memory-mapped-file-backed reading. */
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
const char *data;
|
||||
BLI_mmap_file *mmap;
|
||||
size_t length;
|
||||
} MemoryReader;
|
||||
|
||||
static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
|
||||
/* Don't read more bytes than there are available in the buffer. */
|
||||
size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
|
||||
|
||||
memcpy(buffer, mem->data + mem->reader.offset, readsize);
|
||||
mem->reader.offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t memory_seek(FileReader *reader, off64_t offset, int whence)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
|
||||
off64_t new_pos;
|
||||
if (whence == SEEK_CUR) {
|
||||
new_pos = mem->reader.offset + offset;
|
||||
}
|
||||
else if (whence == SEEK_SET) {
|
||||
new_pos = offset;
|
||||
}
|
||||
else if (whence == SEEK_END) {
|
||||
new_pos = mem->length + offset;
|
||||
}
|
||||
else {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (new_pos < 0 || new_pos > mem->length) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
mem->reader.offset = new_pos;
|
||||
return mem->reader.offset;
|
||||
}
|
||||
|
||||
static void memory_close_raw(FileReader *reader)
|
||||
{
|
||||
MEM_freeN(reader);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_memory(const void *data, size_t len)
|
||||
{
|
||||
MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
|
||||
|
||||
mem->data = (const char *)data;
|
||||
mem->length = len;
|
||||
|
||||
mem->reader.read = memory_read_raw;
|
||||
mem->reader.seek = memory_seek;
|
||||
mem->reader.close = memory_close_raw;
|
||||
|
||||
return (FileReader *)mem;
|
||||
}
|
||||
|
||||
/* Memory-mapped file reading.
|
||||
* By using `mmap()`, we can map a file so that it can be treated like normal memory,
|
||||
* meaning that we can just read from it with `memcpy()` etc.
|
||||
* This avoids system call overhead and can significantly speed up file loading.
|
||||
*/
|
||||
|
||||
static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
|
||||
/* Don't read more bytes than there are available in the buffer. */
|
||||
size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
|
||||
|
||||
if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
mem->reader.offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static void memory_close_mmap(FileReader *reader)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
BLI_mmap_free(mem->mmap);
|
||||
MEM_freeN(mem);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_mmap(int filedes)
|
||||
{
|
||||
BLI_mmap_file *mmap = BLI_mmap_open(filedes);
|
||||
if (mmap == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
|
||||
|
||||
mem->mmap = mmap;
|
||||
mem->length = BLI_lseek(filedes, 0, SEEK_END);
|
||||
|
||||
mem->reader.read = memory_read_mmap;
|
||||
mem->reader.seek = memory_seek;
|
||||
mem->reader.close = memory_close_mmap;
|
||||
|
||||
return (FileReader *)mem;
|
||||
}
|
|
@ -24,6 +24,8 @@
|
|||
* \ingroup blenloader
|
||||
*/
|
||||
|
||||
#include "BLI_filereader.h"
|
||||
|
||||
struct GHash;
|
||||
struct Scene;
|
||||
|
||||
|
@ -65,6 +67,16 @@ typedef struct MemFileUndoData {
|
|||
size_t undo_size;
|
||||
} MemFileUndoData;
|
||||
|
||||
/* FileReader-compatible wrapper for reading MemFiles */
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
MemFile *memfile;
|
||||
int undo_direction;
|
||||
|
||||
bool memchunk_identical;
|
||||
} UndoReader;
|
||||
|
||||
/* actually only used writefile.c */
|
||||
|
||||
void BLO_memfile_write_init(MemFileWriteData *mem_data,
|
||||
|
@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile,
|
|||
struct Main *bmain,
|
||||
struct Scene **r_scene);
|
||||
extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename);
|
||||
|
||||
FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction);
|
||||
|
|
|
@ -21,8 +21,6 @@
|
|||
* \ingroup blenloader
|
||||
*/
|
||||
|
||||
#include "zlib.h"
|
||||
|
||||
#include <ctype.h> /* for isdigit. */
|
||||
#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
|
||||
#include <limits.h>
|
||||
|
@ -71,7 +69,6 @@
|
|||
#include "BLI_math.h"
|
||||
#include "BLI_memarena.h"
|
||||
#include "BLI_mempool.h"
|
||||
#include "BLI_mmap.h"
|
||||
#include "BLI_threads.h"
|
||||
|
||||
#include "PIL_time.h"
|
||||
|
@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd)
|
|||
*/
|
||||
if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) {
|
||||
bhead4.code = DATA;
|
||||
readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL);
|
||||
readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4));
|
||||
|
||||
if (readsize == sizeof(bhead4) || bhead4.code == ENDB) {
|
||||
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
|
||||
|
@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd)
|
|||
}
|
||||
else {
|
||||
bhead8.code = DATA;
|
||||
readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL);
|
||||
readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8));
|
||||
|
||||
if (readsize == sizeof(bhead8) || bhead8.code == ENDB) {
|
||||
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
|
||||
|
@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd)
|
|||
/* pass */
|
||||
}
|
||||
#ifdef USE_BHEAD_READ_ON_DEMAND
|
||||
else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
|
||||
else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
|
||||
/* Delay reading bhead content. */
|
||||
new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead");
|
||||
if (new_bhead) {
|
||||
new_bhead->next = new_bhead->prev = NULL;
|
||||
new_bhead->file_offset = fd->file_offset;
|
||||
new_bhead->file_offset = fd->file->offset;
|
||||
new_bhead->has_data = false;
|
||||
new_bhead->is_memchunk_identical = false;
|
||||
new_bhead->bhead = bhead;
|
||||
off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR);
|
||||
off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR);
|
||||
if (seek_new == -1) {
|
||||
fd->is_eof = true;
|
||||
MEM_freeN(new_bhead);
|
||||
new_bhead = NULL;
|
||||
}
|
||||
BLI_assert(fd->file_offset == seek_new);
|
||||
BLI_assert(fd->file->offset == seek_new);
|
||||
}
|
||||
else {
|
||||
fd->is_eof = true;
|
||||
|
@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd)
|
|||
new_bhead->is_memchunk_identical = false;
|
||||
new_bhead->bhead = bhead;
|
||||
|
||||
readsize = fd->read(
|
||||
fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical);
|
||||
readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len);
|
||||
|
||||
if (readsize != (ssize_t)bhead.len) {
|
||||
if (readsize != bhead.len) {
|
||||
fd->is_eof = true;
|
||||
MEM_freeN(new_bhead);
|
||||
new_bhead = NULL;
|
||||
}
|
||||
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
|
||||
}
|
||||
}
|
||||
else {
|
||||
fd->is_eof = true;
|
||||
|
@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
|
|||
bool success = true;
|
||||
BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock);
|
||||
BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0);
|
||||
off64_t offset_backup = fd->file_offset;
|
||||
if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) {
|
||||
off64_t offset_backup = fd->file->offset;
|
||||
if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) {
|
||||
success = false;
|
||||
}
|
||||
else {
|
||||
if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
|
||||
(ssize_t)new_bhead->bhead.len) {
|
||||
if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) {
|
||||
success = false;
|
||||
}
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
|
||||
}
|
||||
}
|
||||
if (fd->seek(fd, offset_backup, SEEK_SET) == -1) {
|
||||
if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) {
|
||||
success = false;
|
||||
}
|
||||
return success;
|
||||
|
@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd)
|
|||
ssize_t readsize;
|
||||
|
||||
/* read in the header data */
|
||||
readsize = fd->read(fd, header, sizeof(header), NULL);
|
||||
readsize = fd->file->read(fd->file, header, sizeof(header));
|
||||
|
||||
if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') &&
|
||||
ELEM(header[8], 'v', 'V') &&
|
||||
|
@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd)
|
|||
|
||||
/** \} */
|
||||
|
||||
/* -------------------------------------------------------------------- */
|
||||
/** \name File Data API
|
||||
* \{ */
|
||||
|
||||
/* Regular file reading. */
|
||||
|
||||
static ssize_t fd_read_data_from_file(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
ssize_t readsize = read(filedata->filedes, buffer, size);
|
||||
|
||||
if (readsize < 0) {
|
||||
readsize = EOF;
|
||||
}
|
||||
else {
|
||||
filedata->file_offset += readsize;
|
||||
}
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence)
|
||||
{
|
||||
filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence);
|
||||
return filedata->file_offset;
|
||||
}
|
||||
|
||||
/* GZip file reading. */
|
||||
|
||||
static ssize_t fd_read_gzip_from_file(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
BLI_assert(size <= INT_MAX);
|
||||
|
||||
ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size);
|
||||
|
||||
if (readsize < 0) {
|
||||
readsize = EOF;
|
||||
}
|
||||
else {
|
||||
filedata->file_offset += readsize;
|
||||
}
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
/* Memory reading. */
|
||||
|
||||
static ssize_t fd_read_from_memory(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
/* don't read more bytes than there are available in the buffer */
|
||||
ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset);
|
||||
|
||||
memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize);
|
||||
filedata->file_offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
/* Memory-mapped file reading.
|
||||
* By using mmap(), we can map a file so that it can be treated like normal memory,
|
||||
* meaning that we can just read from it with memcpy() etc.
|
||||
* This avoids system call overhead and can significantly speed up file loading.
|
||||
*/
|
||||
|
||||
static ssize_t fd_read_from_mmap(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
/* don't read more bytes than there are available in the buffer */
|
||||
size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset));
|
||||
|
||||
if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
filedata->file_offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence)
|
||||
{
|
||||
off64_t new_pos;
|
||||
if (whence == SEEK_CUR) {
|
||||
new_pos = filedata->file_offset + offset;
|
||||
}
|
||||
else if (whence == SEEK_SET) {
|
||||
new_pos = offset;
|
||||
}
|
||||
else if (whence == SEEK_END) {
|
||||
new_pos = filedata->buffersize + offset;
|
||||
}
|
||||
else {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (new_pos < 0 || new_pos > filedata->buffersize) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
filedata->file_offset = new_pos;
|
||||
return filedata->file_offset;
|
||||
}
|
||||
|
||||
/* MemFile reading. */
|
||||
|
||||
static ssize_t fd_read_from_memfile(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *r_is_memchunck_identical)
|
||||
{
|
||||
static size_t seek = SIZE_MAX; /* the current position */
|
||||
static size_t offset = 0; /* size of previous chunks */
|
||||
static MemFileChunk *chunk = NULL;
|
||||
size_t chunkoffset, readsize, totread;
|
||||
|
||||
if (r_is_memchunck_identical != NULL) {
|
||||
*r_is_memchunck_identical = true;
|
||||
}
|
||||
|
||||
if (size == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (seek != (size_t)filedata->file_offset) {
|
||||
chunk = filedata->memfile->chunks.first;
|
||||
seek = 0;
|
||||
|
||||
while (chunk) {
|
||||
if (seek + chunk->size > (size_t)filedata->file_offset) {
|
||||
break;
|
||||
}
|
||||
seek += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
offset = seek;
|
||||
seek = (size_t)filedata->file_offset;
|
||||
}
|
||||
|
||||
if (chunk) {
|
||||
totread = 0;
|
||||
|
||||
do {
|
||||
/* first check if it's on the end if current chunk */
|
||||
if (seek - offset == chunk->size) {
|
||||
offset += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
|
||||
/* debug, should never happen */
|
||||
if (chunk == NULL) {
|
||||
CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk");
|
||||
return 0;
|
||||
}
|
||||
|
||||
chunkoffset = seek - offset;
|
||||
readsize = size - totread;
|
||||
|
||||
/* data can be spread over multiple chunks, so clamp size
|
||||
* to within this chunk, and then it will read further in
|
||||
* the next chunk */
|
||||
if (chunkoffset + readsize > chunk->size) {
|
||||
readsize = chunk->size - chunkoffset;
|
||||
}
|
||||
|
||||
memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
|
||||
totread += readsize;
|
||||
filedata->file_offset += readsize;
|
||||
seek += readsize;
|
||||
if (r_is_memchunck_identical != NULL) {
|
||||
/* `is_identical` of current chunk represents whether it changed compared to previous undo
|
||||
* step. this is fine in redo case, but not in undo case, where we need an extra flag
|
||||
* defined when saving the next (future) step after the one we want to restore, as we are
|
||||
* supposed to 'come from' that future undo step, and not the one before current one. */
|
||||
*r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ?
|
||||
chunk->is_identical :
|
||||
chunk->is_identical_future;
|
||||
}
|
||||
} while (totread < size);
|
||||
|
||||
return (ssize_t)totread;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FileData *filedata_new(BlendFileReadReport *reports)
|
||||
{
|
||||
BLI_assert(reports != NULL);
|
||||
|
||||
FileData *fd = MEM_callocN(sizeof(FileData), "FileData");
|
||||
|
||||
fd->filedes = -1;
|
||||
fd->gzfiledes = NULL;
|
||||
|
||||
fd->memsdna = DNA_sdna_current_get();
|
||||
|
||||
fd->datamap = oldnewmap_new();
|
||||
|
@ -1387,78 +1191,60 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports)
|
|||
|
||||
static FileData *blo_filedata_from_file_descriptor(const char *filepath,
|
||||
BlendFileReadReport *reports,
|
||||
int file)
|
||||
int filedes)
|
||||
{
|
||||
FileDataReadFn *read_fn = NULL;
|
||||
FileDataSeekFn *seek_fn = NULL; /* Optional. */
|
||||
size_t buffersize = 0;
|
||||
BLI_mmap_file *mmap_file = NULL;
|
||||
|
||||
gzFile gzfile = (gzFile)Z_NULL;
|
||||
|
||||
char header[7];
|
||||
FileReader *rawfile = BLI_filereader_new_file(filedes);
|
||||
FileReader *file = NULL;
|
||||
|
||||
/* Regular file. */
|
||||
errno = 0;
|
||||
if (read(file, header, sizeof(header)) != sizeof(header)) {
|
||||
/* If opening the file failed or we can't read the header, give up. */
|
||||
if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
|
||||
BKE_reportf(reports->reports,
|
||||
RPT_WARNING,
|
||||
"Unable to read '%s': %s",
|
||||
filepath,
|
||||
errno ? strerror(errno) : TIP_("insufficient content"));
|
||||
if (rawfile) {
|
||||
rawfile->close(rawfile);
|
||||
}
|
||||
else {
|
||||
close(filedes);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Regular file. */
|
||||
/* Rewind the file after reading the header. */
|
||||
rawfile->seek(rawfile, 0, SEEK_SET);
|
||||
|
||||
/* Check if we have a regular file. */
|
||||
if (memcmp(header, "BLENDER", sizeof(header)) == 0) {
|
||||
read_fn = fd_read_data_from_file;
|
||||
seek_fn = fd_seek_data_from_file;
|
||||
|
||||
mmap_file = BLI_mmap_open(file);
|
||||
if (mmap_file != NULL) {
|
||||
read_fn = fd_read_from_mmap;
|
||||
seek_fn = fd_seek_from_mmap;
|
||||
buffersize = BLI_lseek(file, 0, SEEK_END);
|
||||
/* Try opening the file with memory-mapped IO. */
|
||||
file = BLI_filereader_new_mmap(filedes);
|
||||
if (file == NULL) {
|
||||
/* mmap failed, so just keep using rawfile. */
|
||||
file = rawfile;
|
||||
rawfile = NULL;
|
||||
}
|
||||
}
|
||||
else if (BLI_file_magic_is_gzip(header)) {
|
||||
file = BLI_filereader_new_gzip(rawfile);
|
||||
if (file != NULL) {
|
||||
rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */
|
||||
}
|
||||
}
|
||||
|
||||
BLI_lseek(file, 0, SEEK_SET);
|
||||
|
||||
/* Gzip file. */
|
||||
errno = 0;
|
||||
if ((read_fn == NULL) &&
|
||||
/* Check header magic. */
|
||||
(header[0] == 0x1f && header[1] == 0x8b)) {
|
||||
gzfile = BLI_gzopen(filepath, "rb");
|
||||
if (gzfile == (gzFile)Z_NULL) {
|
||||
BKE_reportf(reports->reports,
|
||||
RPT_WARNING,
|
||||
"Unable to open '%s': %s",
|
||||
filepath,
|
||||
errno ? strerror(errno) : TIP_("unknown error reading file"));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* 'seek_fn' is too slow for gzip, don't set it. */
|
||||
read_fn = fd_read_gzip_from_file;
|
||||
/* Caller must close. */
|
||||
file = -1;
|
||||
/* Clean up `rawfile` if it wasn't taken over. */
|
||||
if (rawfile != NULL) {
|
||||
rawfile->close(rawfile);
|
||||
}
|
||||
|
||||
if (read_fn == NULL) {
|
||||
if (file == NULL) {
|
||||
BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
FileData *fd = filedata_new(reports);
|
||||
|
||||
fd->filedes = file;
|
||||
fd->gzfiledes = gzfile;
|
||||
|
||||
fd->read = read_fn;
|
||||
fd->seek = seek_fn;
|
||||
fd->mmap_file = mmap_file;
|
||||
fd->buffersize = buffersize;
|
||||
fd->file = file;
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
@ -1475,11 +1261,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead
|
|||
errno ? strerror(errno) : TIP_("unknown error reading file"));
|
||||
return NULL;
|
||||
}
|
||||
FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file);
|
||||
if ((fd == NULL) || (fd->filedes == -1)) {
|
||||
close(file);
|
||||
}
|
||||
return fd;
|
||||
return blo_filedata_from_file_descriptor(filepath, reports, file);
|
||||
}
|
||||
|
||||
/* cannot be called with relative paths anymore! */
|
||||
|
@ -1513,50 +1295,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static ssize_t fd_read_gzip_from_memory(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
int err;
|
||||
|
||||
filedata->strm.next_out = (Bytef *)buffer;
|
||||
filedata->strm.avail_out = (uint)size;
|
||||
|
||||
/* Inflate another chunk. */
|
||||
err = inflate(&filedata->strm, Z_SYNC_FLUSH);
|
||||
|
||||
if (err == Z_STREAM_END) {
|
||||
return 0;
|
||||
}
|
||||
if (err != Z_OK) {
|
||||
CLOG_ERROR(&LOG, "ZLib error (code %d)", err);
|
||||
return 0;
|
||||
}
|
||||
|
||||
filedata->file_offset += size;
|
||||
|
||||
return (ssize_t)size;
|
||||
}
|
||||
|
||||
static int fd_read_gzip_from_memory_init(FileData *fd)
|
||||
{
|
||||
|
||||
fd->strm.next_in = (Bytef *)fd->buffer;
|
||||
fd->strm.avail_in = fd->buffersize;
|
||||
fd->strm.total_out = 0;
|
||||
fd->strm.zalloc = Z_NULL;
|
||||
fd->strm.zfree = Z_NULL;
|
||||
|
||||
if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
fd->read = fd_read_gzip_from_memory;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports)
|
||||
{
|
||||
if (!mem || memsize < SIZEOFBLENDERHEADER) {
|
||||
|
@ -1565,24 +1303,21 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe
|
|||
return NULL;
|
||||
}
|
||||
|
||||
FileReader *mem_file = BLI_filereader_new_memory(mem, memsize);
|
||||
FileReader *file = mem_file;
|
||||
|
||||
if (BLI_file_magic_is_gzip(mem)) {
|
||||
file = BLI_filereader_new_gzip(mem_file);
|
||||
}
|
||||
|
||||
if (file == NULL) {
|
||||
/* Compression initialization failed. */
|
||||
mem_file->close(mem_file);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
FileData *fd = filedata_new(reports);
|
||||
const char *cp = mem;
|
||||
|
||||
fd->buffer = mem;
|
||||
fd->buffersize = memsize;
|
||||
|
||||
/* test if gzip */
|
||||
if (cp[0] == 0x1f && cp[1] == 0x8b) {
|
||||
if (0 == fd_read_gzip_from_memory_init(fd)) {
|
||||
blo_filedata_free(fd);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
fd->read = fd_read_from_memory;
|
||||
}
|
||||
|
||||
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
|
||||
fd->file = file;
|
||||
|
||||
return blo_decode_and_check(fd, reports->reports);
|
||||
}
|
||||
|
@ -1597,11 +1332,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
|
|||
}
|
||||
|
||||
FileData *fd = filedata_new(reports);
|
||||
fd->memfile = memfile;
|
||||
fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction);
|
||||
fd->undo_direction = params->undo_direction;
|
||||
|
||||
fd->read = fd_read_from_memfile;
|
||||
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
|
||||
fd->flags |= FD_FLAGS_IS_MEMFILE;
|
||||
|
||||
return blo_decode_and_check(fd, reports->reports);
|
||||
}
|
||||
|
@ -1609,30 +1342,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
|
|||
void blo_filedata_free(FileData *fd)
|
||||
{
|
||||
if (fd) {
|
||||
if (fd->filedes != -1) {
|
||||
close(fd->filedes);
|
||||
}
|
||||
|
||||
if (fd->gzfiledes != NULL) {
|
||||
gzclose(fd->gzfiledes);
|
||||
}
|
||||
|
||||
if (fd->strm.next_in) {
|
||||
int err = inflateEnd(&fd->strm);
|
||||
if (err != Z_OK) {
|
||||
CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err);
|
||||
}
|
||||
}
|
||||
|
||||
if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) {
|
||||
MEM_freeN((void *)fd->buffer);
|
||||
fd->buffer = NULL;
|
||||
}
|
||||
|
||||
if (fd->mmap_file) {
|
||||
BLI_mmap_free(fd->mmap_file);
|
||||
fd->mmap_file = NULL;
|
||||
}
|
||||
fd->file->close(fd->file);
|
||||
|
||||
/* Free all BHeadN data blocks */
|
||||
#ifndef NDEBUG
|
||||
|
@ -1640,7 +1350,7 @@ void blo_filedata_free(FileData *fd)
|
|||
#else
|
||||
/* Sanity check we're not keeping memory we don't need. */
|
||||
LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) {
|
||||
if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
|
||||
if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
|
||||
BLI_assert(new_bhead->has_data == 0);
|
||||
}
|
||||
MEM_freeN(new_bhead);
|
||||
|
@ -2096,7 +1806,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id),
|
|||
|
||||
void blo_cache_storage_init(FileData *fd, Main *bmain)
|
||||
{
|
||||
if (fd->memfile != NULL) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
BLI_assert(fd->cache_storage == NULL);
|
||||
fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__);
|
||||
fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
|
||||
|
@ -2261,7 +1971,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
|
|||
* undo since DNA must match. */
|
||||
static const void *peek_struct_undo(FileData *fd, BHead *bhead)
|
||||
{
|
||||
BLI_assert(fd->memfile != NULL);
|
||||
BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE);
|
||||
UNUSED_VARS_NDEBUG(fd);
|
||||
return (bhead->len) ? (const void *)(bhead + 1) : NULL;
|
||||
}
|
||||
|
@ -3679,7 +3389,7 @@ static BHead *read_libblock(FileData *fd,
|
|||
* When datablocks are changed but still exist, we restore them at the old
|
||||
* address and inherit recalc flags for the dependency graph. */
|
||||
ID *id_old = NULL;
|
||||
if (fd->memfile != NULL) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) {
|
||||
if (r_id) {
|
||||
*r_id = id_old;
|
||||
|
@ -3980,13 +3690,14 @@ static void lib_link_all(FileData *fd, Main *bmain)
|
|||
continue;
|
||||
}
|
||||
|
||||
if (fd->memfile != NULL && GS(id->name) == ID_WM) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) {
|
||||
/* No load UI for undo memfiles.
|
||||
* Only WM currently, SCR needs it still (see below), and so does WS? */
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo &&
|
||||
(id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
|
||||
/* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across
|
||||
* current undo step, and old IDs re-use their old memory address, we do not need to liblink
|
||||
* it at all. */
|
||||
|
@ -4165,7 +3876,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
|||
BlendFileData *bfd;
|
||||
ListBase mainlist = {NULL, NULL};
|
||||
|
||||
if (fd->memfile != NULL) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step");
|
||||
}
|
||||
|
||||
|
@ -4256,7 +3967,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
|||
}
|
||||
|
||||
/* do before read_libraries, but skip undo case */
|
||||
if (fd->memfile == NULL) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
|
||||
if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) {
|
||||
do_versions(fd, NULL, bfd->main);
|
||||
}
|
||||
|
@ -4278,7 +3989,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
|||
fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries;
|
||||
|
||||
/* Skip in undo case. */
|
||||
if (fd->memfile == NULL) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
|
||||
/* Note that we can't recompute user-counts at this point in undo case, we play too much with
|
||||
* IDs from different memory realms, and Main database is not in a fully valid state yet.
|
||||
*/
|
||||
|
@ -4311,7 +4022,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
|||
|
||||
/* Now that all our data-blocks are loaded,
|
||||
* we can re-generate overrides from their references. */
|
||||
if (fd->memfile == NULL) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
|
||||
/* Do not apply in undo case! */
|
||||
fd->reports->duration.lib_overrides = PIL_check_seconds_timer();
|
||||
|
||||
|
@ -4391,7 +4102,7 @@ static void sort_bhead_old_map(FileData *fd)
|
|||
static BHead *find_previous_lib(FileData *fd, BHead *bhead)
|
||||
{
|
||||
/* Skip library data-blocks in undo, see comment in read_libblock. */
|
||||
if (fd->memfile) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
@ -5850,7 +5561,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p)
|
|||
|
||||
bool BLO_read_data_is_undo(BlendDataReader *reader)
|
||||
{
|
||||
return reader->fd->memfile != NULL;
|
||||
return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
|
||||
}
|
||||
|
||||
void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr)
|
||||
|
@ -5870,7 +5581,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader)
|
|||
|
||||
bool BLO_read_lib_is_undo(BlendLibReader *reader)
|
||||
{
|
||||
return reader->fd->memfile != NULL;
|
||||
return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
|
||||
}
|
||||
|
||||
Main *BLO_read_lib_get_main(BlendLibReader *reader)
|
||||
|
|
|
@ -28,10 +28,10 @@
|
|||
# include "BLI_winstuff.h"
|
||||
#endif
|
||||
|
||||
#include "BLI_filereader.h"
|
||||
#include "DNA_sdna_types.h"
|
||||
#include "DNA_space_types.h"
|
||||
#include "DNA_windowmanager_types.h" /* for ReportType */
|
||||
#include "zlib.h"
|
||||
|
||||
struct BLI_mmap_file;
|
||||
struct BLOCacheStorage;
|
||||
|
@ -50,7 +50,7 @@ enum eFileDataFlag {
|
|||
FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1,
|
||||
FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2,
|
||||
FD_FLAGS_FILE_OK = 1 << 3,
|
||||
FD_FLAGS_NOT_MY_BUFFER = 1 << 4,
|
||||
FD_FLAGS_IS_MEMFILE = 1 << 4,
|
||||
/* XXX Unused in practice (checked once but never set). */
|
||||
FD_FLAGS_NOT_MY_LIBMAP = 1 << 5,
|
||||
};
|
||||
|
@ -60,44 +60,18 @@ enum eFileDataFlag {
|
|||
# pragma GCC poison off_t
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
|
||||
typedef int64_t off64_t;
|
||||
#endif
|
||||
|
||||
typedef ssize_t(FileDataReadFn)(struct FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *r_is_memchunk_identical);
|
||||
typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence);
|
||||
|
||||
typedef struct FileData {
|
||||
/** Linked list of BHeadN's. */
|
||||
ListBase bhead_list;
|
||||
enum eFileDataFlag flags;
|
||||
bool is_eof;
|
||||
size_t buffersize;
|
||||
off64_t file_offset;
|
||||
|
||||
FileDataReadFn *read;
|
||||
FileDataSeekFn *seek;
|
||||
FileReader *file;
|
||||
|
||||
/** Regular file reading. */
|
||||
int filedes;
|
||||
|
||||
/** Variables needed for reading from memory / stream / memory-mapped files. */
|
||||
const char *buffer;
|
||||
struct BLI_mmap_file *mmap_file;
|
||||
/** Variables needed for reading from memfile (undo). */
|
||||
struct MemFile *memfile;
|
||||
/** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use
|
||||
* to detect unchanged data from memfile. */
|
||||
int undo_direction; /* eUndoStepDir */
|
||||
|
||||
/** Variables needed for reading from file. */
|
||||
gzFile gzfiledes;
|
||||
/** Gzip stream for memory decompression. */
|
||||
z_stream strm;
|
||||
|
||||
/** Now only in use for library appending. */
|
||||
char relabase[FILE_MAX];
|
||||
|
||||
|
|
|
@ -48,6 +48,7 @@
|
|||
|
||||
#include "BKE_lib_id.h"
|
||||
#include "BKE_main.h"
|
||||
#include "BKE_undo_system.h"
|
||||
|
||||
/* keep last */
|
||||
#include "BLI_strict_flags.h"
|
||||
|
@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename)
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static ssize_t undo_read(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
UndoReader *undo = (UndoReader *)reader;
|
||||
|
||||
static size_t seek = SIZE_MAX; /* The current position. */
|
||||
static size_t offset = 0; /* Size of previous chunks. */
|
||||
static MemFileChunk *chunk = NULL;
|
||||
size_t chunkoffset, readsize, totread;
|
||||
|
||||
undo->memchunk_identical = true;
|
||||
|
||||
if (size == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (seek != (size_t)undo->reader.offset) {
|
||||
chunk = undo->memfile->chunks.first;
|
||||
seek = 0;
|
||||
|
||||
while (chunk) {
|
||||
if (seek + chunk->size > (size_t)undo->reader.offset) {
|
||||
break;
|
||||
}
|
||||
seek += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
offset = seek;
|
||||
seek = (size_t)undo->reader.offset;
|
||||
}
|
||||
|
||||
if (chunk) {
|
||||
totread = 0;
|
||||
|
||||
do {
|
||||
/* First check if it's on the end if current chunk. */
|
||||
if (seek - offset == chunk->size) {
|
||||
offset += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
|
||||
/* Debug, should never happen. */
|
||||
if (chunk == NULL) {
|
||||
printf("illegal read, chunk zero\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
chunkoffset = seek - offset;
|
||||
readsize = size - totread;
|
||||
|
||||
/* Data can be spread over multiple chunks, so clamp size
|
||||
* to within this chunk, and then it will read further in
|
||||
* the next chunk. */
|
||||
if (chunkoffset + readsize > chunk->size) {
|
||||
readsize = chunk->size - chunkoffset;
|
||||
}
|
||||
|
||||
memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
|
||||
totread += readsize;
|
||||
undo->reader.offset += (off64_t)readsize;
|
||||
seek += readsize;
|
||||
|
||||
/* `is_identical` of current chunk represents whether it changed compared to previous undo
|
||||
* step. this is fine in redo case, but not in undo case, where we need an extra flag
|
||||
* defined when saving the next (future) step after the one we want to restore, as we are
|
||||
* supposed to 'come from' that future undo step, and not the one before current one. */
|
||||
undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical :
|
||||
chunk->is_identical_future;
|
||||
} while (totread < size);
|
||||
|
||||
return (ssize_t)totread;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void undo_close(FileReader *reader)
|
||||
{
|
||||
MEM_freeN(reader);
|
||||
}
|
||||
|
||||
FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction)
|
||||
{
|
||||
UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__);
|
||||
|
||||
undo->memfile = memfile;
|
||||
undo->undo_direction = undo_direction;
|
||||
|
||||
undo->reader.read = undo_read;
|
||||
undo->reader.seek = NULL;
|
||||
undo->reader.close = undo_close;
|
||||
|
||||
return (FileReader *)undo;
|
||||
}
|
||||
|
|
|
@ -23,8 +23,7 @@
|
|||
#else
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h> /* for open close read */
|
||||
# include <zlib.h> /* odd include order-issue */
|
||||
# include <io.h> /* for open close read */
|
||||
#endif
|
||||
|
||||
/* allow readfile to use deprecated functionality */
|
||||
|
|
|
@ -28,8 +28,7 @@
|
|||
#else
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h> /* for open close read */
|
||||
# include <zlib.h> /* odd include order-issue */
|
||||
# include <io.h> /* for open close read */
|
||||
#endif
|
||||
|
||||
/* allow readfile to use deprecated functionality */
|
||||
|
|
|
@ -78,12 +78,12 @@
|
|||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <zlib.h>
|
||||
|
||||
#ifdef WIN32
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h>
|
||||
# include <zlib.h> /* odd include order-issue */
|
||||
#else
|
||||
# include <unistd.h> /* FreeBSD, for write() and close(). */
|
||||
#endif
|
||||
|
|
|
@ -28,11 +28,10 @@
|
|||
* winsock stuff.
|
||||
*/
|
||||
#include <errno.h>
|
||||
#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "zlib.h" /* wm_read_exotic() */
|
||||
|
||||
#ifdef WIN32
|
||||
/* Need to include windows.h so _WIN32_IE is defined. */
|
||||
# include <windows.h>
|
||||
|
@ -51,6 +50,7 @@
|
|||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_fileops_types.h"
|
||||
#include "BLI_filereader.h"
|
||||
#include "BLI_linklist.h"
|
||||
#include "BLI_math.h"
|
||||
#include "BLI_system.h"
|
||||
|
@ -481,53 +481,59 @@ static void wm_init_userdef(Main *bmain)
|
|||
/* intended to check for non-blender formats but for now it only reads blends */
|
||||
static int wm_read_exotic(const char *name)
|
||||
{
|
||||
int len;
|
||||
gzFile gzfile;
|
||||
char header[7];
|
||||
int retval;
|
||||
|
||||
/* make sure we're not trying to read a directory.... */
|
||||
|
||||
len = strlen(name);
|
||||
if (len > 0 && ELEM(name[len - 1], '/', '\\')) {
|
||||
retval = BKE_READ_EXOTIC_FAIL_PATH;
|
||||
int namelen = strlen(name);
|
||||
if (namelen > 0 && ELEM(name[namelen - 1], '/', '\\')) {
|
||||
return BKE_READ_EXOTIC_FAIL_PATH;
|
||||
}
|
||||
|
||||
/* open the file. */
|
||||
const int filedes = BLI_open(name, O_BINARY | O_RDONLY, 0);
|
||||
if (filedes == -1) {
|
||||
return BKE_READ_EXOTIC_FAIL_OPEN;
|
||||
}
|
||||
|
||||
FileReader *rawfile = BLI_filereader_new_file(filedes);
|
||||
if (rawfile == NULL) {
|
||||
return BKE_READ_EXOTIC_FAIL_OPEN;
|
||||
}
|
||||
|
||||
/* read the header (7 bytes are enough to identify all known types). */
|
||||
char header[7];
|
||||
if (rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
|
||||
rawfile->close(rawfile);
|
||||
return BKE_READ_EXOTIC_FAIL_FORMAT;
|
||||
}
|
||||
rawfile->seek(rawfile, 0, SEEK_SET);
|
||||
|
||||
/* check for uncompressed .blend */
|
||||
if (STREQLEN(header, "BLENDER", 7)) {
|
||||
rawfile->close(rawfile);
|
||||
return BKE_READ_EXOTIC_OK_BLEND;
|
||||
}
|
||||
|
||||
/* check for compressed .blend */
|
||||
FileReader *compressed_file = NULL;
|
||||
if (BLI_file_magic_is_gzip(header)) {
|
||||
compressed_file = BLI_filereader_new_gzip(rawfile);
|
||||
}
|
||||
|
||||
/* If a compression signature matches, try decompressing the start and check if it's a .blend */
|
||||
if (compressed_file != NULL) {
|
||||
size_t len = compressed_file->read(compressed_file, header, sizeof(header));
|
||||
compressed_file->close(compressed_file);
|
||||
if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
|
||||
return BKE_READ_EXOTIC_OK_BLEND;
|
||||
}
|
||||
}
|
||||
else {
|
||||
gzfile = BLI_gzopen(name, "rb");
|
||||
if (gzfile == NULL) {
|
||||
retval = BKE_READ_EXOTIC_FAIL_OPEN;
|
||||
}
|
||||
else {
|
||||
len = gzread(gzfile, header, sizeof(header));
|
||||
gzclose(gzfile);
|
||||
if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
|
||||
retval = BKE_READ_EXOTIC_OK_BLEND;
|
||||
}
|
||||
else {
|
||||
/* We may want to support loading other file formats
|
||||
* from their header bytes or file extension.
|
||||
* This used to be supported in the code below and may be added
|
||||
* back at some point. */
|
||||
#if 0
|
||||
WM_cursor_wait(true);
|
||||
|
||||
if (is_foo_format(name)) {
|
||||
read_foo(name);
|
||||
retval = BKE_READ_EXOTIC_OK_OTHER;
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
retval = BKE_READ_EXOTIC_FAIL_FORMAT;
|
||||
}
|
||||
#if 0
|
||||
WM_cursor_wait(false);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
rawfile->close(rawfile);
|
||||
}
|
||||
|
||||
return retval;
|
||||
/* Add check for future file formats here. */
|
||||
|
||||
return BKE_READ_EXOTIC_FAIL_FORMAT;
|
||||
}
|
||||
|
||||
/** \} */
|
||||
|
|
Loading…
Reference in New Issue