From 28fa5bc347390480fe190294c6c385b6a9f0d68b Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Sun, 20 May 2012 13:28:05 -0700 Subject: [PATCH 01/16] system/core: move libsparse into system/core This moves an exact copy of libsparse from system/extras/ext4_utils/libsparse to system/core/libsparse in preparation for linking tools in system/core against it. Change-Id: If664e4fcfd6612844ac745589beb1517e7f9fe58 --- libsparse/Android.mk | 86 +++++ libsparse/backed_block.c | 179 +++++++++ libsparse/backed_block.h | 39 ++ libsparse/img2simg.c | 327 ++++++++++++++++ libsparse/include/sparse/sparse.h | 134 +++++++ libsparse/output_file.c | 613 ++++++++++++++++++++++++++++++ libsparse/output_file.h | 36 ++ libsparse/simg2img.c | 321 ++++++++++++++++ libsparse/simg_dump.py | 169 ++++++++ libsparse/sparse.c | 154 ++++++++ libsparse/sparse_crc32.c | 111 ++++++ libsparse/sparse_crc32.h | 18 + libsparse/sparse_defs.h | 48 +++ libsparse/sparse_file.h | 30 ++ libsparse/sparse_format.h | 55 +++ 15 files changed, 2320 insertions(+) create mode 100644 libsparse/Android.mk create mode 100644 libsparse/backed_block.c create mode 100644 libsparse/backed_block.h create mode 100644 libsparse/img2simg.c create mode 100644 libsparse/include/sparse/sparse.h create mode 100644 libsparse/output_file.c create mode 100644 libsparse/output_file.h create mode 100644 libsparse/simg2img.c create mode 100755 libsparse/simg_dump.py create mode 100644 libsparse/sparse.c create mode 100644 libsparse/sparse_crc32.c create mode 100644 libsparse/sparse_crc32.h create mode 100644 libsparse/sparse_defs.h create mode 100644 libsparse/sparse_file.h create mode 100644 libsparse/sparse_format.h diff --git a/libsparse/Android.mk b/libsparse/Android.mk new file mode 100644 index 000000000..dbe4d1842 --- /dev/null +++ b/libsparse/Android.mk @@ -0,0 +1,86 @@ +# Copyright 2010 The Android Open Source Project + +LOCAL_PATH:= $(call my-dir) + +libsparse_src_files := \ + backed_block.c \ + output_file.c \ + sparse.c \ + sparse_crc32.c + +include $(CLEAR_VARS) + +LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/include +LOCAL_SRC_FILES := $(libsparse_src_files) +LOCAL_MODULE := libsparse +LOCAL_MODULE_TAGS := optional +LOCAL_STATIC_LIBRARIES := libz +LOCAL_C_INCLUDES += $(LOCAL_PATH)/include external/zlib + +include $(BUILD_HOST_STATIC_LIBRARY) + +include $(CLEAR_VARS) + +LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/include +LOCAL_SRC_FILES := $(libsparse_src_files) +LOCAL_MODULE := libsparse +LOCAL_MODULE_TAGS := optional +LOCAL_C_INCLUDES += $(LOCAL_PATH)/include external/zlib +LOCAL_SHARED_LIBRARIES := libz + +include $(BUILD_SHARED_LIBRARY) + +include $(CLEAR_VARS) + +LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/include +LOCAL_SRC_FILES := $(libsparse_src_files) +LOCAL_MODULE := libsparse +LOCAL_MODULE_TAGS := optional +LOCAL_C_INCLUDES += $(LOCAL_PATH)/include external/zlib +LOCAL_STATIC_LIBRARIES := libz + +include $(BUILD_STATIC_LIBRARY) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := simg2img.c \ + sparse_crc32.c +LOCAL_MODULE := simg2img +LOCAL_MODULE_TAGS := debug + +include $(BUILD_HOST_EXECUTABLE) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := simg2img.c \ + sparse_crc32.c +LOCAL_MODULE := simg2img +LOCAL_MODULE_TAGS := optional + +include $(BUILD_EXECUTABLE) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := img2simg.c +LOCAL_MODULE := img2simg +LOCAL_MODULE_TAGS := debug + +include $(BUILD_HOST_EXECUTABLE) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := img2simg.c +LOCAL_MODULE := img2simg +LOCAL_MODULE_TAGS := optional + +include $(BUILD_EXECUTABLE) + +include $(CLEAR_VARS) + +LOCAL_MODULE := simg_dump.py +LOCAL_MODULE_TAGS := debug +LOCAL_SRC_FILES := simg_dump.py +LOCAL_MODULE_CLASS := EXECUTABLES +LOCAL_IS_HOST_MODULE := true + +include $(BUILD_PREBUILT) diff --git a/libsparse/backed_block.c b/libsparse/backed_block.c new file mode 100644 index 000000000..254813809 --- /dev/null +++ b/libsparse/backed_block.c @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "backed_block.h" +#include "sparse_defs.h" + +struct data_block { + u32 block; + u32 len; + void *data; + const char *filename; + int64_t offset; + struct data_block *next; + u32 fill_val; + u8 fill; + u8 pad1; + u16 pad2; +}; + +static struct data_block *data_blocks = NULL; +static struct data_block *last_used = NULL; + +static void queue_db(struct data_block *new_db) +{ + struct data_block *db; + + if (data_blocks == NULL) { + data_blocks = new_db; + return; + } + + if (data_blocks->block > new_db->block) { + new_db->next = data_blocks; + data_blocks = new_db; + return; + } + + /* Optimization: blocks are mostly queued in sequence, so save the + pointer to the last db that was added, and start searching from + there if the next block number is higher */ + if (last_used && new_db->block > last_used->block) + db = last_used; + else + db = data_blocks; + last_used = new_db; + + for (; db->next && db->next->block < new_db->block; db = db->next) + ; + + if (db->next == NULL) { + db->next = new_db; + } else { + new_db->next = db->next; + db->next = new_db; + } +} + +/* Queues a fill block of memory to be written to the specified data blocks */ +void queue_fill_block(unsigned int fill_val, unsigned int len, unsigned int block) +{ + struct data_block *db = malloc(sizeof(struct data_block)); + if (db == NULL) { + error_errno("malloc"); + return; + } + + db->block = block; + db->len = len; + db->fill = 1; + db->fill_val = fill_val; + db->data = NULL; + db->filename = NULL; + db->next = NULL; + + queue_db(db); +} + +/* Queues a block of memory to be written to the specified data blocks */ +void queue_data_block(void *data, unsigned int len, unsigned int block) +{ + struct data_block *db = malloc(sizeof(struct data_block)); + if (db == NULL) { + error_errno("malloc"); + return; + } + + db->block = block; + db->len = len; + db->data = data; + db->filename = NULL; + db->fill = 0; + db->next = NULL; + + queue_db(db); +} + +/* Queues a chunk of a file on disk to be written to the specified data blocks */ +void queue_data_file(const char *filename, int64_t offset, unsigned int len, + unsigned int block) +{ + struct data_block *db = malloc(sizeof(struct data_block)); + if (db == NULL) { + error_errno("malloc"); + return; + } + + db->block = block; + db->len = len; + db->filename = strdup(filename); + db->offset = offset; + db->data = NULL; + db->fill = 0; + db->next = NULL; + + queue_db(db); +} + +/* Iterates over the queued data blocks, calling data_func for each contiguous + data block, and file_func for each contiguous file block */ +void for_each_data_block(data_block_callback_t data_func, + data_block_file_callback_t file_func, + data_block_fill_callback_t fill_func, void *priv, unsigned int block_size) +{ + struct data_block *db; + u32 last_block = 0; + + for (db = data_blocks; db; db = db->next) { + if (db->block < last_block) + error("data blocks out of order: %u < %u", db->block, last_block); + last_block = db->block + DIV_ROUND_UP(db->len, block_size) - 1; + + if (db->filename) + file_func(priv, (u64)db->block * block_size, db->filename, db->offset, db->len); + else if (db->fill) + fill_func(priv, (u64)db->block * block_size, db->fill_val, db->len); + else + data_func(priv, (u64)db->block * block_size, db->data, db->len); + } +} + +/* Frees the memory used by the linked list of data blocks */ +void free_data_blocks() +{ + if (!data_blocks) return; + struct data_block *db = data_blocks; + while (db) { + struct data_block *next = db->next; + free((void*)db->filename); + + // There used to be a free() of db->data here, but it + // made the function crash since queue_data_block() is + // sometimes passed pointers it can't take ownership of + // (like a pointer into the middle of an allocated + // block). It's not clear what the queue_data_block + // contract is supposed to be, but we'd rather leak + // memory than crash. + + free(db); + db = next; + } + data_blocks = NULL; + last_used = NULL; +} diff --git a/libsparse/backed_block.h b/libsparse/backed_block.h new file mode 100644 index 000000000..7b7c90aad --- /dev/null +++ b/libsparse/backed_block.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _BACKED_BLOCK_H_ +#define _BACKED_BLOCK_H_ + +#include + +typedef void (*data_block_callback_t)(void *priv, int64_t off, void *data, int len); +typedef void (*data_block_fill_callback_t)(void *priv, int64_t off, unsigned int fill_val, int len); +typedef void (*data_block_file_callback_t)(void *priv, int64_t off, + const char *file, int64_t offset, + int len); + +void for_each_data_block(data_block_callback_t data_func, + data_block_file_callback_t file_func, + data_block_fill_callback_t fill_func, void *priv, unsigned int); + +void queue_data_block(void *data, unsigned int len, unsigned int block); +void queue_fill_block(unsigned int fill_val, unsigned int len, unsigned int block); +void queue_data_file(const char *filename, int64_t offset, unsigned int len, + unsigned int block); + +void free_data_blocks(); + +#endif diff --git a/libsparse/img2simg.c b/libsparse/img2simg.c new file mode 100644 index 000000000..a1594df63 --- /dev/null +++ b/libsparse/img2simg.c @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2010-2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define DEFAULT_BLOCK_SIZE "4K" +#define DEFAULT_CHUNK_SIZE "64M" +#define DEFAULT_SUFFIX "%03d" + +#include "sparse_format.h" +#if 0 /* endian.h is not on all platforms */ +# include +#else + /* For now, just assume we're going to run on little-endian. */ +# define my_htole32(h) (h) +# define my_htole16(h) (h) +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define COPY_BUF_SIZE (1024*1024) +static char *copy_buf; + +static const char *progname(const char *argv0) +{ + const char *prog_name; + if ((prog_name = strrchr(argv0, '/'))) + return(prog_name + 1); /* Advance beyond '/'. */ + return(argv0); /* No '/' in argv0, use it as is. */ +} + +static void error_exit(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + fputc('\n', stderr); + va_end(ap); + + exit(EXIT_FAILURE); +} + +static void usage(const char *argv0, const char *error_fmt, ...) +{ + fprintf(stderr, + "Usage: %s [OPTIONS] \n", + progname(argv0)); + fprintf(stderr, "The will be split into as many sparse\n"); + fprintf(stderr, "files as needed. Each sparse file will contain a single\n"); + fprintf(stderr, "DONT CARE chunk to offset to the correct block and then\n"); + fprintf(stderr, "a single RAW chunk containing a portion of the data from\n"); + fprintf(stderr, "the raw image file. The sparse files will be named by\n"); + fprintf(stderr, "appending a number to the name of the raw image file.\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "OPTIONS (Defaults are enclosed by square brackets):\n"); + fprintf(stderr, " -s SUFFIX Format appended number with SUFFIX [%s]\n", + DEFAULT_SUFFIX); + fprintf(stderr, " -B SIZE Use a block size of SIZE [%s]\n", + DEFAULT_BLOCK_SIZE); + fprintf(stderr, " -C SIZE Use a chunk size of SIZE [%s]\n", + DEFAULT_CHUNK_SIZE); + fprintf(stderr, "SIZE is a decimal integer that may optionally be\n"); + fprintf(stderr, "followed by a suffix that specifies a multiplier for\n"); + fprintf(stderr, "the integer:\n"); + fprintf(stderr, " c 1 byte (the default when omitted)\n"); + fprintf(stderr, " w 2 bytes\n"); + fprintf(stderr, " b 512 bytes\n"); + fprintf(stderr, " kB 1000 bytes\n"); + fprintf(stderr, " K 1024 bytes\n"); + fprintf(stderr, " MB 1000*1000 bytes\n"); + fprintf(stderr, " M 1024*1024 bytes\n"); + fprintf(stderr, " GB 1000*1000*1000 bytes\n"); + fprintf(stderr, " G 1024*1024*1024 bytes\n"); + + if (error_fmt && *error_fmt) + { + fprintf(stderr, "\n"); + va_list ap; + va_start(ap, error_fmt); + vfprintf(stderr, error_fmt, ap); + va_end(ap); + fprintf(stderr, "\n"); + } + + exit(EXIT_FAILURE); +} + +static void cpy_file(int out_fd, char *out_path, int in_fd, char *in_path, + size_t len) +{ + ssize_t s, cpy_len = COPY_BUF_SIZE; + + while (len) { + if (len < COPY_BUF_SIZE) + cpy_len = len; + + s = read(in_fd, copy_buf, cpy_len); + if (s < 0) + error_exit("\"%s\": %s", in_path, strerror(errno)); + if (!s) + error_exit("\"%s\": Unexpected EOF", in_path); + + cpy_len = s; + + s = write(out_fd, copy_buf, cpy_len); + if (s < 0) + error_exit("\"%s\": %s", out_path, strerror(errno)); + if (s != cpy_len) + error_exit("\"%s\": Short data write (%lu)", out_path, + (unsigned long)s); + + len -= cpy_len; + } +} + +static int parse_size(const char *size_str, size_t *size) +{ + static const size_t MAX_SIZE_T = ~(size_t)0; + size_t mult; + unsigned long long int value; + const char *end; + errno = 0; + value = strtoull(size_str, (char **)&end, 10); + if (errno != 0 || end == size_str || value > MAX_SIZE_T) + return -1; + if (*end == '\0') { + *size = value; + return 0; + } + if (!strcmp(end, "c")) + mult = 1; + else if (!strcmp(end, "w")) + mult = 2; + else if (!strcmp(end, "b")) + mult = 512; + else if (!strcmp(end, "kB")) + mult = 1000; + else if (!strcmp(end, "K")) + mult = 1024; + else if (!strcmp(end, "MB")) + mult = (size_t)1000*1000; + else if (!strcmp(end, "M")) + mult = (size_t)1024*1024; + else if (!strcmp(end, "GB")) + mult = (size_t)1000*1000*1000; + else if (!strcmp(end, "G")) + mult = (size_t)1024*1024*1024; + else + return -1; + + if (value > MAX_SIZE_T / mult) + return -1; + *size = value * mult; + return 0; +} + +int main(int argc, char *argv[]) +{ + char *suffix = DEFAULT_SUFFIX; + char *block_size_str = DEFAULT_BLOCK_SIZE; + char *chunk_size_str = DEFAULT_CHUNK_SIZE; + size_t block_size, chunk_size, blocks_per_chunk, to_write; + char *in_path, *out_path, *out_fmt; + int in_fd, out_fd; + struct stat in_st; + off_t left_to_write; + struct { + sparse_header_t sparse_hdr; + chunk_header_t dont_care_hdr; + chunk_header_t raw_hdr; + } file_hdr; + unsigned int file_count; + ssize_t s; + int i; + + /* Parse the command line. */ + while ((i = getopt(argc, argv, "s:B:C:")) != -1) + { + switch (i) { + case 's': + suffix = optarg; + break; + case 'B': + block_size_str = optarg; + break; + case 'C': + chunk_size_str = optarg; + break; + default: + usage(argv[0], NULL); + break; + } + } + + if (parse_size(block_size_str, &block_size)) + usage(argv[0], "Can not parse \"%s\" as a block size.", + block_size_str); + if (block_size % 4096) + usage(argv[0], "Block size is not a multiple of 4096."); + + if (parse_size(chunk_size_str, &chunk_size)) + usage(argv[0], "Can not parse \"%s\" as a chunk size.", + chunk_size_str); + if (chunk_size % block_size) + usage(argv[0], "Chunk size is not a multiple of the block size."); + blocks_per_chunk = chunk_size / block_size; + + if ((argc - optind) != 1) + usage(argv[0], "Missing or extra arguments."); + in_path = argv[optind]; + + /* Open the input file and validate it. */ + if ((in_fd = open(in_path, O_RDONLY)) < 0) + error_exit("open \"%s\": %s", in_path, strerror(errno)); + if (fstat(in_fd, &in_st)) + error_exit("fstat \"%s\": %s", in_path, strerror(errno)); + left_to_write = in_st.st_size; + if (left_to_write % block_size) + error_exit( + "\"%s\" size (%llu) is not a multiple of the block size (%llu).\n", + in_path, + (unsigned long long)left_to_write, (unsigned long long)block_size); + + /* Get a buffer for copying the chunks. */ + if ((copy_buf = malloc(COPY_BUF_SIZE)) == 0) + error_exit("malloc copy buffer: %s", strerror(errno)); + + /* Get a buffer for a sprintf format to form output paths. */ + if ((out_fmt = malloc(sizeof("%s") + strlen(suffix))) == 0) + error_exit("malloc format buffer: %s", strerror(errno)); + out_fmt[0] = '%'; + out_fmt[1] = 's'; + strcpy(out_fmt + 2, suffix); + + /* Get a buffer for an output path. */ + i = snprintf(copy_buf, COPY_BUF_SIZE, out_fmt, in_path, UINT_MAX); + if (i >= COPY_BUF_SIZE) + error_exit("Ridulously long suffix: %s", suffix); + if ((out_path = malloc(i + 1)) == 0) + error_exit("malloc output path buffer: %s", strerror(errno)); + + /* + * Each file gets a sparse_header, a Don't Care chunk to offset to + * where the data belongs and then a Raw chunk with the actual data. + */ + memset((void *)&file_hdr.sparse_hdr, 0, sizeof(file_hdr.sparse_hdr)); + file_hdr.sparse_hdr.magic = my_htole32(SPARSE_HEADER_MAGIC); + file_hdr.sparse_hdr.major_version = my_htole16(1); + file_hdr.sparse_hdr.minor_version = my_htole16(0); + file_hdr.sparse_hdr.file_hdr_sz = my_htole16(sizeof(sparse_header_t)); + file_hdr.sparse_hdr.chunk_hdr_sz = my_htole16(sizeof(chunk_header_t)); + file_hdr.sparse_hdr.blk_sz = my_htole32(block_size); + /* The total_blks will be set in the file loop below. */ + file_hdr.sparse_hdr.total_chunks = my_htole32(2); + file_hdr.sparse_hdr.image_checksum = my_htole32(0); /* Typically unused. */ + + memset((void *)&file_hdr.dont_care_hdr, 0, sizeof(file_hdr.dont_care_hdr)); + file_hdr.dont_care_hdr.chunk_type = my_htole16(CHUNK_TYPE_DONT_CARE); + /* The Don't Care's chunk_sz will be set in the file loop below. */ + file_hdr.dont_care_hdr.total_sz = my_htole32(sizeof(chunk_header_t)); + + memset((void *)&file_hdr.raw_hdr, 0, sizeof(file_hdr.raw_hdr)); + file_hdr.raw_hdr.chunk_type = my_htole16(CHUNK_TYPE_RAW); + file_hdr.raw_hdr.chunk_sz = my_htole32(blocks_per_chunk); + file_hdr.raw_hdr.total_sz = my_htole32(chunk_size + sizeof(chunk_header_t)); + + /* Loop through writing chunk_size to each of the output files. */ + to_write = chunk_size; + for (file_count = 1; left_to_write ; file_count++) { + /* Fix up the headers on the last block. */ + if (left_to_write < (off_t)chunk_size) { + to_write = left_to_write; + file_hdr.raw_hdr.chunk_sz = my_htole32(left_to_write / block_size); + file_hdr.raw_hdr.total_sz = my_htole32(left_to_write + + sizeof(chunk_header_t)); + } + + /* Form the pathname for this output file and open it. */ + sprintf(out_path, out_fmt, in_path, file_count); + if ((out_fd = creat(out_path, 0666)) < 0) + error_exit("\"%s\": %s", out_path, strerror(errno)); + + /* Update and write the headers to this output file. */ + s = (file_count-1) * blocks_per_chunk; + file_hdr.dont_care_hdr.chunk_sz = my_htole32(s); + file_hdr.sparse_hdr.total_blks = my_htole32(s + + (to_write / block_size)); + s = write(out_fd, (void *)&file_hdr, sizeof(file_hdr)); + if (s < 0) + error_exit("\"%s\": %s", out_path, strerror(errno)); + if (s != sizeof(file_hdr)) + error_exit("\"%s\": Short write (%lu)", out_path, (unsigned long)s); + + /* Copy this chunk from the input file to the output file. */ + cpy_file(out_fd, out_path, in_fd, in_path, to_write); + + /* Close this output file and update the amount left to write. */ + if (close(out_fd)) + error_exit("close \"%s\": %s", out_path, strerror(errno)); + left_to_write -= to_write; + } + + if (close(in_fd)) + error_exit("close \"%s\": %s", in_path, strerror(errno)); + + exit(EXIT_SUCCESS); +} diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h new file mode 100644 index 000000000..db0688471 --- /dev/null +++ b/libsparse/include/sparse/sparse.h @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_H_ +#define _LIBSPARSE_SPARSE_H_ + +#include +#include + +struct sparse_file; + +/** + * sparse_file_new - create a new sparse file cookie + * + * @block_size - minimum size of a chunk + * @len - size of the expanded sparse file. + * + * Creates a new sparse_file cookie that can be used to associate data + * blocks. Can later be written to a file with a variety of options. + * block_size specifies the minimum size of a chunk in the file. The maximum + * size of the file is 2**32 * block_size (16TB for 4k block size). + * + * Returns the sparse file cookie, or NULL on error. + */ +struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len); + +/** + * sparse_file_destroy - destroy a sparse file cookie + * + * @s - sparse file cookie + * + * Destroys a sparse file cookie. After destroy, all memory passed in to + * sparse_file_add_data can be freed by the caller + */ +void sparse_file_destroy(struct sparse_file *s); + +/** + * sparse_file_add_data - associate a data chunk with a sparse file + * + * @s - sparse file cookie + * @data - pointer to data block + * @len - length of the data block + * @block - offset in blocks into the sparse file to place the data chunk + * + * Associates a data chunk with a sparse file cookie. The region + * [block * block_size : block * block_size + len) must not already be used in + * the sparse file. If len is not a multiple of the block size the data + * will be padded with zeros. + * + * The data pointer must remain valid until the sparse file is closed or the + * data block is removed from the sparse file. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_data(struct sparse_file *s, + void *data, unsigned int len, unsigned int block); + +/** + * sparse_file_add_fill - associate a fill chunk with a sparse file + * + * @s - sparse file cookie + * @fill_val - 32 bit fill data + * @len - length of the fill block + * @block - offset in blocks into the sparse file to place the fill chunk + * + * Associates a chunk filled with fill_val with a sparse file cookie. + * The region [block * block_size : block * block_size + len) must not already + * be used in the sparse file. If len is not a multiple of the block size the + * data will be padded with zeros. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_fill(struct sparse_file *s, + uint32_t fill_val, unsigned int len, unsigned int block); + +/** + * sparse_file_add_file - associate a chunk of a file with a sparse file + * + * @s - sparse file cookie + * @filename - filename of the file to be copied + * @file_offset - offset into the copied file + * @len - length of the copied block + * @block - offset in blocks into the sparse file to place the file chunk + * + * Associates a chunk of an existing file with a sparse file cookie. + * The region [block * block_size : block * block_size + len) must not already + * be used in the sparse file. If len is not a multiple of the block size the + * data will be padded with zeros. + * + * Allows adding large amounts of data to a sparse file without needing to keep + * it all mapped. File size is limited by available virtual address space, + * exceptionally large files may need to be added in multiple chunks. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_file(struct sparse_file *s, + const char *filename, int64_t file_offset, unsigned int len, + unsigned int block); + +/** + * sparse_file_write - write a sparse file to a file + * + * @s - sparse file cookie + * @fd - file descriptor to write to + * @gz - write a gzipped file + * @sparse - write in the Android sparse file format + * @crc - append a crc chunk + * + * Writes a sparse file to a file. If gz is true, the data will be passed + * through zlib. If sparse is true, the file will be written in the Android + * sparse file format. If sparse is false, the file will be written by seeking + * over unused chunks, producing a smaller file if the filesystem supports + * sparse files. If crc is true, the crc of the expanded data will be + * calculated and appended in a crc chunk. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, + bool crc); + +#endif diff --git a/libsparse/output_file.c b/libsparse/output_file.c new file mode 100644 index 000000000..2c4b5573e --- /dev/null +++ b/libsparse/output_file.c @@ -0,0 +1,613 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "output_file.h" +#include "sparse_format.h" +#include "sparse_crc32.h" + +#ifndef USE_MINGW +#include +#define O_BINARY 0 +#endif + +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define ftruncate64 ftruncate +#define mmap64 mmap +#define off64_t off_t +#endif + +#ifdef __BIONIC__ +extern void* __mmap2(void *, size_t, int, int, int, off_t); +static inline void *mmap64(void *addr, size_t length, int prot, int flags, + int fd, off64_t offset) +{ + return __mmap2(addr, length, prot, flags, fd, offset >> 12); +} +#endif + +#define SPARSE_HEADER_MAJOR_VER 1 +#define SPARSE_HEADER_MINOR_VER 0 +#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) +#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) + +struct output_file_ops { + int (*seek)(struct output_file *, int64_t); + int (*write)(struct output_file *, u8 *, int); + void (*close)(struct output_file *); +}; + +struct output_file { + int fd; + gzFile gz_fd; + bool close_fd; + int sparse; + int64_t cur_out_ptr; + u32 chunk_cnt; + u32 crc32; + struct output_file_ops *ops; + int use_crc; + unsigned int block_size; + int64_t len; +}; + +static int file_seek(struct output_file *out, int64_t off) +{ + off64_t ret; + + ret = lseek64(out->fd, off, SEEK_SET); + if (ret < 0) { + error_errno("lseek64"); + return -1; + } + return 0; +} + +static int file_write(struct output_file *out, u8 *data, int len) +{ + int ret; + ret = write(out->fd, data, len); + if (ret < 0) { + error_errno("write"); + return -1; + } else if (ret < len) { + error("incomplete write"); + return -1; + } + + return 0; +} + +static void file_close(struct output_file *out) +{ + if (out->close_fd) { + close(out->fd); + } +} + + +static struct output_file_ops file_ops = { + .seek = file_seek, + .write = file_write, + .close = file_close, +}; + +static int gz_file_seek(struct output_file *out, int64_t off) +{ + off64_t ret; + + ret = gzseek(out->gz_fd, off, SEEK_SET); + if (ret < 0) { + error_errno("gzseek"); + return -1; + } + return 0; +} + +static int gz_file_write(struct output_file *out, u8 *data, int len) +{ + int ret; + ret = gzwrite(out->gz_fd, data, len); + if (ret < 0) { + error_errno("gzwrite"); + return -1; + } else if (ret < len) { + error("incomplete gzwrite"); + return -1; + } + + return 0; +} + +static void gz_file_close(struct output_file *out) +{ + gzclose(out->gz_fd); +} + +static struct output_file_ops gz_file_ops = { + .seek = gz_file_seek, + .write = gz_file_write, + .close = gz_file_close, +}; + +static sparse_header_t sparse_header = { + .magic = SPARSE_HEADER_MAGIC, + .major_version = SPARSE_HEADER_MAJOR_VER, + .minor_version = SPARSE_HEADER_MINOR_VER, + .file_hdr_sz = SPARSE_HEADER_LEN, + .chunk_hdr_sz = CHUNK_HEADER_LEN, + .blk_sz = 0, + .total_blks = 0, + .total_chunks = 0, + .image_checksum = 0 +}; + +static u8 *zero_buf; + +static int emit_skip_chunk(struct output_file *out, u64 skip_len) +{ + chunk_header_t chunk_header; + int ret, chunk; + + //DBG printf("skip chunk: 0x%llx bytes\n", skip_len); + + if (skip_len % out->block_size) { + error("don't care size %llu is not a multiple of the block size %u", + skip_len, out->block_size); + return -1; + } + + /* We are skipping data, so emit a don't care chunk. */ + chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = skip_len / out->block_size; + chunk_header.total_sz = CHUNK_HEADER_LEN; + ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); + if (ret < 0) + return -1; + + out->cur_out_ptr += skip_len; + out->chunk_cnt++; + + return 0; +} + +static int write_chunk_fill(struct output_file *out, int64_t off, u32 fill_val, int len) +{ + chunk_header_t chunk_header; + int rnd_up_len, zero_len, count; + int ret; + unsigned int i; + u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ + + /* We can assume that all the chunks to be written are in + * ascending order, block-size aligned, and non-overlapping. + * So, if the offset is less than the current output pointer, + * throw an error, and if there is a gap, emit a "don't care" + * chunk. The first write (of the super block) may not be + * blocksize aligned, so we need to deal with that too. + */ + //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); + + if (off < out->cur_out_ptr) { + error("offset %llu is less than the current output offset %llu", + off, out->cur_out_ptr); + return -1; + } + + if (off > out->cur_out_ptr) { + emit_skip_chunk(out, off - out->cur_out_ptr); + } + + if (off % out->block_size) { + error("write chunk offset %llu is not a multiple of the block size %u", + off, out->block_size); + return -1; + } + + if (off != out->cur_out_ptr) { + error("internal error, offset accounting screwy in write_chunk_raw()"); + return -1; + } + + /* Round up the file length to a multiple of the block size */ + rnd_up_len = (len + (out->block_size - 1)) & (~(out->block_size -1)); + + /* Finally we can safely emit a chunk of data */ + chunk_header.chunk_type = CHUNK_TYPE_FILL; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = rnd_up_len / out->block_size; + chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); + ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); + + if (ret < 0) + return -1; + ret = out->ops->write(out, (u8 *)&fill_val, sizeof(fill_val)); + if (ret < 0) + return -1; + + if (out->use_crc) { + /* Initialize fill_buf with the fill_val */ + for (i = 0; i < (out->block_size / sizeof(u32)); i++) { + fill_buf[i] = fill_val; + } + + count = chunk_header.chunk_sz; + while (count) { + out->crc32 = sparse_crc32(out->crc32, fill_buf, out->block_size); + count--; + } + } + + out->cur_out_ptr += rnd_up_len; + out->chunk_cnt++; + + return 0; +} + +static int write_chunk_raw(struct output_file *out, int64_t off, u8 *data, int len) +{ + chunk_header_t chunk_header; + int rnd_up_len, zero_len; + int ret; + + /* We can assume that all the chunks to be written are in + * ascending order, block-size aligned, and non-overlapping. + * So, if the offset is less than the current output pointer, + * throw an error, and if there is a gap, emit a "don't care" + * chunk. The first write (of the super block) may not be + * blocksize aligned, so we need to deal with that too. + */ + //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); + + if (off < out->cur_out_ptr) { + error("offset %llu is less than the current output offset %llu", + off, out->cur_out_ptr); + return -1; + } + + if (off > out->cur_out_ptr) { + emit_skip_chunk(out, off - out->cur_out_ptr); + } + + if (off % out->block_size) { + error("write chunk offset %llu is not a multiple of the block size %u", + off, out->block_size); + return -1; + } + + if (off != out->cur_out_ptr) { + error("internal error, offset accounting screwy in write_chunk_raw()"); + return -1; + } + + /* Round up the file length to a multiple of the block size */ + rnd_up_len = (len + (out->block_size - 1)) & (~(out->block_size -1)); + zero_len = rnd_up_len - len; + + /* Finally we can safely emit a chunk of data */ + chunk_header.chunk_type = CHUNK_TYPE_RAW; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = rnd_up_len / out->block_size; + chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; + ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); + + if (ret < 0) + return -1; + ret = out->ops->write(out, data, len); + if (ret < 0) + return -1; + if (zero_len) { + ret = out->ops->write(out, zero_buf, zero_len); + if (ret < 0) + return -1; + } + + if (out->use_crc) { + out->crc32 = sparse_crc32(out->crc32, data, len); + if (zero_len) + out->crc32 = sparse_crc32(out->crc32, zero_buf, zero_len); + } + + out->cur_out_ptr += rnd_up_len; + out->chunk_cnt++; + + return 0; +} + +void close_output_file(struct output_file *out) +{ + int ret; + chunk_header_t chunk_header; + + if (out->sparse) { + if (out->use_crc) { + chunk_header.chunk_type = CHUNK_TYPE_CRC32; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = 0; + chunk_header.total_sz = CHUNK_HEADER_LEN + 4; + + out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); + out->ops->write(out, (u8 *)&out->crc32, 4); + + out->chunk_cnt++; + } + + if (out->chunk_cnt != sparse_header.total_chunks) + error("sparse chunk count did not match: %d %d", out->chunk_cnt, + sparse_header.total_chunks); + } + out->ops->close(out); +} + +struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, + int gz, int sparse, int chunks, int crc) +{ + int ret; + struct output_file *out = malloc(sizeof(struct output_file)); + if (!out) { + error_errno("malloc struct out"); + return NULL; + } + zero_buf = malloc(out->block_size); + if (!zero_buf) { + error_errno("malloc zero_buf"); + free(out); + return NULL; + } + memset(zero_buf, '\0', out->block_size); + + if (gz) { + out->ops = &gz_file_ops; + out->gz_fd = gzdopen(fd, "wb9"); + if (!out->gz_fd) { + error_errno("gzopen"); + free(out); + return NULL; + } + } else { + out->fd = fd; + out->ops = &file_ops; + } + out->close_fd = false; + out->sparse = sparse; + out->cur_out_ptr = 0ll; + out->chunk_cnt = 0; + + /* Initialize the crc32 value */ + out->crc32 = 0; + out->use_crc = crc; + + out->len = len; + out->block_size = block_size; + + if (out->sparse) { + sparse_header.blk_sz = out->block_size, + sparse_header.total_blks = out->len / out->block_size, + sparse_header.total_chunks = chunks; + if (out->use_crc) + sparse_header.total_chunks++; + + ret = out->ops->write(out, (u8 *)&sparse_header, sizeof(sparse_header)); + if (ret < 0) + return NULL; + } + + return out; +} + +struct output_file *open_output_file(const char *filename, + unsigned int block_size, int64_t len, + int gz, int sparse, int chunks, int crc) +{ + int fd; + struct output_file *file; + + if (strcmp(filename, "-")) { + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); + if (fd < 0) { + error_errno("open"); + return NULL; + } + } else { + fd = STDOUT_FILENO; + } + + file = open_output_fd(fd, block_size, len, gz, sparse, chunks, crc); + if (!file) { + close(fd); + return NULL; + } + + file->close_fd = true; // we opened descriptor thus we responsible for closing it + + return file; +} + +void pad_output_file(struct output_file *out, int64_t len) +{ + int ret; + + if (len > out->len) { + error("attempted to pad file %llu bytes past end of filesystem", + len - out->len); + return; + } + if (out->sparse) { + /* We need to emit a DONT_CARE chunk to pad out the file if the + * cur_out_ptr is not already at the end of the filesystem. + */ + if (len < out->cur_out_ptr) { + error("attempted to pad file %llu bytes less than the current output pointer", + out->cur_out_ptr - len); + return; + } + if (len > out->cur_out_ptr) { + emit_skip_chunk(out, len - out->cur_out_ptr); + } + } else { + //KEN TODO: Fixme. If the filesystem image needs no padding, + // this will overwrite the last byte in the file with 0 + // The answer is to do accounting like the sparse image + // code does and know if there is already data there. + ret = out->ops->seek(out, len - 1); + if (ret < 0) + return; + + ret = out->ops->write(out, (u8*)"", 1); + if (ret < 0) + return; + } +} + +/* Write a contiguous region of data blocks from a memory buffer */ +void write_data_block(struct output_file *out, int64_t off, void *data, int len) +{ + int ret; + + if (off + len > out->len) { + error("attempted to write block %llu past end of filesystem", + off + len - out->len); + return; + } + + if (out->sparse) { + write_chunk_raw(out, off, data, len); + } else { + ret = out->ops->seek(out, off); + if (ret < 0) + return; + + ret = out->ops->write(out, data, len); + if (ret < 0) + return; + } +} + +/* Write a contiguous region of data blocks with a fill value */ +void write_fill_block(struct output_file *out, int64_t off, unsigned int fill_val, int len) +{ + int ret; + unsigned int i; + int write_len; + u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ + + if (off + len > out->len) { + error("attempted to write block %llu past end of filesystem", + off + len - out->len); + return; + } + + if (out->sparse) { + write_chunk_fill(out, off, fill_val, len); + } else { + /* Initialize fill_buf with the fill_val */ + for (i = 0; i < sizeof(fill_buf)/sizeof(u32); i++) { + fill_buf[i] = fill_val; + } + + ret = out->ops->seek(out, off); + if (ret < 0) + return; + + while (len) { + write_len = (len > (int)sizeof(fill_buf) ? (int)sizeof(fill_buf) : len); + ret = out->ops->write(out, (u8 *)fill_buf, write_len); + if (ret < 0) { + return; + } else { + len -= write_len; + } + } + } +} + +/* Write a contiguous region of data blocks from a file */ +void write_data_file(struct output_file *out, int64_t off, const char *file, + int64_t offset, int len) +{ + int ret; + int64_t aligned_offset; + int aligned_diff; + int buffer_size; + + if (off + len >= out->len) { + error("attempted to write block %llu past end of filesystem", + off + len - out->len); + return; + } + + int file_fd = open(file, O_RDONLY | O_BINARY); + if (file_fd < 0) { + error_errno("open"); + return; + } + + aligned_offset = offset & ~(4096 - 1); + aligned_diff = offset - aligned_offset; + buffer_size = len + aligned_diff; + +#ifndef USE_MINGW + u8 *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, file_fd, + aligned_offset); + if (data == MAP_FAILED) { + error_errno("mmap64"); + close(file_fd); + return; + } +#else + u8 *data = malloc(buffer_size); + if (!data) { + error_errno("malloc"); + close(file_fd); + return; + } + memset(data, 0, buffer_size); +#endif + + if (out->sparse) { + write_chunk_raw(out, off, data + aligned_diff, len); + } else { + ret = out->ops->seek(out, off); + if (ret < 0) + goto err; + + ret = out->ops->write(out, data + aligned_diff, len); + if (ret < 0) + goto err; + } + +err: +#ifndef USE_MINGW + munmap(data, buffer_size); +#else + write(file_fd, data, buffer_size); + free(data); +#endif + close(file_fd); +} diff --git a/libsparse/output_file.h b/libsparse/output_file.h new file mode 100644 index 000000000..b12194fc7 --- /dev/null +++ b/libsparse/output_file.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _OUTPUT_FILE_H_ +#define _OUTPUT_FILE_H_ + +#include + +struct output_file; + +struct output_file *open_output_file(const char *filename, + unsigned int block_size, int64_t len, + int gz, int sparse, int chunks, int crc); +struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, + int gz, int sparse, int chunks, int crc); +void write_data_block(struct output_file *out, int64_t off, void *data, int len); +void write_fill_block(struct output_file *out, int64_t off, unsigned int fill_val, int len); +void write_data_file(struct output_file *out, int64_t off, const char *file, + int64_t offset, int len); +void pad_output_file(struct output_file *out, int64_t len); +void close_output_file(struct output_file *out); + +#endif diff --git a/libsparse/simg2img.c b/libsparse/simg2img.c new file mode 100644 index 000000000..486b8054f --- /dev/null +++ b/libsparse/simg2img.c @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 +#include +#include + +#include "sparse_defs.h" +#include "sparse_format.h" +#include "sparse_crc32.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define COPY_BUF_SIZE (1024*1024) +u8 *copybuf; + +/* This will be malloc'ed with the size of blk_sz from the sparse file header */ +u8* zerobuf; + +#define SPARSE_HEADER_MAJOR_VER 1 +#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) +#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) + +void usage() +{ + fprintf(stderr, "Usage: simg2img \n"); +} + +static int read_all(int fd, void *buf, size_t len) +{ + size_t total = 0; + int ret; + char *ptr = buf; + + while (total < len) { + ret = read(fd, ptr, len - total); + + if (ret < 0) + return ret; + + if (ret == 0) + return total; + + ptr += ret; + total += ret; + } + + return total; +} + +static int write_all(int fd, void *buf, size_t len) +{ + size_t total = 0; + int ret; + char *ptr = buf; + + while (total < len) { + ret = write(fd, ptr, len - total); + + if (ret < 0) + return ret; + + if (ret == 0) + return total; + + ptr += ret; + total += ret; + } + + return total; +} + +int process_raw_chunk(int in, int out, u32 blocks, u32 blk_sz, u32 *crc32) +{ + u64 len = (u64)blocks * blk_sz; + int ret; + int chunk; + + while (len) { + chunk = (len > COPY_BUF_SIZE) ? COPY_BUF_SIZE : len; + ret = read_all(in, copybuf, chunk); + if (ret != chunk) { + fprintf(stderr, "read returned an error copying a raw chunk: %d %d\n", + ret, chunk); + exit(-1); + } + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + ret = write_all(out, copybuf, chunk); + if (ret != chunk) { + fprintf(stderr, "write returned an error copying a raw chunk\n"); + exit(-1); + } + len -= chunk; + } + + return blocks; +} + + +int process_fill_chunk(int in, int out, u32 blocks, u32 blk_sz, u32 *crc32) +{ + u64 len = (u64)blocks * blk_sz; + int ret; + int chunk; + u32 fill_val; + u32 *fillbuf; + unsigned int i; + + /* Fill copy_buf with the fill value */ + ret = read_all(in, &fill_val, sizeof(fill_val)); + fillbuf = (u32 *)copybuf; + for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) { + fillbuf[i] = fill_val; + } + + while (len) { + chunk = (len > COPY_BUF_SIZE) ? COPY_BUF_SIZE : len; + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + ret = write_all(out, copybuf, chunk); + if (ret != chunk) { + fprintf(stderr, "write returned an error copying a raw chunk\n"); + exit(-1); + } + len -= chunk; + } + + return blocks; +} + +int process_skip_chunk(int out, u32 blocks, u32 blk_sz, u32 *crc32) +{ + /* len needs to be 64 bits, as the sparse file specifies the skip amount + * as a 32 bit value of blocks. + */ + u64 len = (u64)blocks * blk_sz; + + lseek64(out, len, SEEK_CUR); + + return blocks; +} + +int process_crc32_chunk(int in, u32 crc32) +{ + u32 file_crc32; + int ret; + + ret = read_all(in, &file_crc32, 4); + if (ret != 4) { + fprintf(stderr, "read returned an error copying a crc32 chunk\n"); + exit(-1); + } + + if (file_crc32 != crc32) { + fprintf(stderr, "computed crc32 of 0x%8.8x, expected 0x%8.8x\n", + crc32, file_crc32); + exit(-1); + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + int in; + int out; + unsigned int i; + sparse_header_t sparse_header; + chunk_header_t chunk_header; + u32 crc32 = 0; + u32 total_blocks = 0; + int ret; + + if (argc != 3) { + usage(); + exit(-1); + } + + if ( (copybuf = malloc(COPY_BUF_SIZE)) == 0) { + fprintf(stderr, "Cannot malloc copy buf\n"); + exit(-1); + } + + if (strcmp(argv[1], "-") == 0) { + in = STDIN_FILENO; + } else { + if ((in = open(argv[1], O_RDONLY)) == 0) { + fprintf(stderr, "Cannot open input file %s\n", argv[1]); + exit(-1); + } + } + + if (strcmp(argv[2], "-") == 0) { + out = STDOUT_FILENO; + } else { + if ((out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, 0666)) == 0) { + fprintf(stderr, "Cannot open output file %s\n", argv[2]); + exit(-1); + } + } + + ret = read_all(in, &sparse_header, sizeof(sparse_header)); + if (ret != sizeof(sparse_header)) { + fprintf(stderr, "Error reading sparse file header\n"); + exit(-1); + } + + if (sparse_header.magic != SPARSE_HEADER_MAGIC) { + fprintf(stderr, "Bad magic\n"); + exit(-1); + } + + if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) { + fprintf(stderr, "Unknown major version number\n"); + exit(-1); + } + + if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) { + /* Skip the remaining bytes in a header that is longer than + * we expected. + */ + lseek64(in, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR); + } + + if ( (zerobuf = malloc(sparse_header.blk_sz)) == 0) { + fprintf(stderr, "Cannot malloc zero buf\n"); + exit(-1); + } + + for (i=0; i CHUNK_HEADER_LEN) { + /* Skip the remaining bytes in a header that is longer than + * we expected. + */ + lseek64(in, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR); + } + + switch (chunk_header.chunk_type) { + case CHUNK_TYPE_RAW: + if (chunk_header.total_sz != (sparse_header.chunk_hdr_sz + + (chunk_header.chunk_sz * sparse_header.blk_sz)) ) { + fprintf(stderr, "Bogus chunk size for chunk %d, type Raw\n", i); + exit(-1); + } + total_blocks += process_raw_chunk(in, out, + chunk_header.chunk_sz, sparse_header.blk_sz, &crc32); + break; + case CHUNK_TYPE_FILL: + if (chunk_header.total_sz != (sparse_header.chunk_hdr_sz + sizeof(u32)) ) { + fprintf(stderr, "Bogus chunk size for chunk %d, type Fill\n", i); + exit(-1); + } + total_blocks += process_fill_chunk(in, out, + chunk_header.chunk_sz, sparse_header.blk_sz, &crc32); + break; + case CHUNK_TYPE_DONT_CARE: + if (chunk_header.total_sz != sparse_header.chunk_hdr_sz) { + fprintf(stderr, "Bogus chunk size for chunk %d, type Dont Care\n", i); + exit(-1); + } + total_blocks += process_skip_chunk(out, + chunk_header.chunk_sz, sparse_header.blk_sz, &crc32); + break; + case CHUNK_TYPE_CRC32: + process_crc32_chunk(in, crc32); + break; + default: + fprintf(stderr, "Unknown chunk type 0x%4.4x\n", chunk_header.chunk_type); + } + + } + + /* If the last chunk was a skip, then the code just did a seek, but + * no write, and the file won't actually be the correct size. This + * will make the file the correct size. Make sure the offset is + * computed in 64 bits, and the function called can handle 64 bits. + */ + if (ftruncate64(out, (u64)total_blocks * sparse_header.blk_sz)) { + fprintf(stderr, "Error calling ftruncate() to set the image size\n"); + exit(-1); + } + + close(in); + close(out); + + if (sparse_header.total_blks != total_blocks) { + fprintf(stderr, "Wrote %d blocks, expected to write %d blocks\n", + total_blocks, sparse_header.total_blks); + exit(-1); + } + + exit(0); +} + diff --git a/libsparse/simg_dump.py b/libsparse/simg_dump.py new file mode 100755 index 000000000..6ece31d0c --- /dev/null +++ b/libsparse/simg_dump.py @@ -0,0 +1,169 @@ +#! /usr/bin/env python + +# Copyright (C) 2012 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import getopt, posixpath, signal, struct, sys + +def usage(argv0): + print(""" +Usage: %s [-v] sparse_image_file ... + -v verbose output +""" % ( argv0 )) + sys.exit(2) + +def main(): + + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + me = posixpath.basename(sys.argv[0]) + + # Parse the command line + verbose = 0 # -v + try: + opts, args = getopt.getopt(sys.argv[1:], + "v", + ["verbose"]) + except getopt.GetoptError, e: + print(e) + usage(me) + for o, a in opts: + if o in ("-v", "--verbose"): + verbose += 1 + else: + print("Unrecognized option \"%s\"" % (o)) + usage(me) + + if len(args) == 0: + print("No sparse_image_file specified") + usage(me) + + for path in args: + FH = open(path, 'rb') + header_bin = FH.read(28) + header = struct.unpack(" 1: + header = struct.unpack("<12B", header_bin) + print(" (%02X%02X %02X%02X %02X%02X%02X%02X %02X%02X%02X%02X)" + % (header[0], header[1], header[2], header[3], + header[4], header[5], header[6], header[7], + header[8], header[9], header[10], header[11])) + else: + print() + + offset += chunk_sz + + print(" %10u %7u End" % (FH.tell(), offset)) + + if total_blks != offset: + print("The header said we should have %u output blocks, but we saw %u" + % (total_blks, offset)) + + junk_len = len(FH.read()) + if junk_len: + print("There were %u bytes of extra data at the end of the file." + % (junk_len)) + + sys.exit(0) + +if __name__ == "__main__": + main() diff --git a/libsparse/sparse.c b/libsparse/sparse.c new file mode 100644 index 000000000..d6f556114 --- /dev/null +++ b/libsparse/sparse.c @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "sparse_file.h" + +#include "output_file.h" +#include "backed_block.h" +#include "sparse_defs.h" + + +struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len) +{ + struct sparse_file *s = calloc(sizeof(struct sparse_file), 1); + if (!s) { + return NULL; + } + + /* TODO: allocate backed block list */ + + s->block_size = block_size; + s->len = len; + + return s; +} + +void sparse_file_destroy(struct sparse_file *s) +{ + free_data_blocks(); + free(s); +} + +int sparse_file_add_data(struct sparse_file *s, + void *data, unsigned int len, unsigned int block) +{ + queue_data_block(data, len, block); + + return 0; +} + +int sparse_file_add_fill(struct sparse_file *s, + uint32_t fill_val, unsigned int len, unsigned int block) +{ + queue_fill_block(fill_val, len, block); + + return 0; +} + +int sparse_file_add_file(struct sparse_file *s, + const char *filename, int64_t file_offset, unsigned int len, + unsigned int block) +{ + queue_data_file(filename, file_offset, len, block); + + return 0; +} + +struct count_chunks { + unsigned int chunks; + int64_t cur_ptr; + unsigned int block_size; +}; + +static void count_data_block(void *priv, int64_t off, void *data, int len) +{ + struct count_chunks *count_chunks = priv; + if (off > count_chunks->cur_ptr) + count_chunks->chunks++; + count_chunks->cur_ptr = off + ALIGN(len, count_chunks->block_size); + count_chunks->chunks++; +} + +static void count_fill_block(void *priv, int64_t off, unsigned int fill_val, int len) +{ + struct count_chunks *count_chunks = priv; + if (off > count_chunks->cur_ptr) + count_chunks->chunks++; + count_chunks->cur_ptr = off + ALIGN(len, count_chunks->block_size); + count_chunks->chunks++; +} + +static void count_file_block(void *priv, int64_t off, const char *file, + int64_t offset, int len) +{ + struct count_chunks *count_chunks = priv; + if (off > count_chunks->cur_ptr) + count_chunks->chunks++; + count_chunks->cur_ptr = off + ALIGN(len, count_chunks->block_size); + count_chunks->chunks++; +} + +static int count_sparse_chunks(unsigned int block_size, int64_t len) +{ + struct count_chunks count_chunks = {0, 0, block_size}; + + for_each_data_block(count_data_block, count_file_block, count_fill_block, &count_chunks, block_size); + + if (count_chunks.cur_ptr != len) + count_chunks.chunks++; + + return count_chunks.chunks; +} + +static void ext4_write_data_block(void *priv, int64_t off, void *data, int len) +{ + write_data_block(priv, off, data, len); +} + +static void ext4_write_fill_block(void *priv, int64_t off, unsigned int fill_val, int len) +{ + write_fill_block(priv, off, fill_val, len); +} + +static void ext4_write_data_file(void *priv, int64_t off, const char *file, + int64_t offset, int len) +{ + write_data_file(priv, off, file, offset, len); +} + +int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, + bool crc) +{ + int chunks = count_sparse_chunks(s->block_size, s->len); + struct output_file *out = open_output_fd(fd, s->block_size, s->len, + gz, sparse, chunks, crc); + + if (!out) + return -ENOMEM; + + for_each_data_block(ext4_write_data_block, ext4_write_data_file, ext4_write_fill_block, out, s->block_size); + + if (s->len) + pad_output_file(out, s->len); + + close_output_file(out); + + return 0; +} diff --git a/libsparse/sparse_crc32.c b/libsparse/sparse_crc32.c new file mode 100644 index 000000000..38bfe4aa0 --- /dev/null +++ b/libsparse/sparse_crc32.c @@ -0,0 +1,111 @@ +/*- + * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or + * code or tables extracted from it, as desired without restriction. + */ + +/* + * First, the polynomial itself and its table of feedback terms. The + * polynomial is + * X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 + * + * Note that we take it "backwards" and put the highest-order term in + * the lowest-order bit. The X^32 term is "implied"; the LSB is the + * X^31 term, etc. The X^0 term (usually shown as "+1") results in + * the MSB being 1 + * + * Note that the usual hardware shift register implementation, which + * is what we're using (we're merely optimizing it by doing eight-bit + * chunks at a time) shifts bits into the lowest-order term. In our + * implementation, that means shifting towards the right. Why do we + * do it this way? Because the calculated CRC must be transmitted in + * order from highest-order term to lowest-order term. UARTs transmit + * characters in order from LSB to MSB. By storing the CRC this way + * we hand it to the UART in the order low-byte to high-byte; the UART + * sends each low-bit to hight-bit; and the result is transmission bit + * by bit from highest- to lowest-order term without requiring any bit + * shuffling on our part. Reception works similarly + * + * The feedback terms table consists of 256, 32-bit entries. Notes + * + * The table can be generated at runtime if desired; code to do so + * is shown later. It might not be obvious, but the feedback + * terms simply represent the results of eight shift/xor opera + * tions for all combinations of data and CRC register values + * + * The values must be right-shifted by eight bits by the "updcrc + * logic; the shift must be unsigned (bring in zeroes). On some + * hardware you could probably optimize the shift in assembler by + * using byte-swap instructions + * polynomial $edb88320 + * + * + * CRC32 code derived from work by Gary S. Brown. + */ + +/* Code taken from FreeBSD 8 */ +#include + +static uint32_t crc32_tab[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +/* + * A function that calculates the CRC-32 based on the table above is + * given below for documentation purposes. An equivalent implementation + * of this function that's actually used in the kernel can be found + * in sys/libkern.h, where it can be inlined. + */ + +uint32_t sparse_crc32(uint32_t crc_in, const void *buf, int size) +{ + const uint8_t *p = buf; + uint32_t crc; + + crc = crc_in ^ ~0U; + while (size--) + crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); + return crc ^ ~0U; +} + diff --git a/libsparse/sparse_crc32.h b/libsparse/sparse_crc32.h new file mode 100644 index 000000000..21625bac9 --- /dev/null +++ b/libsparse/sparse_crc32.h @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +u32 sparse_crc32(u32 crc, const void *buf, size_t size); + diff --git a/libsparse/sparse_defs.h b/libsparse/sparse_defs.h new file mode 100644 index 000000000..9f32d592e --- /dev/null +++ b/libsparse/sparse_defs.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_DEFS_ +#define _LIBSPARSE_SPARSE_DEFS_ + +#include +#include + +#define __le64 u64 +#define __le32 u32 +#define __le16 u16 + +#define __be64 u64 +#define __be32 u32 +#define __be16 u16 + +#define __u64 u64 +#define __u32 u32 +#define __u16 u16 +#define __u8 u8 + +typedef unsigned long long u64; +typedef signed long long s64; +typedef unsigned int u32; +typedef unsigned short int u16; +typedef unsigned char u8; + +#define DIV_ROUND_UP(x, y) (((x) + (y) - 1)/(y)) +#define ALIGN(x, y) ((y) * DIV_ROUND_UP((x), (y))) + +#define error(fmt, args...) do { fprintf(stderr, "error: %s: " fmt "\n", __func__, ## args); } while (0) +#define error_errno(s, args...) error(s ": %s", ##args, strerror(errno)) + +#endif diff --git a/libsparse/sparse_file.h b/libsparse/sparse_file.h new file mode 100644 index 000000000..05a78d96e --- /dev/null +++ b/libsparse/sparse_file.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_FILE_H_ +#define _LIBSPARSE_SPARSE_FILE_H_ + +#include + +struct sparse_file { + unsigned int block_size; + int64_t len; + + struct output_file *out; +}; + + +#endif /* _LIBSPARSE_SPARSE_FILE_H_ */ diff --git a/libsparse/sparse_format.h b/libsparse/sparse_format.h new file mode 100644 index 000000000..c41f12a54 --- /dev/null +++ b/libsparse/sparse_format.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _LIBSPARSE_SPARSE_FORMAT_H_ +#define _LIBSPARSE_SPARSE_FORMAT_H_ +#include "sparse_defs.h" + +typedef struct sparse_header { + __le32 magic; /* 0xed26ff3a */ + __le16 major_version; /* (0x1) - reject images with higher major versions */ + __le16 minor_version; /* (0x0) - allow images with higer minor versions */ + __le16 file_hdr_sz; /* 28 bytes for first revision of the file format */ + __le16 chunk_hdr_sz; /* 12 bytes for first revision of the file format */ + __le32 blk_sz; /* block size in bytes, must be a multiple of 4 (4096) */ + __le32 total_blks; /* total blocks in the non-sparse output image */ + __le32 total_chunks; /* total chunks in the sparse input image */ + __le32 image_checksum; /* CRC32 checksum of the original data, counting "don't care" */ + /* as 0. Standard 802.3 polynomial, use a Public Domain */ + /* table implementation */ +} sparse_header_t; + +#define SPARSE_HEADER_MAGIC 0xed26ff3a + +#define CHUNK_TYPE_RAW 0xCAC1 +#define CHUNK_TYPE_FILL 0xCAC2 +#define CHUNK_TYPE_DONT_CARE 0xCAC3 +#define CHUNK_TYPE_CRC32 0xCAC4 + +typedef struct chunk_header { + __le16 chunk_type; /* 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care */ + __le16 reserved1; + __le32 chunk_sz; /* in blocks in output image */ + __le32 total_sz; /* in bytes of chunk input file including chunk header and data */ +} chunk_header_t; + +/* Following a Raw or Fill or CRC32 chunk is data. + * For a Raw chunk, it's the data in chunk_sz * blk_sz. + * For a Fill chunk, it's 4 bytes of the fill data. + * For a CRC32 chunk, it's 4 bytes of CRC32 + */ + +#endif From 411619e921904b896eddae81c086c1f687c8304d Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 24 Apr 2012 18:51:42 -0700 Subject: [PATCH 02/16] libsparse: remove static variables Removes static variables in backed_block.c to allow multiple sparse files to be open at the same time. Change-Id: I012d8a424c6e21a7352408416adb7c72ee8add21 --- libsparse/backed_block.c | 99 +++++++++++++++++++++------------------- libsparse/backed_block.h | 28 +++++++----- libsparse/sparse.c | 26 +++++++---- libsparse/sparse_file.h | 1 + 4 files changed, 87 insertions(+), 67 deletions(-) diff --git a/libsparse/backed_block.c b/libsparse/backed_block.c index 254813809..6e8ef4424 100644 --- a/libsparse/backed_block.c +++ b/libsparse/backed_block.c @@ -33,32 +33,57 @@ struct data_block { u16 pad2; }; -static struct data_block *data_blocks = NULL; -static struct data_block *last_used = NULL; +struct backed_block_list { + struct data_block *data_blocks; + struct data_block *last_used; +}; -static void queue_db(struct data_block *new_db) +struct backed_block_list *backed_block_list_new(void) +{ + struct backed_block_list *b = calloc(sizeof(struct backed_block_list), 1); + + return b; +} + +void backed_block_list_destroy(struct backed_block_list *b) +{ + if (b->data_blocks) { + struct data_block *db = b->data_blocks; + while (db) { + struct data_block *next = db->next; + free((void*)db->filename); + + free(db); + db = next; + } + } + + free(b); +} + +static void queue_db(struct backed_block_list *b, struct data_block *new_db) { struct data_block *db; - if (data_blocks == NULL) { - data_blocks = new_db; + if (b->data_blocks == NULL) { + b->data_blocks = new_db; return; } - if (data_blocks->block > new_db->block) { - new_db->next = data_blocks; - data_blocks = new_db; + if (b->data_blocks->block > new_db->block) { + new_db->next = b->data_blocks; + b->data_blocks = new_db; return; } /* Optimization: blocks are mostly queued in sequence, so save the pointer to the last db that was added, and start searching from there if the next block number is higher */ - if (last_used && new_db->block > last_used->block) - db = last_used; + if (b->last_used && new_db->block > b->last_used->block) + db = b->last_used; else - db = data_blocks; - last_used = new_db; + db = b->data_blocks; + b->last_used = new_db; for (; db->next && db->next->block < new_db->block; db = db->next) ; @@ -72,9 +97,10 @@ static void queue_db(struct data_block *new_db) } /* Queues a fill block of memory to be written to the specified data blocks */ -void queue_fill_block(unsigned int fill_val, unsigned int len, unsigned int block) +void queue_fill_block(struct backed_block_list *b, unsigned int fill_val, + unsigned int len, unsigned int block) { - struct data_block *db = malloc(sizeof(struct data_block)); + struct data_block *db = calloc(1, sizeof(struct data_block)); if (db == NULL) { error_errno("malloc"); return; @@ -88,11 +114,12 @@ void queue_fill_block(unsigned int fill_val, unsigned int len, unsigned int bloc db->filename = NULL; db->next = NULL; - queue_db(db); + queue_db(b, db); } /* Queues a block of memory to be written to the specified data blocks */ -void queue_data_block(void *data, unsigned int len, unsigned int block) +void queue_data_block(struct backed_block_list *b, void *data, unsigned int len, + unsigned int block) { struct data_block *db = malloc(sizeof(struct data_block)); if (db == NULL) { @@ -107,12 +134,12 @@ void queue_data_block(void *data, unsigned int len, unsigned int block) db->fill = 0; db->next = NULL; - queue_db(db); + queue_db(b, db); } /* Queues a chunk of a file on disk to be written to the specified data blocks */ -void queue_data_file(const char *filename, int64_t offset, unsigned int len, - unsigned int block) +void queue_data_file(struct backed_block_list *b, const char *filename, + int64_t offset, unsigned int len, unsigned int block) { struct data_block *db = malloc(sizeof(struct data_block)); if (db == NULL) { @@ -128,19 +155,21 @@ void queue_data_file(const char *filename, int64_t offset, unsigned int len, db->fill = 0; db->next = NULL; - queue_db(db); + queue_db(b, db); } /* Iterates over the queued data blocks, calling data_func for each contiguous data block, and file_func for each contiguous file block */ -void for_each_data_block(data_block_callback_t data_func, +void for_each_data_block(struct backed_block_list *b, + data_block_callback_t data_func, data_block_file_callback_t file_func, - data_block_fill_callback_t fill_func, void *priv, unsigned int block_size) + data_block_fill_callback_t fill_func, + void *priv, unsigned int block_size) { struct data_block *db; u32 last_block = 0; - for (db = data_blocks; db; db = db->next) { + for (db = b->data_blocks; db; db = db->next) { if (db->block < last_block) error("data blocks out of order: %u < %u", db->block, last_block); last_block = db->block + DIV_ROUND_UP(db->len, block_size) - 1; @@ -153,27 +182,3 @@ void for_each_data_block(data_block_callback_t data_func, data_func(priv, (u64)db->block * block_size, db->data, db->len); } } - -/* Frees the memory used by the linked list of data blocks */ -void free_data_blocks() -{ - if (!data_blocks) return; - struct data_block *db = data_blocks; - while (db) { - struct data_block *next = db->next; - free((void*)db->filename); - - // There used to be a free() of db->data here, but it - // made the function crash since queue_data_block() is - // sometimes passed pointers it can't take ownership of - // (like a pointer into the middle of an allocated - // block). It's not clear what the queue_data_block - // contract is supposed to be, but we'd rather leak - // memory than crash. - - free(db); - db = next; - } - data_blocks = NULL; - last_used = NULL; -} diff --git a/libsparse/backed_block.h b/libsparse/backed_block.h index 7b7c90aad..d1bfa1ec3 100644 --- a/libsparse/backed_block.h +++ b/libsparse/backed_block.h @@ -17,23 +17,29 @@ #ifndef _BACKED_BLOCK_H_ #define _BACKED_BLOCK_H_ -#include +struct backed_block_list; -typedef void (*data_block_callback_t)(void *priv, int64_t off, void *data, int len); -typedef void (*data_block_fill_callback_t)(void *priv, int64_t off, unsigned int fill_val, int len); +typedef void (*data_block_callback_t)(void *priv, int64_t off, void *data, + int len); +typedef void (*data_block_fill_callback_t)(void *priv, int64_t off, + unsigned int fill_val, int len); typedef void (*data_block_file_callback_t)(void *priv, int64_t off, - const char *file, int64_t offset, - int len); + const char *file, int64_t offset, int len); -void for_each_data_block(data_block_callback_t data_func, +void for_each_data_block(struct backed_block_list *b, + data_block_callback_t data_func, data_block_file_callback_t file_func, - data_block_fill_callback_t fill_func, void *priv, unsigned int); + data_block_fill_callback_t fill_func, + void *priv, unsigned int); -void queue_data_block(void *data, unsigned int len, unsigned int block); -void queue_fill_block(unsigned int fill_val, unsigned int len, unsigned int block); -void queue_data_file(const char *filename, int64_t offset, unsigned int len, +void queue_data_block(struct backed_block_list *b,void *data, unsigned int len, unsigned int block); +void queue_fill_block(struct backed_block_list *b,unsigned int fill_val, + unsigned int len, unsigned int block); +void queue_data_file(struct backed_block_list *b,const char *filename, + int64_t offset, unsigned int len, unsigned int block); -void free_data_blocks(); +struct backed_block_list *backed_block_list_new(void); +void backed_block_list_destroy(struct backed_block_list *b); #endif diff --git a/libsparse/sparse.c b/libsparse/sparse.c index d6f556114..a6134c9d2 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -32,7 +32,11 @@ struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len) return NULL; } - /* TODO: allocate backed block list */ + s->backed_block_list = backed_block_list_new(); + if (!s->backed_block_list) { + free(s); + return NULL; + } s->block_size = block_size; s->len = len; @@ -42,14 +46,14 @@ struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len) void sparse_file_destroy(struct sparse_file *s) { - free_data_blocks(); + backed_block_list_destroy(s->backed_block_list); free(s); } int sparse_file_add_data(struct sparse_file *s, void *data, unsigned int len, unsigned int block) { - queue_data_block(data, len, block); + queue_data_block(s->backed_block_list, data, len, block); return 0; } @@ -57,7 +61,7 @@ int sparse_file_add_data(struct sparse_file *s, int sparse_file_add_fill(struct sparse_file *s, uint32_t fill_val, unsigned int len, unsigned int block) { - queue_fill_block(fill_val, len, block); + queue_fill_block(s->backed_block_list, fill_val, len, block); return 0; } @@ -66,7 +70,7 @@ int sparse_file_add_file(struct sparse_file *s, const char *filename, int64_t file_offset, unsigned int len, unsigned int block) { - queue_data_file(filename, file_offset, len, block); + queue_data_file(s->backed_block_list, filename, file_offset, len, block); return 0; } @@ -105,11 +109,13 @@ static void count_file_block(void *priv, int64_t off, const char *file, count_chunks->chunks++; } -static int count_sparse_chunks(unsigned int block_size, int64_t len) +static int count_sparse_chunks(struct backed_block_list *b, + unsigned int block_size, int64_t len) { struct count_chunks count_chunks = {0, 0, block_size}; - for_each_data_block(count_data_block, count_file_block, count_fill_block, &count_chunks, block_size); + for_each_data_block(b, count_data_block, count_file_block, + count_fill_block, &count_chunks, block_size); if (count_chunks.cur_ptr != len) count_chunks.chunks++; @@ -136,14 +142,16 @@ static void ext4_write_data_file(void *priv, int64_t off, const char *file, int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, bool crc) { - int chunks = count_sparse_chunks(s->block_size, s->len); + int chunks = count_sparse_chunks(s->backed_block_list, s->block_size, + s->len); struct output_file *out = open_output_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc); if (!out) return -ENOMEM; - for_each_data_block(ext4_write_data_block, ext4_write_data_file, ext4_write_fill_block, out, s->block_size); + for_each_data_block(s->backed_block_list, ext4_write_data_block, + ext4_write_data_file, ext4_write_fill_block, out, s->block_size); if (s->len) pad_output_file(out, s->len); diff --git a/libsparse/sparse_file.h b/libsparse/sparse_file.h index 05a78d96e..fae1c168c 100644 --- a/libsparse/sparse_file.h +++ b/libsparse/sparse_file.h @@ -23,6 +23,7 @@ struct sparse_file { unsigned int block_size; int64_t len; + struct backed_block_list *backed_block_list; struct output_file *out; }; From b55dceea986ab24f8b836b5116b389ed619c816e Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 24 Apr 2012 23:07:49 -0700 Subject: [PATCH 03/16] libsparse: cleanups Move block loops into sparse.c with iterator helpers in backed_block.c. Simplify chunk writing by moving skip chunk calls from output_file.c to sparse.c. Rename variables to be consistent with new naming. Remove use of u8, u32, u64. Change-Id: Ic138ad58bef9f96239266ccee12ee83ea285e7eb --- libsparse/backed_block.c | 241 ++++++++++--------- libsparse/backed_block.h | 43 ++-- libsparse/output_file.c | 488 ++++++++++++++++++--------------------- libsparse/output_file.h | 11 +- libsparse/sparse.c | 135 +++++------ libsparse/sparse_crc32.h | 4 +- 6 files changed, 448 insertions(+), 474 deletions(-) diff --git a/libsparse/backed_block.c b/libsparse/backed_block.c index 6e8ef4424..b25919008 100644 --- a/libsparse/backed_block.c +++ b/libsparse/backed_block.c @@ -14,30 +14,87 @@ * limitations under the License. */ +#include +#include +#include #include #include #include "backed_block.h" -#include "sparse_defs.h" -struct data_block { - u32 block; - u32 len; - void *data; - const char *filename; - int64_t offset; - struct data_block *next; - u32 fill_val; - u8 fill; - u8 pad1; - u16 pad2; +struct backed_block { + unsigned int block; + unsigned int len; + enum backed_block_type type; + union { + struct { + void *data; + } data; + struct { + char *filename; + int64_t offset; + } file; + struct { + uint32_t val; + } fill; + }; + struct backed_block *next; }; struct backed_block_list { - struct data_block *data_blocks; - struct data_block *last_used; + struct backed_block *data_blocks; + struct backed_block *last_used; }; +struct backed_block *backed_block_iter_new(struct backed_block_list *bbl) +{ + return bbl->data_blocks; +} + +struct backed_block *backed_block_iter_next(struct backed_block *bb) +{ + return bb->next; +} + +unsigned int backed_block_len(struct backed_block *bb) +{ + return bb->len; +} + +unsigned int backed_block_block(struct backed_block *bb) +{ + return bb->block; +} + +void *backed_block_data(struct backed_block *bb) +{ + assert(bb->type == BACKED_BLOCK_DATA); + return bb->data.data; +} + +const char *backed_block_filename(struct backed_block *bb) +{ + assert(bb->type == BACKED_BLOCK_FILE); + return bb->file.filename; +} + +int64_t backed_block_file_offset(struct backed_block *bb) +{ + assert(bb->type == BACKED_BLOCK_FILE); + return bb->file.offset; +} + +uint32_t backed_block_fill_val(struct backed_block *bb) +{ + assert(bb->type == BACKED_BLOCK_FILL); + return bb->fill.val; +} + +enum backed_block_type backed_block_type(struct backed_block *bb) +{ + return bb->type; +} + struct backed_block_list *backed_block_list_new(void) { struct backed_block_list *b = calloc(sizeof(struct backed_block_list), 1); @@ -45,140 +102,112 @@ struct backed_block_list *backed_block_list_new(void) return b; } -void backed_block_list_destroy(struct backed_block_list *b) +void backed_block_list_destroy(struct backed_block_list *bbl) { - if (b->data_blocks) { - struct data_block *db = b->data_blocks; - while (db) { - struct data_block *next = db->next; - free((void*)db->filename); + if (bbl->data_blocks) { + struct backed_block *bb = bbl->data_blocks; + while (bb) { + struct backed_block *next = bb->next; + if (bb->type == BACKED_BLOCK_FILE) { + free(bb->file.filename); + } - free(db); - db = next; + free(bb); + bb = next; } } - free(b); + free(bbl); } -static void queue_db(struct backed_block_list *b, struct data_block *new_db) +static int queue_bb(struct backed_block_list *bbl, struct backed_block *new_bb) { - struct data_block *db; + struct backed_block *bb; - if (b->data_blocks == NULL) { - b->data_blocks = new_db; - return; + if (bbl->data_blocks == NULL) { + bbl->data_blocks = new_bb; + return 0; } - if (b->data_blocks->block > new_db->block) { - new_db->next = b->data_blocks; - b->data_blocks = new_db; - return; + if (bbl->data_blocks->block > new_bb->block) { + new_bb->next = bbl->data_blocks; + bbl->data_blocks = new_bb; + return 0; } /* Optimization: blocks are mostly queued in sequence, so save the - pointer to the last db that was added, and start searching from + pointer to the last bb that was added, and start searching from there if the next block number is higher */ - if (b->last_used && new_db->block > b->last_used->block) - db = b->last_used; + if (bbl->last_used && new_bb->block > bbl->last_used->block) + bb = bbl->last_used; else - db = b->data_blocks; - b->last_used = new_db; + bb = bbl->data_blocks; + bbl->last_used = new_bb; - for (; db->next && db->next->block < new_db->block; db = db->next) + for (; bb->next && bb->next->block < new_bb->block; bb = bb->next) ; - if (db->next == NULL) { - db->next = new_db; + if (bb->next == NULL) { + bb->next = new_bb; } else { - new_db->next = db->next; - db->next = new_db; + new_bb->next = bb->next; + bb->next = new_bb; } + + return 0; } /* Queues a fill block of memory to be written to the specified data blocks */ -void queue_fill_block(struct backed_block_list *b, unsigned int fill_val, +int backed_block_add_fill(struct backed_block_list *bbl, unsigned int fill_val, unsigned int len, unsigned int block) { - struct data_block *db = calloc(1, sizeof(struct data_block)); - if (db == NULL) { - error_errno("malloc"); - return; + struct backed_block *bb = calloc(1, sizeof(struct backed_block)); + if (bb == NULL) { + return -ENOMEM; } - db->block = block; - db->len = len; - db->fill = 1; - db->fill_val = fill_val; - db->data = NULL; - db->filename = NULL; - db->next = NULL; + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_FILL; + bb->fill.val = fill_val; + bb->next = NULL; - queue_db(b, db); + return queue_bb(bbl, bb); } /* Queues a block of memory to be written to the specified data blocks */ -void queue_data_block(struct backed_block_list *b, void *data, unsigned int len, - unsigned int block) +int backed_block_add_data(struct backed_block_list *bbl, void *data, + unsigned int len, unsigned int block) { - struct data_block *db = malloc(sizeof(struct data_block)); - if (db == NULL) { - error_errno("malloc"); - return; + struct backed_block *bb = calloc(1, sizeof(struct backed_block)); + if (bb == NULL) { + return -ENOMEM; } - db->block = block; - db->len = len; - db->data = data; - db->filename = NULL; - db->fill = 0; - db->next = NULL; + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_DATA; + bb->data.data = data; + bb->next = NULL; - queue_db(b, db); + return queue_bb(bbl, bb); } /* Queues a chunk of a file on disk to be written to the specified data blocks */ -void queue_data_file(struct backed_block_list *b, const char *filename, +int backed_block_add_file(struct backed_block_list *bbl, const char *filename, int64_t offset, unsigned int len, unsigned int block) { - struct data_block *db = malloc(sizeof(struct data_block)); - if (db == NULL) { - error_errno("malloc"); - return; + struct backed_block *bb = calloc(1, sizeof(struct backed_block)); + if (bb == NULL) { + return -ENOMEM; } - db->block = block; - db->len = len; - db->filename = strdup(filename); - db->offset = offset; - db->data = NULL; - db->fill = 0; - db->next = NULL; + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_FILE; + bb->file.filename = strdup(filename); + bb->file.offset = offset; + bb->next = NULL; - queue_db(b, db); -} - -/* Iterates over the queued data blocks, calling data_func for each contiguous - data block, and file_func for each contiguous file block */ -void for_each_data_block(struct backed_block_list *b, - data_block_callback_t data_func, - data_block_file_callback_t file_func, - data_block_fill_callback_t fill_func, - void *priv, unsigned int block_size) -{ - struct data_block *db; - u32 last_block = 0; - - for (db = b->data_blocks; db; db = db->next) { - if (db->block < last_block) - error("data blocks out of order: %u < %u", db->block, last_block); - last_block = db->block + DIV_ROUND_UP(db->len, block_size) - 1; - - if (db->filename) - file_func(priv, (u64)db->block * block_size, db->filename, db->offset, db->len); - else if (db->fill) - fill_func(priv, (u64)db->block * block_size, db->fill_val, db->len); - else - data_func(priv, (u64)db->block * block_size, db->data, db->len); - } + return queue_bb(bbl, bb); } diff --git a/libsparse/backed_block.h b/libsparse/backed_block.h index d1bfa1ec3..316650505 100644 --- a/libsparse/backed_block.h +++ b/libsparse/backed_block.h @@ -17,29 +17,38 @@ #ifndef _BACKED_BLOCK_H_ #define _BACKED_BLOCK_H_ +#include + struct backed_block_list; +struct backed_block; -typedef void (*data_block_callback_t)(void *priv, int64_t off, void *data, - int len); -typedef void (*data_block_fill_callback_t)(void *priv, int64_t off, - unsigned int fill_val, int len); -typedef void (*data_block_file_callback_t)(void *priv, int64_t off, - const char *file, int64_t offset, int len); +enum backed_block_type { + BACKED_BLOCK_DATA, + BACKED_BLOCK_FILE, + BACKED_BLOCK_FILL, +}; -void for_each_data_block(struct backed_block_list *b, - data_block_callback_t data_func, - data_block_file_callback_t file_func, - data_block_fill_callback_t fill_func, - void *priv, unsigned int); - -void queue_data_block(struct backed_block_list *b,void *data, unsigned int len, - unsigned int block); -void queue_fill_block(struct backed_block_list *b,unsigned int fill_val, +int backed_block_add_data(struct backed_block_list *bbl, void *data, unsigned int len, unsigned int block); -void queue_data_file(struct backed_block_list *b,const char *filename, +int backed_block_add_fill(struct backed_block_list *bbl, unsigned int fill_val, + unsigned int len, unsigned int block); +int backed_block_add_file(struct backed_block_list *bbl, const char *filename, int64_t offset, unsigned int len, unsigned int block); +struct backed_block *backed_block_iter_new(struct backed_block_list *bbl); +struct backed_block *backed_block_iter_next(struct backed_block *bb); +unsigned int backed_block_len(struct backed_block *bb); +unsigned int backed_block_block(struct backed_block *bb); +void *backed_block_data(struct backed_block *bb); +const char *backed_block_filename(struct backed_block *bb); +int64_t backed_block_file_offset(struct backed_block *bb); +uint32_t backed_block_fill_val(struct backed_block *bb); +enum backed_block_type backed_block_type(struct backed_block *bb); + +struct backed_block *backed_block_iter_new(struct backed_block_list *bbl); +struct backed_block *backed_block_iter_next(struct backed_block *bb); + struct backed_block_list *backed_block_list_new(void); -void backed_block_list_destroy(struct backed_block_list *b); +void backed_block_list_destroy(struct backed_block_list *bbl); #endif diff --git a/libsparse/output_file.c b/libsparse/output_file.c index 2c4b5573e..f911f8cc5 100644 --- a/libsparse/output_file.c +++ b/libsparse/output_file.c @@ -51,36 +51,50 @@ static inline void *mmap64(void *addr, size_t length, int prot, int flags, } #endif +#define min(a, b) \ + ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) + #define SPARSE_HEADER_MAJOR_VER 1 #define SPARSE_HEADER_MINOR_VER 0 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) #define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) struct output_file_ops { - int (*seek)(struct output_file *, int64_t); - int (*write)(struct output_file *, u8 *, int); + int (*skip)(struct output_file *, int64_t); + int (*write)(struct output_file *, void *, int); void (*close)(struct output_file *); }; +struct sparse_file_ops { + int (*write_data_chunk)(struct output_file *out, unsigned int len, + void *data); + int (*write_fill_chunk)(struct output_file *out, unsigned int len, + uint32_t fill_val); + int (*write_skip_chunk)(struct output_file *out, int64_t len); + int (*write_end_chunk)(struct output_file *out); +}; + struct output_file { int fd; gzFile gz_fd; bool close_fd; - int sparse; int64_t cur_out_ptr; - u32 chunk_cnt; - u32 crc32; + unsigned int chunk_cnt; + uint32_t crc32; struct output_file_ops *ops; + struct sparse_file_ops *sparse_ops; int use_crc; unsigned int block_size; int64_t len; + char *zero_buf; + uint32_t *fill_buf; }; -static int file_seek(struct output_file *out, int64_t off) +static int file_skip(struct output_file *out, int64_t cnt) { off64_t ret; - ret = lseek64(out->fd, off, SEEK_SET); + ret = lseek64(out->fd, cnt, SEEK_CUR); if (ret < 0) { error_errno("lseek64"); return -1; @@ -88,7 +102,7 @@ static int file_seek(struct output_file *out, int64_t off) return 0; } -static int file_write(struct output_file *out, u8 *data, int len) +static int file_write(struct output_file *out, void *data, int len) { int ret; ret = write(out->fd, data, len); @@ -110,18 +124,17 @@ static void file_close(struct output_file *out) } } - static struct output_file_ops file_ops = { - .seek = file_seek, + .skip = file_skip, .write = file_write, .close = file_close, }; -static int gz_file_seek(struct output_file *out, int64_t off) +static int gz_file_skip(struct output_file *out, int64_t cnt) { off64_t ret; - ret = gzseek(out->gz_fd, off, SEEK_SET); + ret = gzseek(out->gz_fd, cnt, SEEK_CUR); if (ret < 0) { error_errno("gzseek"); return -1; @@ -129,7 +142,7 @@ static int gz_file_seek(struct output_file *out, int64_t off) return 0; } -static int gz_file_write(struct output_file *out, u8 *data, int len) +static int gz_file_write(struct output_file *out, void *data, int len) { int ret; ret = gzwrite(out->gz_fd, data, len); @@ -150,32 +163,16 @@ static void gz_file_close(struct output_file *out) } static struct output_file_ops gz_file_ops = { - .seek = gz_file_seek, + .skip = gz_file_skip, .write = gz_file_write, .close = gz_file_close, }; -static sparse_header_t sparse_header = { - .magic = SPARSE_HEADER_MAGIC, - .major_version = SPARSE_HEADER_MAJOR_VER, - .minor_version = SPARSE_HEADER_MINOR_VER, - .file_hdr_sz = SPARSE_HEADER_LEN, - .chunk_hdr_sz = CHUNK_HEADER_LEN, - .blk_sz = 0, - .total_blks = 0, - .total_chunks = 0, - .image_checksum = 0 -}; - -static u8 *zero_buf; - -static int emit_skip_chunk(struct output_file *out, u64 skip_len) +static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) { chunk_header_t chunk_header; int ret, chunk; - //DBG printf("skip chunk: 0x%llx bytes\n", skip_len); - if (skip_len % out->block_size) { error("don't care size %llu is not a multiple of the block size %u", skip_len, out->block_size); @@ -187,7 +184,7 @@ static int emit_skip_chunk(struct output_file *out, u64 skip_len) chunk_header.reserved1 = 0; chunk_header.chunk_sz = skip_len / out->block_size; chunk_header.total_sz = CHUNK_HEADER_LEN; - ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); if (ret < 0) return -1; @@ -197,71 +194,34 @@ static int emit_skip_chunk(struct output_file *out, u64 skip_len) return 0; } -static int write_chunk_fill(struct output_file *out, int64_t off, u32 fill_val, int len) +static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, + uint32_t fill_val) { chunk_header_t chunk_header; int rnd_up_len, zero_len, count; int ret; unsigned int i; - u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ - /* We can assume that all the chunks to be written are in - * ascending order, block-size aligned, and non-overlapping. - * So, if the offset is less than the current output pointer, - * throw an error, and if there is a gap, emit a "don't care" - * chunk. The first write (of the super block) may not be - * blocksize aligned, so we need to deal with that too. - */ - //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); - - if (off < out->cur_out_ptr) { - error("offset %llu is less than the current output offset %llu", - off, out->cur_out_ptr); - return -1; - } - - if (off > out->cur_out_ptr) { - emit_skip_chunk(out, off - out->cur_out_ptr); - } - - if (off % out->block_size) { - error("write chunk offset %llu is not a multiple of the block size %u", - off, out->block_size); - return -1; - } - - if (off != out->cur_out_ptr) { - error("internal error, offset accounting screwy in write_chunk_raw()"); - return -1; - } - - /* Round up the file length to a multiple of the block size */ - rnd_up_len = (len + (out->block_size - 1)) & (~(out->block_size -1)); + /* Round up the fill length to a multiple of the block size */ + rnd_up_len = ALIGN(len, out->block_size); /* Finally we can safely emit a chunk of data */ chunk_header.chunk_type = CHUNK_TYPE_FILL; chunk_header.reserved1 = 0; chunk_header.chunk_sz = rnd_up_len / out->block_size; chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); - ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); if (ret < 0) return -1; - ret = out->ops->write(out, (u8 *)&fill_val, sizeof(fill_val)); + ret = out->ops->write(out, &fill_val, sizeof(fill_val)); if (ret < 0) return -1; if (out->use_crc) { - /* Initialize fill_buf with the fill_val */ - for (i = 0; i < (out->block_size / sizeof(u32)); i++) { - fill_buf[i] = fill_val; - } - - count = chunk_header.chunk_sz; - while (count) { - out->crc32 = sparse_crc32(out->crc32, fill_buf, out->block_size); - count--; - } + count = out->block_size / sizeof(uint32_t); + while (count--) + out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); } out->cur_out_ptr += rnd_up_len; @@ -270,44 +230,15 @@ static int write_chunk_fill(struct output_file *out, int64_t off, u32 fill_val, return 0; } -static int write_chunk_raw(struct output_file *out, int64_t off, u8 *data, int len) +static int write_sparse_data_chunk(struct output_file *out, unsigned int len, + void *data) { chunk_header_t chunk_header; int rnd_up_len, zero_len; int ret; - /* We can assume that all the chunks to be written are in - * ascending order, block-size aligned, and non-overlapping. - * So, if the offset is less than the current output pointer, - * throw an error, and if there is a gap, emit a "don't care" - * chunk. The first write (of the super block) may not be - * blocksize aligned, so we need to deal with that too. - */ - //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); - - if (off < out->cur_out_ptr) { - error("offset %llu is less than the current output offset %llu", - off, out->cur_out_ptr); - return -1; - } - - if (off > out->cur_out_ptr) { - emit_skip_chunk(out, off - out->cur_out_ptr); - } - - if (off % out->block_size) { - error("write chunk offset %llu is not a multiple of the block size %u", - off, out->block_size); - return -1; - } - - if (off != out->cur_out_ptr) { - error("internal error, offset accounting screwy in write_chunk_raw()"); - return -1; - } - - /* Round up the file length to a multiple of the block size */ - rnd_up_len = (len + (out->block_size - 1)) & (~(out->block_size -1)); + /* Round up the data length to a multiple of the block size */ + rnd_up_len = ALIGN(len, out->block_size); zero_len = rnd_up_len - len; /* Finally we can safely emit a chunk of data */ @@ -315,7 +246,7 @@ static int write_chunk_raw(struct output_file *out, int64_t off, u8 *data, int l chunk_header.reserved1 = 0; chunk_header.chunk_sz = rnd_up_len / out->block_size; chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; - ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); if (ret < 0) return -1; @@ -323,7 +254,7 @@ static int write_chunk_raw(struct output_file *out, int64_t off, u8 *data, int l if (ret < 0) return -1; if (zero_len) { - ret = out->ops->write(out, zero_buf, zero_len); + ret = out->ops->write(out, out->zero_buf, zero_len); if (ret < 0) return -1; } @@ -331,7 +262,7 @@ static int write_chunk_raw(struct output_file *out, int64_t off, u8 *data, int l if (out->use_crc) { out->crc32 = sparse_crc32(out->crc32, data, len); if (zero_len) - out->crc32 = sparse_crc32(out->crc32, zero_buf, zero_len); + out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); } out->cur_out_ptr += rnd_up_len; @@ -340,29 +271,115 @@ static int write_chunk_raw(struct output_file *out, int64_t off, u8 *data, int l return 0; } +int write_sparse_end_chunk(struct output_file *out) +{ + chunk_header_t chunk_header; + int ret; + + if (out->use_crc) { + chunk_header.chunk_type = CHUNK_TYPE_CRC32; + chunk_header.reserved1 = 0; + chunk_header.chunk_sz = 0; + chunk_header.total_sz = CHUNK_HEADER_LEN + 4; + + ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); + if (ret < 0) { + return ret; + } + out->ops->write(out, &out->crc32, 4); + if (ret < 0) { + return ret; + } + + out->chunk_cnt++; + } + + return 0; +} + +static struct sparse_file_ops sparse_file_ops = { + .write_data_chunk = write_sparse_data_chunk, + .write_fill_chunk = write_sparse_fill_chunk, + .write_skip_chunk = write_sparse_skip_chunk, + .write_end_chunk = write_sparse_end_chunk, +}; + +static int write_normal_data_chunk(struct output_file *out, unsigned int len, + void *data) +{ + int ret; + unsigned int rnd_up_len = ALIGN(len, out->block_size); + + ret = out->ops->write(out, data, len); + if (ret < 0) { + return ret; + } + + if (rnd_up_len > len) { + ret = out->ops->skip(out, rnd_up_len - len); + } + + return ret; +} + +static int write_normal_fill_chunk(struct output_file *out, unsigned int len, + uint32_t fill_val) +{ + int ret; + unsigned int i; + unsigned int write_len; + + /* Initialize fill_buf with the fill_val */ + for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { + out->fill_buf[i] = fill_val; + } + + while (len) { + write_len = min(len, out->block_size); + ret = out->ops->write(out, out->fill_buf, write_len); + if (ret < 0) { + return ret; + } + + len -= write_len; + } + + return 0; +} + +static int write_normal_skip_chunk(struct output_file *out, int64_t len) +{ + int ret; + + return out->ops->skip(out, len); +} + +int write_normal_end_chunk(struct output_file *out) +{ + int ret; + + ret = ftruncate64(out->fd, out->len); + if (ret < 0) { + return -errno; + } + + return 0; +} + +static struct sparse_file_ops normal_file_ops = { + .write_data_chunk = write_normal_data_chunk, + .write_fill_chunk = write_normal_fill_chunk, + .write_skip_chunk = write_normal_skip_chunk, + .write_end_chunk = write_normal_end_chunk, +}; + void close_output_file(struct output_file *out) { int ret; - chunk_header_t chunk_header; - if (out->sparse) { - if (out->use_crc) { - chunk_header.chunk_type = CHUNK_TYPE_CRC32; - chunk_header.reserved1 = 0; - chunk_header.chunk_sz = 0; - chunk_header.total_sz = CHUNK_HEADER_LEN + 4; - - out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); - out->ops->write(out, (u8 *)&out->crc32, 4); - - out->chunk_cnt++; - } - - if (out->chunk_cnt != sparse_header.total_chunks) - error("sparse chunk count did not match: %d %d", out->chunk_cnt, - sparse_header.total_chunks); - } + out->sparse_ops->write_end_chunk(out); out->ops->close(out); + free(out); } struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, @@ -374,28 +391,37 @@ struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, error_errno("malloc struct out"); return NULL; } - zero_buf = malloc(out->block_size); - if (!zero_buf) { + out->zero_buf = calloc(block_size, 1); + if (!out->zero_buf) { error_errno("malloc zero_buf"); - free(out); - return NULL; + goto err_zero_buf; + } + + out->fill_buf = calloc(block_size, 1); + if (!out->fill_buf) { + error_errno("malloc fill_buf"); + goto err_fill_buf; } - memset(zero_buf, '\0', out->block_size); if (gz) { out->ops = &gz_file_ops; out->gz_fd = gzdopen(fd, "wb9"); if (!out->gz_fd) { error_errno("gzopen"); - free(out); - return NULL; + goto err_gzopen; } } else { out->fd = fd; out->ops = &file_ops; } + + if (sparse) { + out->sparse_ops = &sparse_file_ops; + } else { + out->sparse_ops = &normal_file_ops; + } + out->close_fd = false; - out->sparse = sparse; out->cur_out_ptr = 0ll; out->chunk_cnt = 0; @@ -406,19 +432,42 @@ struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, out->len = len; out->block_size = block_size; - if (out->sparse) { - sparse_header.blk_sz = out->block_size, - sparse_header.total_blks = out->len / out->block_size, - sparse_header.total_chunks = chunks; - if (out->use_crc) - sparse_header.total_chunks++; + if (sparse) { + sparse_header_t sparse_header = { + .magic = SPARSE_HEADER_MAGIC, + .major_version = SPARSE_HEADER_MAJOR_VER, + .minor_version = SPARSE_HEADER_MINOR_VER, + .file_hdr_sz = SPARSE_HEADER_LEN, + .chunk_hdr_sz = CHUNK_HEADER_LEN, + .blk_sz = out->block_size, + .total_blks = out->len / out->block_size, + .total_chunks = chunks, + .image_checksum = 0 + }; - ret = out->ops->write(out, (u8 *)&sparse_header, sizeof(sparse_header)); - if (ret < 0) - return NULL; + if (out->use_crc) { + sparse_header.total_chunks++; + } + + ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); + if (ret < 0) { + goto err_write; + } } return out; + +err_write: + if (gz) { + gzclose(out->gz_fd); + } +err_gzopen: + free(out->fill_buf); +err_fill_buf: + free(out->zero_buf); +err_zero_buf: + free(out); + return NULL; } struct output_file *open_output_file(const char *filename, @@ -449,123 +498,31 @@ struct output_file *open_output_file(const char *filename, return file; } -void pad_output_file(struct output_file *out, int64_t len) -{ - int ret; - - if (len > out->len) { - error("attempted to pad file %llu bytes past end of filesystem", - len - out->len); - return; - } - if (out->sparse) { - /* We need to emit a DONT_CARE chunk to pad out the file if the - * cur_out_ptr is not already at the end of the filesystem. - */ - if (len < out->cur_out_ptr) { - error("attempted to pad file %llu bytes less than the current output pointer", - out->cur_out_ptr - len); - return; - } - if (len > out->cur_out_ptr) { - emit_skip_chunk(out, len - out->cur_out_ptr); - } - } else { - //KEN TODO: Fixme. If the filesystem image needs no padding, - // this will overwrite the last byte in the file with 0 - // The answer is to do accounting like the sparse image - // code does and know if there is already data there. - ret = out->ops->seek(out, len - 1); - if (ret < 0) - return; - - ret = out->ops->write(out, (u8*)"", 1); - if (ret < 0) - return; - } -} - /* Write a contiguous region of data blocks from a memory buffer */ -void write_data_block(struct output_file *out, int64_t off, void *data, int len) +int write_data_chunk(struct output_file *out, unsigned int len, void *data) { - int ret; - - if (off + len > out->len) { - error("attempted to write block %llu past end of filesystem", - off + len - out->len); - return; - } - - if (out->sparse) { - write_chunk_raw(out, off, data, len); - } else { - ret = out->ops->seek(out, off); - if (ret < 0) - return; - - ret = out->ops->write(out, data, len); - if (ret < 0) - return; - } + return out->sparse_ops->write_data_chunk(out, len, data); } /* Write a contiguous region of data blocks with a fill value */ -void write_fill_block(struct output_file *out, int64_t off, unsigned int fill_val, int len) +int write_fill_chunk(struct output_file *out, unsigned int len, + uint32_t fill_val) { - int ret; - unsigned int i; - int write_len; - u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ - - if (off + len > out->len) { - error("attempted to write block %llu past end of filesystem", - off + len - out->len); - return; - } - - if (out->sparse) { - write_chunk_fill(out, off, fill_val, len); - } else { - /* Initialize fill_buf with the fill_val */ - for (i = 0; i < sizeof(fill_buf)/sizeof(u32); i++) { - fill_buf[i] = fill_val; - } - - ret = out->ops->seek(out, off); - if (ret < 0) - return; - - while (len) { - write_len = (len > (int)sizeof(fill_buf) ? (int)sizeof(fill_buf) : len); - ret = out->ops->write(out, (u8 *)fill_buf, write_len); - if (ret < 0) { - return; - } else { - len -= write_len; - } - } - } + return out->sparse_ops->write_fill_chunk(out, len, fill_val); } /* Write a contiguous region of data blocks from a file */ -void write_data_file(struct output_file *out, int64_t off, const char *file, - int64_t offset, int len) +int write_file_chunk(struct output_file *out, unsigned int len, + const char *file, int64_t offset) { int ret; int64_t aligned_offset; int aligned_diff; int buffer_size; - if (off + len >= out->len) { - error("attempted to write block %llu past end of filesystem", - off + len - out->len); - return; - } - int file_fd = open(file, O_RDONLY | O_BINARY); if (file_fd < 0) { - error_errno("open"); - return; + return -errno; } aligned_offset = offset & ~(4096 - 1); @@ -573,41 +530,36 @@ void write_data_file(struct output_file *out, int64_t off, const char *file, buffer_size = len + aligned_diff; #ifndef USE_MINGW - u8 *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, file_fd, + char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, file_fd, aligned_offset); if (data == MAP_FAILED) { - error_errno("mmap64"); + ret = -errno; close(file_fd); - return; + return ret; } #else - u8 *data = malloc(buffer_size); + char *data = malloc(buffer_size); if (!data) { - error_errno("malloc"); + ret = -errno; close(file_fd); - return; + return ret; } memset(data, 0, buffer_size); #endif - if (out->sparse) { - write_chunk_raw(out, off, data + aligned_diff, len); - } else { - ret = out->ops->seek(out, off); - if (ret < 0) - goto err; + ret = out->sparse_ops->write_data_chunk(out, len, data + aligned_diff); - ret = out->ops->write(out, data + aligned_diff, len); - if (ret < 0) - goto err; - } - -err: #ifndef USE_MINGW munmap(data, buffer_size); #else - write(file_fd, data, buffer_size); free(data); #endif close(file_fd); + + return ret; +} + +int write_skip_chunk(struct output_file *out, int64_t len) +{ + return out->sparse_ops->write_skip_chunk(out, len); } diff --git a/libsparse/output_file.h b/libsparse/output_file.h index b12194fc7..cb2feb7fb 100644 --- a/libsparse/output_file.h +++ b/libsparse/output_file.h @@ -26,11 +26,12 @@ struct output_file *open_output_file(const char *filename, int gz, int sparse, int chunks, int crc); struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc); -void write_data_block(struct output_file *out, int64_t off, void *data, int len); -void write_fill_block(struct output_file *out, int64_t off, unsigned int fill_val, int len); -void write_data_file(struct output_file *out, int64_t off, const char *file, - int64_t offset, int len); -void pad_output_file(struct output_file *out, int64_t len); +int write_data_chunk(struct output_file *out, unsigned int len, void *data); +int write_fill_chunk(struct output_file *out, unsigned int len, + uint32_t fill_val); +int write_file_chunk(struct output_file *out, unsigned int len, + const char *file, int64_t offset); +int write_skip_chunk(struct output_file *out, int64_t len); void close_output_file(struct output_file *out); #endif diff --git a/libsparse/sparse.c b/libsparse/sparse.c index a6134c9d2..fce9dbbb6 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -14,6 +14,7 @@ * limitations under the License. */ +#include #include #include @@ -24,7 +25,6 @@ #include "backed_block.h" #include "sparse_defs.h" - struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len) { struct sparse_file *s = calloc(sizeof(struct sparse_file), 1); @@ -53,108 +53,89 @@ void sparse_file_destroy(struct sparse_file *s) int sparse_file_add_data(struct sparse_file *s, void *data, unsigned int len, unsigned int block) { - queue_data_block(s->backed_block_list, data, len, block); - - return 0; + return backed_block_add_data(s->backed_block_list, data, len, block); } int sparse_file_add_fill(struct sparse_file *s, uint32_t fill_val, unsigned int len, unsigned int block) { - queue_fill_block(s->backed_block_list, fill_val, len, block); - - return 0; + return backed_block_add_fill(s->backed_block_list, fill_val, len, block); } int sparse_file_add_file(struct sparse_file *s, const char *filename, int64_t file_offset, unsigned int len, unsigned int block) { - queue_data_file(s->backed_block_list, filename, file_offset, len, block); - - return 0; + return backed_block_add_file(s->backed_block_list, filename, file_offset, + len, block); } -struct count_chunks { - unsigned int chunks; - int64_t cur_ptr; - unsigned int block_size; -}; - -static void count_data_block(void *priv, int64_t off, void *data, int len) +unsigned int sparse_count_chunks(struct sparse_file *s) { - struct count_chunks *count_chunks = priv; - if (off > count_chunks->cur_ptr) - count_chunks->chunks++; - count_chunks->cur_ptr = off + ALIGN(len, count_chunks->block_size); - count_chunks->chunks++; -} + struct backed_block *bb; + unsigned int last_block = 0; + unsigned int chunks = 0; -static void count_fill_block(void *priv, int64_t off, unsigned int fill_val, int len) -{ - struct count_chunks *count_chunks = priv; - if (off > count_chunks->cur_ptr) - count_chunks->chunks++; - count_chunks->cur_ptr = off + ALIGN(len, count_chunks->block_size); - count_chunks->chunks++; -} + for (bb = backed_block_iter_new(s->backed_block_list); bb; + bb = backed_block_iter_next(bb)) { + if (backed_block_block(bb) > last_block) { + /* If there is a gap between chunks, add a skip chunk */ + chunks++; + } + chunks++; + last_block = backed_block_block(bb) + + DIV_ROUND_UP(backed_block_len(bb), s->block_size); + } + if (last_block < DIV_ROUND_UP(s->len, s->block_size)) { + chunks++; + } -static void count_file_block(void *priv, int64_t off, const char *file, - int64_t offset, int len) -{ - struct count_chunks *count_chunks = priv; - if (off > count_chunks->cur_ptr) - count_chunks->chunks++; - count_chunks->cur_ptr = off + ALIGN(len, count_chunks->block_size); - count_chunks->chunks++; -} - -static int count_sparse_chunks(struct backed_block_list *b, - unsigned int block_size, int64_t len) -{ - struct count_chunks count_chunks = {0, 0, block_size}; - - for_each_data_block(b, count_data_block, count_file_block, - count_fill_block, &count_chunks, block_size); - - if (count_chunks.cur_ptr != len) - count_chunks.chunks++; - - return count_chunks.chunks; -} - -static void ext4_write_data_block(void *priv, int64_t off, void *data, int len) -{ - write_data_block(priv, off, data, len); -} - -static void ext4_write_fill_block(void *priv, int64_t off, unsigned int fill_val, int len) -{ - write_fill_block(priv, off, fill_val, len); -} - -static void ext4_write_data_file(void *priv, int64_t off, const char *file, - int64_t offset, int len) -{ - write_data_file(priv, off, file, offset, len); + return chunks; } int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, bool crc) { - int chunks = count_sparse_chunks(s->backed_block_list, s->block_size, - s->len); - struct output_file *out = open_output_fd(fd, s->block_size, s->len, - gz, sparse, chunks, crc); + struct backed_block *bb; + unsigned int last_block = 0; + int64_t pad; + int chunks; + struct output_file *out; + + chunks = sparse_count_chunks(s); + out = open_output_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc); if (!out) return -ENOMEM; - for_each_data_block(s->backed_block_list, ext4_write_data_block, - ext4_write_data_file, ext4_write_fill_block, out, s->block_size); + for (bb = backed_block_iter_new(s->backed_block_list); bb; + bb = backed_block_iter_next(bb)) { + if (backed_block_block(bb) > last_block) { + unsigned int blocks = backed_block_block(bb) - last_block; + write_skip_chunk(out, (int64_t)blocks * s->block_size); + } + switch (backed_block_type(bb)) { + case BACKED_BLOCK_DATA: + write_data_chunk(out, backed_block_len(bb), backed_block_data(bb)); + break; + case BACKED_BLOCK_FILE: + write_file_chunk(out, backed_block_len(bb), + backed_block_filename(bb), backed_block_file_offset(bb)); + break; + case BACKED_BLOCK_FILL: + write_fill_chunk(out, backed_block_len(bb), + backed_block_fill_val(bb)); + break; + } + last_block = backed_block_block(bb) + + DIV_ROUND_UP(backed_block_len(bb), s->block_size); + } - if (s->len) - pad_output_file(out, s->len); + pad = s->len - last_block * s->block_size; + assert(pad >= 0); + if (pad > 0) { + write_skip_chunk(out, pad); + } close_output_file(out); diff --git a/libsparse/sparse_crc32.h b/libsparse/sparse_crc32.h index 21625bac9..cad8a8630 100644 --- a/libsparse/sparse_crc32.h +++ b/libsparse/sparse_crc32.h @@ -14,5 +14,7 @@ * limitations under the License. */ -u32 sparse_crc32(u32 crc, const void *buf, size_t size); +#include + +uint32_t sparse_crc32(uint32_t crc, const void *buf, size_t size); From 9e1f17e926fa20255c5f4b4d2f68aa98a964253a Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 25 Apr 2012 18:31:39 -0700 Subject: [PATCH 04/16] libsparse: add support for including fds Add sparse_file_add_fd to include all or part of the contents of an fd in the output file. Will be useful for re-sparsing files where fd will point to the input sparse file. Change-Id: I5d4ab07fb37231e8e9c1912f62a2968c8b0a00ef --- libsparse/backed_block.c | 37 +++++++++++++++++++++++++++++-- libsparse/backed_block.h | 4 ++++ libsparse/include/sparse/sparse.h | 26 ++++++++++++++++++++++ libsparse/output_file.c | 37 ++++++++++++++++++------------- libsparse/output_file.h | 2 ++ libsparse/sparse.c | 10 +++++++++ 6 files changed, 99 insertions(+), 17 deletions(-) diff --git a/libsparse/backed_block.c b/libsparse/backed_block.c index b25919008..8c3fab04a 100644 --- a/libsparse/backed_block.c +++ b/libsparse/backed_block.c @@ -34,6 +34,10 @@ struct backed_block { char *filename; int64_t offset; } file; + struct { + int fd; + int64_t offset; + } fd; struct { uint32_t val; } fill; @@ -78,10 +82,20 @@ const char *backed_block_filename(struct backed_block *bb) return bb->file.filename; } +int backed_block_fd(struct backed_block *bb) +{ + assert(bb->type == BACKED_BLOCK_FD); + return bb->fd.fd; +} + int64_t backed_block_file_offset(struct backed_block *bb) { - assert(bb->type == BACKED_BLOCK_FILE); - return bb->file.offset; + assert(bb->type == BACKED_BLOCK_FILE || bb->type == BACKED_BLOCK_FD); + if (bb->type == BACKED_BLOCK_FILE) { + return bb->file.offset; + } else { /* bb->type == BACKED_BLOCK_FD */ + return bb->fd.offset; + } } uint32_t backed_block_fill_val(struct backed_block *bb) @@ -211,3 +225,22 @@ int backed_block_add_file(struct backed_block_list *bbl, const char *filename, return queue_bb(bbl, bb); } + +/* Queues a chunk of a fd to be written to the specified data blocks */ +int backed_block_add_fd(struct backed_block_list *bbl, int fd, int64_t offset, + unsigned int len, unsigned int block) +{ + struct backed_block *bb = calloc(1, sizeof(struct backed_block)); + if (bb == NULL) { + return -ENOMEM; + } + + bb->block = block; + bb->len = len; + bb->type = BACKED_BLOCK_FD; + bb->fd.fd = fd; + bb->fd.offset = offset; + bb->next = NULL; + + return queue_bb(bbl, bb); +} diff --git a/libsparse/backed_block.h b/libsparse/backed_block.h index 316650505..ca2ad1d9b 100644 --- a/libsparse/backed_block.h +++ b/libsparse/backed_block.h @@ -25,6 +25,7 @@ struct backed_block; enum backed_block_type { BACKED_BLOCK_DATA, BACKED_BLOCK_FILE, + BACKED_BLOCK_FD, BACKED_BLOCK_FILL, }; @@ -34,6 +35,8 @@ int backed_block_add_fill(struct backed_block_list *bbl, unsigned int fill_val, unsigned int len, unsigned int block); int backed_block_add_file(struct backed_block_list *bbl, const char *filename, int64_t offset, unsigned int len, unsigned int block); +int backed_block_add_fd(struct backed_block_list *bbl, int fd, + int64_t offset, unsigned int len, unsigned int block); struct backed_block *backed_block_iter_new(struct backed_block_list *bbl); struct backed_block *backed_block_iter_next(struct backed_block *bb); @@ -41,6 +44,7 @@ unsigned int backed_block_len(struct backed_block *bb); unsigned int backed_block_block(struct backed_block *bb); void *backed_block_data(struct backed_block *bb); const char *backed_block_filename(struct backed_block *bb); +int backed_block_fd(struct backed_block *bb); int64_t backed_block_file_offset(struct backed_block *bb); uint32_t backed_block_fill_val(struct backed_block *bb); enum backed_block_type backed_block_type(struct backed_block *bb); diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h index db0688471..6484333ef 100644 --- a/libsparse/include/sparse/sparse.h +++ b/libsparse/include/sparse/sparse.h @@ -110,6 +110,32 @@ int sparse_file_add_file(struct sparse_file *s, const char *filename, int64_t file_offset, unsigned int len, unsigned int block); +/** + * sparse_file_add_file - associate a chunk of a file with a sparse file + * + * @s - sparse file cookie + * @filename - filename of the file to be copied + * @file_offset - offset into the copied file + * @len - length of the copied block + * @block - offset in blocks into the sparse file to place the file chunk + * + * Associates a chunk of an existing fd with a sparse file cookie. + * The region [block * block_size : block * block_size + len) must not already + * be used in the sparse file. If len is not a multiple of the block size the + * data will be padded with zeros. + * + * Allows adding large amounts of data to a sparse file without needing to keep + * it all mapped. File size is limited by available virtual address space, + * exceptionally large files may need to be added in multiple chunks. + * + * The fd must remain open until the sparse file is closed or the fd block is + * removed from the sparse file. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_add_fd(struct sparse_file *s, + int fd, int64_t file_offset, unsigned int len, unsigned int block); + /** * sparse_file_write - write a sparse file to a file * diff --git a/libsparse/output_file.c b/libsparse/output_file.c index f911f8cc5..4193fd17d 100644 --- a/libsparse/output_file.c +++ b/libsparse/output_file.c @@ -511,38 +511,28 @@ int write_fill_chunk(struct output_file *out, unsigned int len, return out->sparse_ops->write_fill_chunk(out, len, fill_val); } -/* Write a contiguous region of data blocks from a file */ -int write_file_chunk(struct output_file *out, unsigned int len, - const char *file, int64_t offset) +int write_fd_chunk(struct output_file *out, unsigned int len, + int fd, int64_t offset) { int ret; int64_t aligned_offset; int aligned_diff; int buffer_size; - int file_fd = open(file, O_RDONLY | O_BINARY); - if (file_fd < 0) { - return -errno; - } - aligned_offset = offset & ~(4096 - 1); aligned_diff = offset - aligned_offset; buffer_size = len + aligned_diff; #ifndef USE_MINGW - char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, file_fd, + char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, aligned_offset); if (data == MAP_FAILED) { - ret = -errno; - close(file_fd); - return ret; + return -errno; } #else char *data = malloc(buffer_size); if (!data) { - ret = -errno; - close(file_fd); - return ret; + return -errno; } memset(data, 0, buffer_size); #endif @@ -554,6 +544,23 @@ int write_file_chunk(struct output_file *out, unsigned int len, #else free(data); #endif + + return ret; +} + +/* Write a contiguous region of data blocks from a file */ +int write_file_chunk(struct output_file *out, unsigned int len, + const char *file, int64_t offset) +{ + int ret; + + int file_fd = open(file, O_RDONLY | O_BINARY); + if (file_fd < 0) { + return -errno; + } + + ret = write_fd_chunk(out, len, file_fd, offset); + close(file_fd); return ret; diff --git a/libsparse/output_file.h b/libsparse/output_file.h index cb2feb7fb..d23abf381 100644 --- a/libsparse/output_file.h +++ b/libsparse/output_file.h @@ -31,6 +31,8 @@ int write_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val); int write_file_chunk(struct output_file *out, unsigned int len, const char *file, int64_t offset); +int write_fd_chunk(struct output_file *out, unsigned int len, + int fd, int64_t offset); int write_skip_chunk(struct output_file *out, int64_t len); void close_output_file(struct output_file *out); diff --git a/libsparse/sparse.c b/libsparse/sparse.c index fce9dbbb6..4ebcf0f8d 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -70,6 +70,12 @@ int sparse_file_add_file(struct sparse_file *s, len, block); } +int sparse_file_add_fd(struct sparse_file *s, + int fd, int64_t file_offset, unsigned int len, unsigned int block) +{ + return backed_block_add_fd(s->backed_block_list, fd, file_offset, + len, block); +} unsigned int sparse_count_chunks(struct sparse_file *s) { struct backed_block *bb; @@ -122,6 +128,10 @@ int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, write_file_chunk(out, backed_block_len(bb), backed_block_filename(bb), backed_block_file_offset(bb)); break; + case BACKED_BLOCK_FD: + write_fd_chunk(out, backed_block_len(bb), + backed_block_fd(bb), backed_block_file_offset(bb)); + break; case BACKED_BLOCK_FILL: write_fill_chunk(out, backed_block_len(bb), backed_block_fill_val(bb)); From a21930b6b0dbb04a52948566d58fb48c6db58bab Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 26 Apr 2012 14:24:35 -0700 Subject: [PATCH 05/16] libsparse: add error reporting functions Change-Id: I2f21355b6c5339d1d724b4c121ea30d575b2d366 --- libsparse/Android.mk | 3 ++- libsparse/include/sparse/sparse.h | 18 +++++++++++++++++ libsparse/sparse.c | 5 +++++ libsparse/sparse_err.c | 33 +++++++++++++++++++++++++++++++ libsparse/sparse_file.h | 1 + 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 libsparse/sparse_err.c diff --git a/libsparse/Android.mk b/libsparse/Android.mk index dbe4d1842..d3eeae651 100644 --- a/libsparse/Android.mk +++ b/libsparse/Android.mk @@ -6,7 +6,8 @@ libsparse_src_files := \ backed_block.c \ output_file.c \ sparse.c \ - sparse_crc32.c + sparse_crc32.c \ + sparse_err.c include $(CLEAR_VARS) diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h index 6484333ef..09a513725 100644 --- a/libsparse/include/sparse/sparse.h +++ b/libsparse/include/sparse/sparse.h @@ -157,4 +157,22 @@ int sparse_file_add_fd(struct sparse_file *s, int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, bool crc); +/** + * sparse_file_verbose - set a sparse file cookie to print verbose errors + * + * @s - sparse file cookie + * + * Print verbose sparse file errors whenever using the sparse file cookie. + */ +void sparse_file_verbose(struct sparse_file *s); + +/** + * sparse_print_verbose - function called to print verbose errors + * + * By default, verbose errors will print to standard error. + * sparse_print_verbose may be overridden to log verbose errors somewhere else. + * + */ +extern void (*sparse_print_verbose)(const char *fmt, ...); + #endif diff --git a/libsparse/sparse.c b/libsparse/sparse.c index 4ebcf0f8d..3403604d4 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -151,3 +151,8 @@ int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, return 0; } + +void sparse_file_verbose(struct sparse_file *s) +{ + s->verbose = true; +} diff --git a/libsparse/sparse_err.c b/libsparse/sparse_err.c new file mode 100644 index 000000000..0f392ad60 --- /dev/null +++ b/libsparse/sparse_err.c @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +void sparse_default_print(const char *fmt, ...) +{ + va_list argp; + + va_start(argp, fmt); + vfprintf(stderr, fmt, argp); + va_end(argp); +} + +void (*sparse_print_error)(const char *fmt, ...) = sparse_default_print; +void (*sparse_print_verbose)(const char *fmt, ...) = sparse_default_print; diff --git a/libsparse/sparse_file.h b/libsparse/sparse_file.h index fae1c168c..91a12e678 100644 --- a/libsparse/sparse_file.h +++ b/libsparse/sparse_file.h @@ -22,6 +22,7 @@ struct sparse_file { unsigned int block_size; int64_t len; + bool verbose; struct backed_block_list *backed_block_list; struct output_file *out; From be8ddcb35a459481c0bcf5bfe645c1fefe963f5c Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 25 Apr 2012 21:02:16 -0700 Subject: [PATCH 06/16] libsparse: merge adjacent blocks of the same type When a block is added that is adjacent to another block and of the same type, merge it. This will be useful for converting regular images to sparse images, allowing the reader to add a single block at a time and letting libsparse optimize into larger blocks as it goes. Does not support merge two blocks that are backed by a data pointer, only blocks that are backed by a file for now. Change-Id: I95aa231714cbe01ac194e868c21385806c0bdb97 --- libsparse/backed_block.c | 81 ++++++++++++++++++++++++++++++++++++---- libsparse/backed_block.h | 2 +- libsparse/sparse.c | 2 +- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/libsparse/backed_block.c b/libsparse/backed_block.c index 8c3fab04a..629fc284f 100644 --- a/libsparse/backed_block.c +++ b/libsparse/backed_block.c @@ -21,6 +21,7 @@ #include #include "backed_block.h" +#include "sparse_defs.h" struct backed_block { unsigned int block; @@ -48,6 +49,7 @@ struct backed_block { struct backed_block_list { struct backed_block *data_blocks; struct backed_block *last_used; + unsigned int block_size; }; struct backed_block *backed_block_iter_new(struct backed_block_list *bbl) @@ -109,10 +111,19 @@ enum backed_block_type backed_block_type(struct backed_block *bb) return bb->type; } -struct backed_block_list *backed_block_list_new(void) +void backed_block_destroy(struct backed_block *bb) +{ + if (bb->type == BACKED_BLOCK_FILE) { + free(bb->file.filename); + } + + free(bb); +} + +struct backed_block_list *backed_block_list_new(unsigned int block_size) { struct backed_block_list *b = calloc(sizeof(struct backed_block_list), 1); - + b->block_size = block_size; return b; } @@ -122,11 +133,7 @@ void backed_block_list_destroy(struct backed_block_list *bbl) struct backed_block *bb = bbl->data_blocks; while (bb) { struct backed_block *next = bb->next; - if (bb->type == BACKED_BLOCK_FILE) { - free(bb->file.filename); - } - - free(bb); + backed_block_destroy(bb); bb = next; } } @@ -134,6 +141,63 @@ void backed_block_list_destroy(struct backed_block_list *bbl) free(bbl); } +/* may free b */ +static int merge_bb(struct backed_block_list *bbl, + struct backed_block *a, struct backed_block *b) +{ + unsigned int block_len; + + /* Block doesn't exist (possible if one block is the last block) */ + if (!a || !b) { + return -EINVAL; + } + + assert(a->block < b->block); + + /* Blocks are of different types */ + if (a->type != b->type) { + return -EINVAL; + } + + /* Blocks are not adjacent */ + block_len = a->len / bbl->block_size; /* rounds down */ + if (a->block + block_len != b->block) { + return -EINVAL; + } + + switch (a->type) { + case BACKED_BLOCK_DATA: + /* Don't support merging data for now */ + return -EINVAL; + case BACKED_BLOCK_FILL: + if (a->fill.val != b->fill.val) { + return -EINVAL; + } + break; + case BACKED_BLOCK_FILE: + if (a->file.filename != b->file.filename || + a->file.offset + a->len != b->file.offset) { + return -EINVAL; + } + break; + case BACKED_BLOCK_FD: + if (a->fd.fd != b->fd.fd || + a->fd.offset + a->len != b->fd.offset) { + return -EINVAL; + } + break; + } + + /* Blocks are compatible and adjacent, with a before b. Merge b into a, + * and free b */ + a->len += b->len; + a->next = b->next; + + backed_block_destroy(b); + + return 0; +} + static int queue_bb(struct backed_block_list *bbl, struct backed_block *new_bb) { struct backed_block *bb; @@ -168,6 +232,9 @@ static int queue_bb(struct backed_block_list *bbl, struct backed_block *new_bb) bb->next = new_bb; } + merge_bb(bbl, new_bb, new_bb->next); + merge_bb(bbl, bb, new_bb); + return 0; } diff --git a/libsparse/backed_block.h b/libsparse/backed_block.h index ca2ad1d9b..692691712 100644 --- a/libsparse/backed_block.h +++ b/libsparse/backed_block.h @@ -52,7 +52,7 @@ enum backed_block_type backed_block_type(struct backed_block *bb); struct backed_block *backed_block_iter_new(struct backed_block_list *bbl); struct backed_block *backed_block_iter_next(struct backed_block *bb); -struct backed_block_list *backed_block_list_new(void); +struct backed_block_list *backed_block_list_new(unsigned int block_size); void backed_block_list_destroy(struct backed_block_list *bbl); #endif diff --git a/libsparse/sparse.c b/libsparse/sparse.c index 3403604d4..d778e1dc6 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -32,7 +32,7 @@ struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len) return NULL; } - s->backed_block_list = backed_block_list_new(); + s->backed_block_list = backed_block_list_new(block_size); if (!s->backed_block_list) { free(s); return NULL; From 13a560659381b34ce3edbfa8dbe6c0aa6c076f20 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 19 Jun 2012 16:45:48 -0700 Subject: [PATCH 07/16] libsparse: fix windows image writing Fix write_fd_chunk on windows. Uses malloc and read instead of mmap. Change-Id: I75f10db2e04f19e7f3a6ff46b6978d143cb5254e --- libsparse/output_file.c | 39 ++++++++++++++++++++++++++++++++++++--- libsparse/output_file.h | 2 ++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/libsparse/output_file.c b/libsparse/output_file.c index 4193fd17d..5e8a68c54 100644 --- a/libsparse/output_file.c +++ b/libsparse/output_file.c @@ -168,6 +168,28 @@ static struct output_file_ops gz_file_ops = { .close = gz_file_close, }; +int read_all(int fd, void *buf, size_t len) +{ + size_t total = 0; + int ret; + char *ptr = buf; + + while (total < len) { + ret = read(fd, ptr, len - total); + + if (ret < 0) + return -errno; + + if (ret == 0) + return -EINVAL; + + ptr += ret; + total += ret; + } + + return 0; +} + static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) { chunk_header_t chunk_header; @@ -518,6 +540,7 @@ int write_fd_chunk(struct output_file *out, unsigned int len, int64_t aligned_offset; int aligned_diff; int buffer_size; + char *ptr; aligned_offset = offset & ~(4096 - 1); aligned_diff = offset - aligned_offset; @@ -529,15 +552,25 @@ int write_fd_chunk(struct output_file *out, unsigned int len, if (data == MAP_FAILED) { return -errno; } + ptr = data + aligned_diff; #else - char *data = malloc(buffer_size); + off64_t pos; + char *data = malloc(len); if (!data) { return -errno; } - memset(data, 0, buffer_size); + pos = lseek64(fd, offset, SEEK_SET); + if (pos < 0) { + return -errno; + } + ret = read_all(fd, data, len); + if (ret < 0) { + return ret; + } + ptr = data; #endif - ret = out->sparse_ops->write_data_chunk(out, len, data + aligned_diff); + ret = out->sparse_ops->write_data_chunk(out, len, ptr); #ifndef USE_MINGW munmap(data, buffer_size); diff --git a/libsparse/output_file.h b/libsparse/output_file.h index d23abf381..b86528b44 100644 --- a/libsparse/output_file.h +++ b/libsparse/output_file.h @@ -36,4 +36,6 @@ int write_fd_chunk(struct output_file *out, unsigned int len, int write_skip_chunk(struct output_file *out, int64_t len); void close_output_file(struct output_file *out); +int read_all(int fd, void *buf, size_t len); + #endif From 0c4c47f88dfc15cada154a1cf9b4db88b49890f0 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 25 Apr 2012 19:02:58 -0700 Subject: [PATCH 08/16] libsparse: add sparse_file read and convert tools to use it Abstract the logic from simg2img into libsparse, and add logic for reading a regular image into libsparse. simg2img then becomes a simple wrapper around libsparse. img2simg was not actually making the file sparse, it was using sparse files to create multiple files that could be pieced back together. Replace it with a simple wrapper around libsparse. Its functionality will be replaced by an simg2simg that can resparse a file into smaller chunks. Change-Id: I266f70e1c750454183ce46c71a7bb66bbb033a26 --- libsparse/Android.mk | 7 +- libsparse/img2simg.c | 367 +++++---------------- libsparse/include/sparse/sparse.h | 49 +++ libsparse/simg2img.c | 258 +-------------- libsparse/sparse_read.c | 509 ++++++++++++++++++++++++++++++ 5 files changed, 649 insertions(+), 541 deletions(-) create mode 100644 libsparse/sparse_read.c diff --git a/libsparse/Android.mk b/libsparse/Android.mk index d3eeae651..e83ee1cb0 100644 --- a/libsparse/Android.mk +++ b/libsparse/Android.mk @@ -7,7 +7,8 @@ libsparse_src_files := \ output_file.c \ sparse.c \ sparse_crc32.c \ - sparse_err.c + sparse_err.c \ + sparse_read.c include $(CLEAR_VARS) @@ -48,6 +49,7 @@ LOCAL_SRC_FILES := simg2img.c \ sparse_crc32.c LOCAL_MODULE := simg2img LOCAL_MODULE_TAGS := debug +LOCAL_STATIC_LIBRARIES := libsparse libz include $(BUILD_HOST_EXECUTABLE) @@ -57,6 +59,7 @@ LOCAL_SRC_FILES := simg2img.c \ sparse_crc32.c LOCAL_MODULE := simg2img LOCAL_MODULE_TAGS := optional +LOCAL_STATIC_LIBRARIES := libsparse libz include $(BUILD_EXECUTABLE) @@ -65,6 +68,7 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES := img2simg.c LOCAL_MODULE := img2simg LOCAL_MODULE_TAGS := debug +LOCAL_STATIC_LIBRARIES := libsparse libz include $(BUILD_HOST_EXECUTABLE) @@ -73,6 +77,7 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES := img2simg.c LOCAL_MODULE := img2simg LOCAL_MODULE_TAGS := optional +LOCAL_STATIC_LIBRARIES := libsparse libz include $(BUILD_EXECUTABLE) diff --git a/libsparse/img2simg.c b/libsparse/img2simg.c index a1594df63..6b1caa506 100644 --- a/libsparse/img2simg.c +++ b/libsparse/img2simg.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2012 The Android Open Source Project + * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,314 +14,103 @@ * limitations under the License. */ -#define DEFAULT_BLOCK_SIZE "4K" -#define DEFAULT_CHUNK_SIZE "64M" -#define DEFAULT_SUFFIX "%03d" +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 -#include "sparse_format.h" -#if 0 /* endian.h is not on all platforms */ -# include -#else - /* For now, just assume we're going to run on little-endian. */ -# define my_htole32(h) (h) -# define my_htole16(h) (h) -#endif -#include #include -#include -#include -#include +#include #include #include #include -#include +#include #include #include +#include -#define COPY_BUF_SIZE (1024*1024) -static char *copy_buf; +#include -static const char *progname(const char *argv0) +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define off64_t off_t +#endif + +void usage() { - const char *prog_name; - if ((prog_name = strrchr(argv0, '/'))) - return(prog_name + 1); /* Advance beyond '/'. */ - return(argv0); /* No '/' in argv0, use it as is. */ -} - -static void error_exit(const char *fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - fputc('\n', stderr); - va_end(ap); - - exit(EXIT_FAILURE); -} - -static void usage(const char *argv0, const char *error_fmt, ...) -{ - fprintf(stderr, - "Usage: %s [OPTIONS] \n", - progname(argv0)); - fprintf(stderr, "The will be split into as many sparse\n"); - fprintf(stderr, "files as needed. Each sparse file will contain a single\n"); - fprintf(stderr, "DONT CARE chunk to offset to the correct block and then\n"); - fprintf(stderr, "a single RAW chunk containing a portion of the data from\n"); - fprintf(stderr, "the raw image file. The sparse files will be named by\n"); - fprintf(stderr, "appending a number to the name of the raw image file.\n"); - fprintf(stderr, "\n"); - fprintf(stderr, "OPTIONS (Defaults are enclosed by square brackets):\n"); - fprintf(stderr, " -s SUFFIX Format appended number with SUFFIX [%s]\n", - DEFAULT_SUFFIX); - fprintf(stderr, " -B SIZE Use a block size of SIZE [%s]\n", - DEFAULT_BLOCK_SIZE); - fprintf(stderr, " -C SIZE Use a chunk size of SIZE [%s]\n", - DEFAULT_CHUNK_SIZE); - fprintf(stderr, "SIZE is a decimal integer that may optionally be\n"); - fprintf(stderr, "followed by a suffix that specifies a multiplier for\n"); - fprintf(stderr, "the integer:\n"); - fprintf(stderr, " c 1 byte (the default when omitted)\n"); - fprintf(stderr, " w 2 bytes\n"); - fprintf(stderr, " b 512 bytes\n"); - fprintf(stderr, " kB 1000 bytes\n"); - fprintf(stderr, " K 1024 bytes\n"); - fprintf(stderr, " MB 1000*1000 bytes\n"); - fprintf(stderr, " M 1024*1024 bytes\n"); - fprintf(stderr, " GB 1000*1000*1000 bytes\n"); - fprintf(stderr, " G 1024*1024*1024 bytes\n"); - - if (error_fmt && *error_fmt) - { - fprintf(stderr, "\n"); - va_list ap; - va_start(ap, error_fmt); - vfprintf(stderr, error_fmt, ap); - va_end(ap); - fprintf(stderr, "\n"); - } - - exit(EXIT_FAILURE); -} - -static void cpy_file(int out_fd, char *out_path, int in_fd, char *in_path, - size_t len) -{ - ssize_t s, cpy_len = COPY_BUF_SIZE; - - while (len) { - if (len < COPY_BUF_SIZE) - cpy_len = len; - - s = read(in_fd, copy_buf, cpy_len); - if (s < 0) - error_exit("\"%s\": %s", in_path, strerror(errno)); - if (!s) - error_exit("\"%s\": Unexpected EOF", in_path); - - cpy_len = s; - - s = write(out_fd, copy_buf, cpy_len); - if (s < 0) - error_exit("\"%s\": %s", out_path, strerror(errno)); - if (s != cpy_len) - error_exit("\"%s\": Short data write (%lu)", out_path, - (unsigned long)s); - - len -= cpy_len; - } -} - -static int parse_size(const char *size_str, size_t *size) -{ - static const size_t MAX_SIZE_T = ~(size_t)0; - size_t mult; - unsigned long long int value; - const char *end; - errno = 0; - value = strtoull(size_str, (char **)&end, 10); - if (errno != 0 || end == size_str || value > MAX_SIZE_T) - return -1; - if (*end == '\0') { - *size = value; - return 0; - } - if (!strcmp(end, "c")) - mult = 1; - else if (!strcmp(end, "w")) - mult = 2; - else if (!strcmp(end, "b")) - mult = 512; - else if (!strcmp(end, "kB")) - mult = 1000; - else if (!strcmp(end, "K")) - mult = 1024; - else if (!strcmp(end, "MB")) - mult = (size_t)1000*1000; - else if (!strcmp(end, "M")) - mult = (size_t)1024*1024; - else if (!strcmp(end, "GB")) - mult = (size_t)1000*1000*1000; - else if (!strcmp(end, "G")) - mult = (size_t)1024*1024*1024; - else - return -1; - - if (value > MAX_SIZE_T / mult) - return -1; - *size = value * mult; - return 0; + fprintf(stderr, "Usage: img2simg []\n"); } int main(int argc, char *argv[]) { - char *suffix = DEFAULT_SUFFIX; - char *block_size_str = DEFAULT_BLOCK_SIZE; - char *chunk_size_str = DEFAULT_CHUNK_SIZE; - size_t block_size, chunk_size, blocks_per_chunk, to_write; - char *in_path, *out_path, *out_fmt; - int in_fd, out_fd; - struct stat in_st; - off_t left_to_write; - struct { - sparse_header_t sparse_hdr; - chunk_header_t dont_care_hdr; - chunk_header_t raw_hdr; - } file_hdr; - unsigned int file_count; - ssize_t s; - int i; + int in; + int out; + unsigned int i; + int ret; + struct sparse_file *s; + unsigned int block_size = 4096; + off64_t len; - /* Parse the command line. */ - while ((i = getopt(argc, argv, "s:B:C:")) != -1) - { - switch (i) { - case 's': - suffix = optarg; - break; - case 'B': - block_size_str = optarg; - break; - case 'C': - chunk_size_str = optarg; - break; - default: - usage(argv[0], NULL); - break; - } - } - - if (parse_size(block_size_str, &block_size)) - usage(argv[0], "Can not parse \"%s\" as a block size.", - block_size_str); - if (block_size % 4096) - usage(argv[0], "Block size is not a multiple of 4096."); - - if (parse_size(chunk_size_str, &chunk_size)) - usage(argv[0], "Can not parse \"%s\" as a chunk size.", - chunk_size_str); - if (chunk_size % block_size) - usage(argv[0], "Chunk size is not a multiple of the block size."); - blocks_per_chunk = chunk_size / block_size; - - if ((argc - optind) != 1) - usage(argv[0], "Missing or extra arguments."); - in_path = argv[optind]; - - /* Open the input file and validate it. */ - if ((in_fd = open(in_path, O_RDONLY)) < 0) - error_exit("open \"%s\": %s", in_path, strerror(errno)); - if (fstat(in_fd, &in_st)) - error_exit("fstat \"%s\": %s", in_path, strerror(errno)); - left_to_write = in_st.st_size; - if (left_to_write % block_size) - error_exit( - "\"%s\" size (%llu) is not a multiple of the block size (%llu).\n", - in_path, - (unsigned long long)left_to_write, (unsigned long long)block_size); - - /* Get a buffer for copying the chunks. */ - if ((copy_buf = malloc(COPY_BUF_SIZE)) == 0) - error_exit("malloc copy buffer: %s", strerror(errno)); - - /* Get a buffer for a sprintf format to form output paths. */ - if ((out_fmt = malloc(sizeof("%s") + strlen(suffix))) == 0) - error_exit("malloc format buffer: %s", strerror(errno)); - out_fmt[0] = '%'; - out_fmt[1] = 's'; - strcpy(out_fmt + 2, suffix); - - /* Get a buffer for an output path. */ - i = snprintf(copy_buf, COPY_BUF_SIZE, out_fmt, in_path, UINT_MAX); - if (i >= COPY_BUF_SIZE) - error_exit("Ridulously long suffix: %s", suffix); - if ((out_path = malloc(i + 1)) == 0) - error_exit("malloc output path buffer: %s", strerror(errno)); - - /* - * Each file gets a sparse_header, a Don't Care chunk to offset to - * where the data belongs and then a Raw chunk with the actual data. - */ - memset((void *)&file_hdr.sparse_hdr, 0, sizeof(file_hdr.sparse_hdr)); - file_hdr.sparse_hdr.magic = my_htole32(SPARSE_HEADER_MAGIC); - file_hdr.sparse_hdr.major_version = my_htole16(1); - file_hdr.sparse_hdr.minor_version = my_htole16(0); - file_hdr.sparse_hdr.file_hdr_sz = my_htole16(sizeof(sparse_header_t)); - file_hdr.sparse_hdr.chunk_hdr_sz = my_htole16(sizeof(chunk_header_t)); - file_hdr.sparse_hdr.blk_sz = my_htole32(block_size); - /* The total_blks will be set in the file loop below. */ - file_hdr.sparse_hdr.total_chunks = my_htole32(2); - file_hdr.sparse_hdr.image_checksum = my_htole32(0); /* Typically unused. */ - - memset((void *)&file_hdr.dont_care_hdr, 0, sizeof(file_hdr.dont_care_hdr)); - file_hdr.dont_care_hdr.chunk_type = my_htole16(CHUNK_TYPE_DONT_CARE); - /* The Don't Care's chunk_sz will be set in the file loop below. */ - file_hdr.dont_care_hdr.total_sz = my_htole32(sizeof(chunk_header_t)); - - memset((void *)&file_hdr.raw_hdr, 0, sizeof(file_hdr.raw_hdr)); - file_hdr.raw_hdr.chunk_type = my_htole16(CHUNK_TYPE_RAW); - file_hdr.raw_hdr.chunk_sz = my_htole32(blocks_per_chunk); - file_hdr.raw_hdr.total_sz = my_htole32(chunk_size + sizeof(chunk_header_t)); - - /* Loop through writing chunk_size to each of the output files. */ - to_write = chunk_size; - for (file_count = 1; left_to_write ; file_count++) { - /* Fix up the headers on the last block. */ - if (left_to_write < (off_t)chunk_size) { - to_write = left_to_write; - file_hdr.raw_hdr.chunk_sz = my_htole32(left_to_write / block_size); - file_hdr.raw_hdr.total_sz = my_htole32(left_to_write - + sizeof(chunk_header_t)); + if (argc < 3 || argc > 4) { + usage(); + exit(-1); } - /* Form the pathname for this output file and open it. */ - sprintf(out_path, out_fmt, in_path, file_count); - if ((out_fd = creat(out_path, 0666)) < 0) - error_exit("\"%s\": %s", out_path, strerror(errno)); + if (argc == 4) { + block_size = atoi(argv[3]); + } - /* Update and write the headers to this output file. */ - s = (file_count-1) * blocks_per_chunk; - file_hdr.dont_care_hdr.chunk_sz = my_htole32(s); - file_hdr.sparse_hdr.total_blks = my_htole32(s - + (to_write / block_size)); - s = write(out_fd, (void *)&file_hdr, sizeof(file_hdr)); - if (s < 0) - error_exit("\"%s\": %s", out_path, strerror(errno)); - if (s != sizeof(file_hdr)) - error_exit("\"%s\": Short write (%lu)", out_path, (unsigned long)s); + if (block_size < 1024 || block_size % 4 != 0) { + usage(); + exit(-1); + } - /* Copy this chunk from the input file to the output file. */ - cpy_file(out_fd, out_path, in_fd, in_path, to_write); + if (strcmp(argv[1], "-") == 0) { + in = STDIN_FILENO; + } else { + in = open(argv[1], O_RDONLY | O_BINARY); + if (in < 0) { + fprintf(stderr, "Cannot open input file %s\n", argv[1]); + exit(-1); + } + } - /* Close this output file and update the amount left to write. */ - if (close(out_fd)) - error_exit("close \"%s\": %s", out_path, strerror(errno)); - left_to_write -= to_write; - } + if (strcmp(argv[2], "-") == 0) { + out = STDOUT_FILENO; + } else { + out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664); + if (out < 0) { + fprintf(stderr, "Cannot open output file %s\n", argv[2]); + exit(-1); + } + } - if (close(in_fd)) - error_exit("close \"%s\": %s", in_path, strerror(errno)); + len = lseek64(in, 0, SEEK_END); + lseek64(in, 0, SEEK_SET); - exit(EXIT_SUCCESS); + s = sparse_file_new(block_size, len); + if (!s) { + fprintf(stderr, "Failed to create sparse file\n"); + exit(-1); + } + + sparse_file_verbose(s); + ret = sparse_file_read(s, in, false, false); + if (ret) { + fprintf(stderr, "Failed to read file\n"); + exit(-1); + } + + ret = sparse_file_write(s, out, false, true, false); + if (ret) { + fprintf(stderr, "Failed to write sparse file\n"); + exit(-1); + } + + close(in); + close(out); + + exit(0); } diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h index 09a513725..ae5495599 100644 --- a/libsparse/include/sparse/sparse.h +++ b/libsparse/include/sparse/sparse.h @@ -157,6 +157,55 @@ int sparse_file_add_fd(struct sparse_file *s, int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, bool crc); +/** + * sparse_file_read - read a file into a sparse file cookie + * + * @s - sparse file cookie + * @fd - file descriptor to read from + * @sparse - read a file in the Android sparse file format + * @crc - verify the crc of a file in the Android sparse file format + * + * Reads a file into a sparse file cookie. If sparse is true, the file is + * assumed to be in the Android sparse file format. If sparse is false, the + * file will be sparsed by looking for block aligned chunks of all zeros or + * another 32 bit value. If crc is true, the crc of the sparse file will be + * verified. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc); + +/** + * sparse_file_import - import an existing sparse file + * + * @s - sparse file cookie + * @verbose - print verbose errors while reading the sparse file + * @crc - verify the crc of a file in the Android sparse file format + * + * Reads an existing sparse file into a sparse file cookie, recreating the same + * sparse cookie that was used to write it. If verbose is true, prints verbose + * errors when the sparse file is formatted incorrectly. + * + * Returns a new sparse file cookie on success, NULL on error. + */ +struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc); + +/** + * sparse_file_import_auto - import an existing sparse or normal file + * + * @fd - file descriptor to read from + * @crc - verify the crc of a file in the Android sparse file format + * + * Reads an existing sparse or normal file into a sparse file cookie. + * Attempts to determine if the file is sparse or not by looking for the sparse + * file magic number in the first 4 bytes. If the file is not sparse, the file + * will be sparsed by looking for block aligned chunks of all zeros or another + * 32 bit value. If crc is true, the crc of the sparse file will be verified. + * + * Returns a new sparse file cookie on success, NULL on error. + */ +struct sparse_file *sparse_file_import_auto(int fd, bool crc); + /** * sparse_file_verbose - set a sparse file cookie to print verbose errors * diff --git a/libsparse/simg2img.c b/libsparse/simg2img.c index 486b8054f..ab355838f 100644 --- a/libsparse/simg2img.c +++ b/libsparse/simg2img.c @@ -14,194 +14,36 @@ * limitations under the License. */ -#define _FILE_OFFSET_BITS 64 -#define _LARGEFILE64_SOURCE 1 -#include -#include - -#include "sparse_defs.h" -#include "sparse_format.h" -#include "sparse_crc32.h" +#include #include +#include #include #include #include #include #include #include -#include #include -#define COPY_BUF_SIZE (1024*1024) -u8 *copybuf; - -/* This will be malloc'ed with the size of blk_sz from the sparse file header */ -u8* zerobuf; - -#define SPARSE_HEADER_MAJOR_VER 1 -#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) -#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) - void usage() { fprintf(stderr, "Usage: simg2img \n"); } -static int read_all(int fd, void *buf, size_t len) -{ - size_t total = 0; - int ret; - char *ptr = buf; - - while (total < len) { - ret = read(fd, ptr, len - total); - - if (ret < 0) - return ret; - - if (ret == 0) - return total; - - ptr += ret; - total += ret; - } - - return total; -} - -static int write_all(int fd, void *buf, size_t len) -{ - size_t total = 0; - int ret; - char *ptr = buf; - - while (total < len) { - ret = write(fd, ptr, len - total); - - if (ret < 0) - return ret; - - if (ret == 0) - return total; - - ptr += ret; - total += ret; - } - - return total; -} - -int process_raw_chunk(int in, int out, u32 blocks, u32 blk_sz, u32 *crc32) -{ - u64 len = (u64)blocks * blk_sz; - int ret; - int chunk; - - while (len) { - chunk = (len > COPY_BUF_SIZE) ? COPY_BUF_SIZE : len; - ret = read_all(in, copybuf, chunk); - if (ret != chunk) { - fprintf(stderr, "read returned an error copying a raw chunk: %d %d\n", - ret, chunk); - exit(-1); - } - *crc32 = sparse_crc32(*crc32, copybuf, chunk); - ret = write_all(out, copybuf, chunk); - if (ret != chunk) { - fprintf(stderr, "write returned an error copying a raw chunk\n"); - exit(-1); - } - len -= chunk; - } - - return blocks; -} - - -int process_fill_chunk(int in, int out, u32 blocks, u32 blk_sz, u32 *crc32) -{ - u64 len = (u64)blocks * blk_sz; - int ret; - int chunk; - u32 fill_val; - u32 *fillbuf; - unsigned int i; - - /* Fill copy_buf with the fill value */ - ret = read_all(in, &fill_val, sizeof(fill_val)); - fillbuf = (u32 *)copybuf; - for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) { - fillbuf[i] = fill_val; - } - - while (len) { - chunk = (len > COPY_BUF_SIZE) ? COPY_BUF_SIZE : len; - *crc32 = sparse_crc32(*crc32, copybuf, chunk); - ret = write_all(out, copybuf, chunk); - if (ret != chunk) { - fprintf(stderr, "write returned an error copying a raw chunk\n"); - exit(-1); - } - len -= chunk; - } - - return blocks; -} - -int process_skip_chunk(int out, u32 blocks, u32 blk_sz, u32 *crc32) -{ - /* len needs to be 64 bits, as the sparse file specifies the skip amount - * as a 32 bit value of blocks. - */ - u64 len = (u64)blocks * blk_sz; - - lseek64(out, len, SEEK_CUR); - - return blocks; -} - -int process_crc32_chunk(int in, u32 crc32) -{ - u32 file_crc32; - int ret; - - ret = read_all(in, &file_crc32, 4); - if (ret != 4) { - fprintf(stderr, "read returned an error copying a crc32 chunk\n"); - exit(-1); - } - - if (file_crc32 != crc32) { - fprintf(stderr, "computed crc32 of 0x%8.8x, expected 0x%8.8x\n", - crc32, file_crc32); - exit(-1); - } - - return 0; -} - int main(int argc, char *argv[]) { int in; int out; unsigned int i; - sparse_header_t sparse_header; - chunk_header_t chunk_header; - u32 crc32 = 0; - u32 total_blocks = 0; int ret; + struct sparse_file *s; if (argc != 3) { usage(); exit(-1); } - if ( (copybuf = malloc(COPY_BUF_SIZE)) == 0) { - fprintf(stderr, "Cannot malloc copy buf\n"); - exit(-1); - } - if (strcmp(argv[1], "-") == 0) { in = STDIN_FILENO; } else { @@ -220,102 +62,16 @@ int main(int argc, char *argv[]) } } - ret = read_all(in, &sparse_header, sizeof(sparse_header)); - if (ret != sizeof(sparse_header)) { - fprintf(stderr, "Error reading sparse file header\n"); - exit(-1); - } - - if (sparse_header.magic != SPARSE_HEADER_MAGIC) { - fprintf(stderr, "Bad magic\n"); - exit(-1); - } - - if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) { - fprintf(stderr, "Unknown major version number\n"); - exit(-1); - } - - if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) { - /* Skip the remaining bytes in a header that is longer than - * we expected. - */ - lseek64(in, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR); - } - - if ( (zerobuf = malloc(sparse_header.blk_sz)) == 0) { - fprintf(stderr, "Cannot malloc zero buf\n"); - exit(-1); - } - - for (i=0; i CHUNK_HEADER_LEN) { - /* Skip the remaining bytes in a header that is longer than - * we expected. - */ - lseek64(in, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR); - } - - switch (chunk_header.chunk_type) { - case CHUNK_TYPE_RAW: - if (chunk_header.total_sz != (sparse_header.chunk_hdr_sz + - (chunk_header.chunk_sz * sparse_header.blk_sz)) ) { - fprintf(stderr, "Bogus chunk size for chunk %d, type Raw\n", i); - exit(-1); - } - total_blocks += process_raw_chunk(in, out, - chunk_header.chunk_sz, sparse_header.blk_sz, &crc32); - break; - case CHUNK_TYPE_FILL: - if (chunk_header.total_sz != (sparse_header.chunk_hdr_sz + sizeof(u32)) ) { - fprintf(stderr, "Bogus chunk size for chunk %d, type Fill\n", i); - exit(-1); - } - total_blocks += process_fill_chunk(in, out, - chunk_header.chunk_sz, sparse_header.blk_sz, &crc32); - break; - case CHUNK_TYPE_DONT_CARE: - if (chunk_header.total_sz != sparse_header.chunk_hdr_sz) { - fprintf(stderr, "Bogus chunk size for chunk %d, type Dont Care\n", i); - exit(-1); - } - total_blocks += process_skip_chunk(out, - chunk_header.chunk_sz, sparse_header.blk_sz, &crc32); - break; - case CHUNK_TYPE_CRC32: - process_crc32_chunk(in, crc32); - break; - default: - fprintf(stderr, "Unknown chunk type 0x%4.4x\n", chunk_header.chunk_type); - } - - } - - /* If the last chunk was a skip, then the code just did a seek, but - * no write, and the file won't actually be the correct size. This - * will make the file the correct size. Make sure the offset is - * computed in 64 bits, and the function called can handle 64 bits. - */ - if (ftruncate64(out, (u64)total_blocks * sparse_header.blk_sz)) { - fprintf(stderr, "Error calling ftruncate() to set the image size\n"); + s = sparse_file_import(in, true, false); + if (!s) { + fprintf(stderr, "Failed to read sparse file\n"); exit(-1); } + ret = sparse_file_write(s, out, false, false, false); close(in); close(out); - if (sparse_header.total_blks != total_blocks) { - fprintf(stderr, "Wrote %d blocks, expected to write %d blocks\n", - total_blocks, sparse_header.total_blks); - exit(-1); - } - exit(0); } diff --git a/libsparse/sparse_read.c b/libsparse/sparse_read.c new file mode 100644 index 000000000..704bcfadd --- /dev/null +++ b/libsparse/sparse_read.c @@ -0,0 +1,509 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "sparse_crc32.h" +#include "sparse_file.h" +#include "sparse_format.h" + +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define off64_t off_t +#endif + +#define SPARSE_HEADER_MAJOR_VER 1 +#define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) +#define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) + +#define COPY_BUF_SIZE (1024U*1024U) +static char *copybuf; + +#define min(a, b) \ + ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) + +static void verbose_error(bool verbose, int err, const char *fmt, ...) +{ + char *s = ""; + char *at = ""; + if (fmt) { + va_list argp; + int size; + + va_start(argp, fmt); + size = vsnprintf(NULL, 0, fmt, argp); + va_end(argp); + + if (size < 0) { + return; + } + + at = malloc(size + 1); + if (at == NULL) { + return; + } + + va_start(argp, fmt); + vsnprintf(at, size, fmt, argp); + va_end(argp); + at[size] = 0; + s = " at "; + } + if (verbose) { +#ifndef USE_MINGW + if (err == -EOVERFLOW) { + sparse_print_verbose("EOF while reading file%s%s\n", s, at); + } else +#endif + if (err == -EINVAL) { + sparse_print_verbose("Invalid sparse file format%s%s\n", s, at); + } else if (err == -ENOMEM) { + sparse_print_verbose("Failed allocation while reading file%s%s\n", + s, at); + } else { + sparse_print_verbose("Unknown error %d%s%s\n", err, s, at); + } + } + if (fmt) { + free(at); + } +} + +static int process_raw_chunk(struct sparse_file *s, unsigned int chunk_size, + int fd, int64_t offset, unsigned int blocks, unsigned int block, + uint32_t *crc32) +{ + int ret; + int chunk; + unsigned int len = blocks * s->block_size; + + if (chunk_size % s->block_size != 0) { + return -EINVAL; + } + + if (chunk_size / s->block_size != blocks) { + return -EINVAL; + } + + ret = sparse_file_add_fd(s, fd, offset, len, block); + if (ret < 0) { + return ret; + } + + if (crc32) { + while (len) { + chunk = min(len, COPY_BUF_SIZE); + ret = read_all(fd, copybuf, chunk); + if (ret < 0) { + return ret; + } + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + len -= chunk; + } + } else { + lseek64(fd, len, SEEK_CUR); + } + + return 0; +} + +static int process_fill_chunk(struct sparse_file *s, unsigned int chunk_size, + int fd, unsigned int blocks, unsigned int block, uint32_t *crc32) +{ + int ret; + int chunk; + int64_t len = (int64_t)blocks * s->block_size; + uint32_t fill_val; + uint32_t *fillbuf; + unsigned int i; + + if (chunk_size != sizeof(fill_val)) { + return -EINVAL; + } + + ret = read_all(fd, &fill_val, sizeof(fill_val)); + if (ret < 0) { + return ret; + } + + ret = sparse_file_add_fill(s, fill_val, len, block); + if (ret < 0) { + return ret; + } + + if (crc32) { + /* Fill copy_buf with the fill value */ + fillbuf = (uint32_t *)copybuf; + for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) { + fillbuf[i] = fill_val; + } + + while (len) { + chunk = min(len, COPY_BUF_SIZE); + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + len -= chunk; + } + } + + return 0; +} + +static int process_skip_chunk(struct sparse_file *s, unsigned int chunk_size, + int fd, unsigned int blocks, unsigned int block, uint32_t *crc32) +{ + int ret; + int chunk; + int64_t len = (int64_t)blocks * s->block_size; + uint32_t fill_val; + uint32_t *fillbuf; + unsigned int i; + + if (chunk_size != 0) { + return -EINVAL; + } + + if (crc32) { + memset(copybuf, 0, COPY_BUF_SIZE); + + while (len) { + chunk = min(len, COPY_BUF_SIZE); + *crc32 = sparse_crc32(*crc32, copybuf, chunk); + len -= chunk; + } + } + + return 0; +} + +static int process_crc32_chunk(int fd, unsigned int chunk_size, uint32_t crc32) +{ + uint32_t file_crc32; + int ret; + + if (chunk_size != sizeof(file_crc32)) { + return -EINVAL; + } + + ret = read_all(fd, &file_crc32, sizeof(file_crc32)); + if (ret < 0) { + return ret; + } + + if (file_crc32 != crc32) { + return -EINVAL; + } + + return 0; +} + +static int process_chunk(struct sparse_file *s, int fd, off64_t offset, + unsigned int chunk_hdr_sz, chunk_header_t *chunk_header, + unsigned int cur_block, uint32_t *crc_ptr) +{ + int ret; + unsigned int chunk_data_size; + + chunk_data_size = chunk_header->total_sz - chunk_hdr_sz; + + switch (chunk_header->chunk_type) { + case CHUNK_TYPE_RAW: + ret = process_raw_chunk(s, chunk_data_size, fd, offset, + chunk_header->chunk_sz, cur_block, crc_ptr); + if (ret < 0) { + verbose_error(s->verbose, ret, "data block at %lld", offset); + return ret; + } + return chunk_header->chunk_sz; + case CHUNK_TYPE_FILL: + ret = process_fill_chunk(s, chunk_data_size, fd, + chunk_header->chunk_sz, cur_block, crc_ptr); + if (ret < 0) { + verbose_error(s->verbose, ret, "fill block at %lld", offset); + return ret; + } + return chunk_header->chunk_sz; + case CHUNK_TYPE_DONT_CARE: + ret = process_skip_chunk(s, chunk_data_size, fd, + chunk_header->chunk_sz, cur_block, crc_ptr); + if (chunk_data_size != 0) { + if (ret < 0) { + verbose_error(s->verbose, ret, "skip block at %lld", offset); + return ret; + } + } + return chunk_header->chunk_sz; + case CHUNK_TYPE_CRC32: + ret = process_crc32_chunk(fd, chunk_data_size, *crc_ptr); + if (ret < 0) { + verbose_error(s->verbose, -EINVAL, "crc block at %lld", + offset); + return ret; + } + return 0; + default: + verbose_error(s->verbose, -EINVAL, "unknown block %04X at %lld", + chunk_header->chunk_type, offset); + } + + return 0; +} + +static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc) +{ + int ret; + unsigned int i; + sparse_header_t sparse_header; + chunk_header_t chunk_header; + uint32_t crc32 = 0; + uint32_t *crc_ptr = 0; + unsigned int cur_block = 0; + off64_t offset; + + if (!copybuf) { + copybuf = malloc(COPY_BUF_SIZE); + } + + if (!copybuf) { + return -ENOMEM; + } + + if (crc) { + crc_ptr = &crc32; + } + + ret = read_all(fd, &sparse_header, sizeof(sparse_header)); + if (ret < 0) { + return ret; + } + + if (sparse_header.magic != SPARSE_HEADER_MAGIC) { + return -EINVAL; + } + + if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) { + return -EINVAL; + } + + if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) { + return -EINVAL; + } + + if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) { + return -EINVAL; + } + + if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) { + /* Skip the remaining bytes in a header that is longer than + * we expected. + */ + lseek64(fd, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR); + } + + for (i = 0; i < sparse_header.total_chunks; i++) { + ret = read_all(fd, &chunk_header, sizeof(chunk_header)); + if (ret < 0) { + return ret; + } + + if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) { + /* Skip the remaining bytes in a header that is longer than + * we expected. + */ + lseek64(fd, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR); + } + + offset = lseek64(fd, 0, SEEK_CUR); + + ret = process_chunk(s, fd, offset, sparse_header.chunk_hdr_sz, &chunk_header, + cur_block, crc_ptr); + if (ret < 0) { + return ret; + } + + cur_block += ret; + } + + if (sparse_header.total_blks != cur_block) { + return -EINVAL; + } + + return 0; +} + +static int sparse_file_read_normal(struct sparse_file *s, int fd) +{ + int ret; + uint32_t *buf = malloc(s->block_size); + unsigned int block = 0; + int64_t remain = s->len; + int64_t offset = 0; + unsigned int to_read; + char *ptr; + unsigned int i; + bool sparse_block; + + if (!buf) { + return -ENOMEM; + } + + while (remain > 0) { + to_read = min(remain, s->block_size); + ret = read_all(fd, buf, to_read); + if (ret < 0) { + error("failed to read sparse file"); + return ret; + } + + if (to_read == s->block_size) { + sparse_block = true; + for (i = 1; i < s->block_size / sizeof(uint32_t); i++) { + if (buf[0] != buf[i]) { + sparse_block = false; + break; + } + } + } else { + sparse_block = false; + } + + if (sparse_block) { + /* TODO: add flag to use skip instead of fill for buf[0] == 0 */ + sparse_file_add_fill(s, buf[0], to_read, block); + } else { + sparse_file_add_fd(s, fd, offset, to_read, block); + } + + remain -= to_read; + offset += to_read; + block++; + } + + return 0; +} + +int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc) +{ + if (crc && !sparse) { + return -EINVAL; + } + + if (sparse) { + return sparse_file_read_sparse(s, fd, crc); + } else { + return sparse_file_read_normal(s, fd); + } +} + +struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc) +{ + int ret; + sparse_header_t sparse_header; + int64_t len; + struct sparse_file *s; + + ret = read_all(fd, &sparse_header, sizeof(sparse_header)); + if (ret < 0) { + verbose_error(verbose, ret, "header"); + return NULL; + } + + if (sparse_header.magic != SPARSE_HEADER_MAGIC) { + verbose_error(verbose, -EINVAL, "header magic"); + return NULL; + } + + if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) { + verbose_error(verbose, -EINVAL, "header major version"); + return NULL; + } + + if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) { + return NULL; + } + + if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) { + return NULL; + } + + len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz; + s = sparse_file_new(sparse_header.blk_sz, len); + if (!s) { + verbose_error(verbose, -EINVAL, NULL); + return NULL; + } + + ret = lseek64(fd, 0, SEEK_SET); + if (ret < 0) { + verbose_error(verbose, ret, "seeking"); + sparse_file_destroy(s); + return NULL; + } + + s->verbose = verbose; + + ret = sparse_file_read(s, fd, true, crc); + if (ret < 0) { + sparse_file_destroy(s); + return NULL; + } + + return s; +} + +struct sparse_file *sparse_file_import_auto(int fd, bool crc) +{ + struct sparse_file *s; + int64_t len; + int ret; + + s = sparse_file_import(fd, true, crc); + if (s) { + return s; + } + + len = lseek64(fd, 0, SEEK_END); + if (len < 0) { + return NULL; + } + + lseek64(fd, 0, SEEK_SET); + + s = sparse_file_new(4096, len); + if (!s) { + return NULL; + } + + ret = sparse_file_read_normal(s, fd); + if (ret < 0) { + sparse_file_destroy(s); + return NULL; + } + + return s; +} From b4cd267db30c152245e6308598e0066d87c5c55d Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 18 May 2012 14:49:50 -0700 Subject: [PATCH 09/16] libsparse: pseudo-subclass output_file for normal and gz files Create two subclasses of output_file that can handle normal and gzipped files, and refactor open_output_fd. Will allow adding support for an output_file type that is not file backed. Change-Id: I26744c74d13f205cf17df1ea9caac1eea9c57357 --- libsparse/output_file.c | 244 ++++++++++++++++++++++++++-------------- libsparse/output_file.h | 3 - 2 files changed, 162 insertions(+), 85 deletions(-) diff --git a/libsparse/output_file.c b/libsparse/output_file.c index 5e8a68c54..14b057a9e 100644 --- a/libsparse/output_file.c +++ b/libsparse/output_file.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -33,6 +34,8 @@ #ifndef USE_MINGW #include #define O_BINARY 0 +#else +#define ftruncate64 ftruncate #endif #if defined(__APPLE__) && defined(__MACH__) @@ -59,8 +62,13 @@ static inline void *mmap64(void *addr, size_t length, int prot, int flags, #define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) #define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) +#define container_of(inner, outer_t, elem) \ + ((outer_t *)((char *)inner - offsetof(outer_t, elem))) + struct output_file_ops { + int (*open)(struct output_file *, int fd); int (*skip)(struct output_file *, int64_t); + int (*pad)(struct output_file *, int64_t); int (*write)(struct output_file *, void *, int); void (*close)(struct output_file *); }; @@ -75,9 +83,6 @@ struct sparse_file_ops { }; struct output_file { - int fd; - gzFile gz_fd; - bool close_fd; int64_t cur_out_ptr; unsigned int chunk_cnt; uint32_t crc32; @@ -88,13 +93,39 @@ struct output_file { int64_t len; char *zero_buf; uint32_t *fill_buf; + char *buf; }; +struct output_file_gz { + struct output_file out; + gzFile gz_fd; +}; + +#define to_output_file_gz(_o) \ + container_of((_o), struct output_file_gz, out) + +struct output_file_normal { + struct output_file out; + int fd; +}; + +#define to_output_file_normal(_o) \ + container_of((_o), struct output_file_normal, out) + +static int file_open(struct output_file *out, int fd) +{ + struct output_file_normal *outn = to_output_file_normal(out); + + outn->fd = fd; + return 0; +} + static int file_skip(struct output_file *out, int64_t cnt) { off64_t ret; + struct output_file_normal *outn = to_output_file_normal(out); - ret = lseek64(out->fd, cnt, SEEK_CUR); + ret = lseek64(outn->fd, cnt, SEEK_CUR); if (ret < 0) { error_errno("lseek64"); return -1; @@ -102,10 +133,25 @@ static int file_skip(struct output_file *out, int64_t cnt) return 0; } +static int file_pad(struct output_file *out, int64_t len) +{ + int ret; + struct output_file_normal *outn = to_output_file_normal(out); + + ret = ftruncate64(outn->fd, len); + if (ret < 0) { + return -errno; + } + + return 0; +} + static int file_write(struct output_file *out, void *data, int len) { int ret; - ret = write(out->fd, data, len); + struct output_file_normal *outn = to_output_file_normal(out); + + ret = write(outn->fd, data, len); if (ret < 0) { error_errno("write"); return -1; @@ -119,22 +165,39 @@ static int file_write(struct output_file *out, void *data, int len) static void file_close(struct output_file *out) { - if (out->close_fd) { - close(out->fd); - } + struct output_file_normal *outn = to_output_file_normal(out); + + free(outn); } static struct output_file_ops file_ops = { + .open = file_open, .skip = file_skip, + .pad = file_pad, .write = file_write, .close = file_close, }; +static int gz_file_open(struct output_file *out, int fd) +{ + struct output_file_gz *outgz = to_output_file_gz(out); + + outgz->gz_fd = gzdopen(fd, "wb9"); + if (!outgz->gz_fd) { + error_errno("gzopen"); + return -errno; + } + + return 0; +} + + static int gz_file_skip(struct output_file *out, int64_t cnt) { off64_t ret; + struct output_file_gz *outgz = to_output_file_gz(out); - ret = gzseek(out->gz_fd, cnt, SEEK_CUR); + ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); if (ret < 0) { error_errno("gzseek"); return -1; @@ -142,10 +205,36 @@ static int gz_file_skip(struct output_file *out, int64_t cnt) return 0; } +static int gz_file_pad(struct output_file *out, int64_t len) +{ + off64_t ret; + struct output_file_gz *outgz = to_output_file_gz(out); + + ret = gztell(outgz->gz_fd); + if (ret < 0) { + return -1; + } + + if (ret >= len) { + return 0; + } + + ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); + if (ret < 0) { + return -1; + } + + gzwrite(outgz->gz_fd, "", 1); + + return 0; +} + static int gz_file_write(struct output_file *out, void *data, int len) { int ret; - ret = gzwrite(out->gz_fd, data, len); + struct output_file_gz *outgz = to_output_file_gz(out); + + ret = gzwrite(outgz->gz_fd, data, len); if (ret < 0) { error_errno("gzwrite"); return -1; @@ -159,11 +248,16 @@ static int gz_file_write(struct output_file *out, void *data, int len) static void gz_file_close(struct output_file *out) { - gzclose(out->gz_fd); + struct output_file_gz *outgz = to_output_file_gz(out); + + gzclose(outgz->gz_fd); + free(outgz); } static struct output_file_ops gz_file_ops = { + .open = gz_file_open, .skip = gz_file_skip, + .pad = gz_file_pad, .write = gz_file_write, .close = gz_file_close, }; @@ -371,21 +465,12 @@ static int write_normal_fill_chunk(struct output_file *out, unsigned int len, static int write_normal_skip_chunk(struct output_file *out, int64_t len) { - int ret; - return out->ops->skip(out, len); } int write_normal_end_chunk(struct output_file *out) { - int ret; - - ret = ftruncate64(out->fd, out->len); - if (ret < 0) { - return -errno; - } - - return 0; + return out->ops->pad(out, out->len); } static struct sparse_file_ops normal_file_ops = { @@ -401,59 +486,39 @@ void close_output_file(struct output_file *out) out->sparse_ops->write_end_chunk(out); out->ops->close(out); - free(out); } -struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, - int gz, int sparse, int chunks, int crc) +static int output_file_init(struct output_file *out, int block_size, + int64_t len, bool sparse, int chunks, bool crc) { int ret; - struct output_file *out = malloc(sizeof(struct output_file)); - if (!out) { - error_errno("malloc struct out"); - return NULL; - } + + out->len = len; + out->block_size = block_size; + out->cur_out_ptr = 0ll; + out->chunk_cnt = 0; + out->crc32 = 0; + out->use_crc = crc; + out->zero_buf = calloc(block_size, 1); if (!out->zero_buf) { error_errno("malloc zero_buf"); - goto err_zero_buf; + return -ENOMEM; } out->fill_buf = calloc(block_size, 1); if (!out->fill_buf) { error_errno("malloc fill_buf"); + ret = -ENOMEM; goto err_fill_buf; } - if (gz) { - out->ops = &gz_file_ops; - out->gz_fd = gzdopen(fd, "wb9"); - if (!out->gz_fd) { - error_errno("gzopen"); - goto err_gzopen; - } - } else { - out->fd = fd; - out->ops = &file_ops; - } - if (sparse) { out->sparse_ops = &sparse_file_ops; } else { out->sparse_ops = &normal_file_ops; } - out->close_fd = false; - out->cur_out_ptr = 0ll; - out->chunk_cnt = 0; - - /* Initialize the crc32 value */ - out->crc32 = 0; - out->use_crc = crc; - - out->len = len; - out->block_size = block_size; - if (sparse) { sparse_header_t sparse_header = { .magic = SPARSE_HEADER_MAGIC, @@ -477,47 +542,62 @@ struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, } } - return out; + return 0; err_write: - if (gz) { - gzclose(out->gz_fd); - } -err_gzopen: free(out->fill_buf); err_fill_buf: free(out->zero_buf); -err_zero_buf: - free(out); - return NULL; + return ret; } -struct output_file *open_output_file(const char *filename, - unsigned int block_size, int64_t len, - int gz, int sparse, int chunks, int crc) +static struct output_file *output_file_new_gz(void) { - int fd; - struct output_file *file; - - if (strcmp(filename, "-")) { - fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); - if (fd < 0) { - error_errno("open"); - return NULL; - } - } else { - fd = STDOUT_FILENO; - } - - file = open_output_fd(fd, block_size, len, gz, sparse, chunks, crc); - if (!file) { - close(fd); + struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz)); + if (!outgz) { + error_errno("malloc struct outgz"); return NULL; } - file->close_fd = true; // we opened descriptor thus we responsible for closing it + outgz->out.ops = &gz_file_ops; - return file; + return &outgz->out; +} + +static struct output_file *output_file_new_normal(void) +{ + struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal)); + if (!outn) { + error_errno("malloc struct outn"); + return NULL; + } + + outn->out.ops = &file_ops; + + return &outn->out; +} + +struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, + int gz, int sparse, int chunks, int crc) +{ + int ret; + struct output_file *out; + + if (gz) { + out = output_file_new_gz(); + } else { + out = output_file_new_normal(); + } + + out->ops->open(out, fd); + + ret = output_file_init(out, block_size, len, sparse, chunks, crc); + if (ret < 0) { + free(out); + return NULL; + } + + return out; } /* Write a contiguous region of data blocks from a memory buffer */ diff --git a/libsparse/output_file.h b/libsparse/output_file.h index b86528b44..24496f7d4 100644 --- a/libsparse/output_file.h +++ b/libsparse/output_file.h @@ -21,9 +21,6 @@ struct output_file; -struct output_file *open_output_file(const char *filename, - unsigned int block_size, int64_t len, - int gz, int sparse, int chunks, int crc); struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc); int write_data_chunk(struct output_file *out, unsigned int len, void *data); From 1e17b313a6257b7b5081e178e81435c09d60378e Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 21 May 2012 16:35:45 -0700 Subject: [PATCH 10/16] libsparse: add callback output file type Add a new output file subclass that will call a callback for each block as it is written. Will be used to measure the space used by each sparse block to allow resparsing files. Also add sparse_file_callback, which will write out a sparse file by calling the provided write function. Change-Id: I18707bd9c357b68da319cc07982e93d1c2b2bee2 --- libsparse/include/sparse/sparse.h | 21 +++++++ libsparse/output_file.c | 87 +++++++++++++++++++++++++++++ libsparse/output_file.h | 3 + libsparse/sparse.c | 92 +++++++++++++++++++++---------- 4 files changed, 175 insertions(+), 28 deletions(-) diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h index ae5495599..b2152270b 100644 --- a/libsparse/include/sparse/sparse.h +++ b/libsparse/include/sparse/sparse.h @@ -157,6 +157,27 @@ int sparse_file_add_fd(struct sparse_file *s, int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, bool crc); +/** + * sparse_file_callback - call a callback for blocks in sparse file + * + * @s - sparse file cookie + * @sparse - write in the Android sparse file format + * @crc - append a crc chunk + * @write - function to call for each block + * @priv - value that will be passed as the first argument to write + * + * Writes a sparse file by calling a callback function. If sparse is true, the + * file will be written in the Android sparse file format. If crc is true, the + * crc of the expanded data will be calculated and appended in a crc chunk. + * The callback 'write' will be called with data and length for each data, + * and with data==NULL to skip over a region (only used for non-sparse format). + * The callback should return negative on error, 0 on success. + * + * Returns 0 on success, negative errno on error. + */ +int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc, + int (*write)(void *priv, const void *data, int len), void *priv); + /** * sparse_file_read - read a file into a sparse file cookie * diff --git a/libsparse/output_file.c b/libsparse/output_file.c index 14b057a9e..dc56149e7 100644 --- a/libsparse/output_file.c +++ b/libsparse/output_file.c @@ -18,6 +18,7 @@ #define _LARGEFILE64_SOURCE 1 #include +#include #include #include #include @@ -112,6 +113,15 @@ struct output_file_normal { #define to_output_file_normal(_o) \ container_of((_o), struct output_file_normal, out) +struct output_file_callback { + struct output_file out; + void *priv; + int (*write)(void *priv, const void *buf, int len); +}; + +#define to_output_file_callback(_o) \ + container_of((_o), struct output_file_callback, out) + static int file_open(struct output_file *out, int fd) { struct output_file_normal *outn = to_output_file_normal(out); @@ -262,6 +272,57 @@ static struct output_file_ops gz_file_ops = { .close = gz_file_close, }; +static int callback_file_open(struct output_file *out, int fd) +{ + return 0; +} + +static int callback_file_skip(struct output_file *out, int64_t off) +{ + struct output_file_callback *outc = to_output_file_callback(out); + int to_write; + int ret; + + while (off > 0) { + to_write = min(off, (int64_t)INT_MAX); + ret = outc->write(outc->priv, NULL, to_write); + if (ret < 0) { + return ret; + } + off -= to_write; + } + + return 0; +} + +static int callback_file_pad(struct output_file *out, int64_t len) +{ + return -1; +} + +static int callback_file_write(struct output_file *out, void *data, int len) +{ + int ret; + struct output_file_callback *outc = to_output_file_callback(out); + + return outc->write(outc->priv, data, len); +} + +static void callback_file_close(struct output_file *out) +{ + struct output_file_callback *outc = to_output_file_callback(out); + + free(outc); +} + +static struct output_file_ops callback_file_ops = { + .open = callback_file_open, + .skip = callback_file_skip, + .pad = callback_file_pad, + .write = callback_file_write, + .close = callback_file_close, +}; + int read_all(int fd, void *buf, size_t len) { size_t total = 0; @@ -577,6 +638,32 @@ static struct output_file *output_file_new_normal(void) return &outn->out; } +struct output_file *open_output_callback(int (*write)(void *, const void *, int), + void *priv, unsigned int block_size, int64_t len, int gz, int sparse, + int chunks, int crc) +{ + int ret; + struct output_file_callback *outc; + + outc = calloc(1, sizeof(struct output_file_callback)); + if (!outc) { + error_errno("malloc struct outc"); + return NULL; + } + + outc->out.ops = &callback_file_ops; + outc->priv = priv; + outc->write = write; + + ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc); + if (ret < 0) { + free(outc); + return NULL; + } + + return &outc->out; +} + struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc) { diff --git a/libsparse/output_file.h b/libsparse/output_file.h index 24496f7d4..7a9fa2434 100644 --- a/libsparse/output_file.h +++ b/libsparse/output_file.h @@ -23,6 +23,9 @@ struct output_file; struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc); +struct output_file *open_output_callback(int (*write)(void *, const void *, int), + void *priv, unsigned int block_size, int64_t len, int gz, int sparse, + int chunks, int crc); int write_data_chunk(struct output_file *out, unsigned int len, void *data); int write_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val); diff --git a/libsparse/sparse.c b/libsparse/sparse.c index d778e1dc6..c560ec93c 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -99,20 +99,33 @@ unsigned int sparse_count_chunks(struct sparse_file *s) return chunks; } -int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, - bool crc) +static void sparse_file_write_block(struct output_file *out, + struct backed_block *bb) +{ + switch (backed_block_type(bb)) { + case BACKED_BLOCK_DATA: + write_data_chunk(out, backed_block_len(bb), backed_block_data(bb)); + break; + case BACKED_BLOCK_FILE: + write_file_chunk(out, backed_block_len(bb), + backed_block_filename(bb), backed_block_file_offset(bb)); + break; + case BACKED_BLOCK_FD: + write_fd_chunk(out, backed_block_len(bb), + backed_block_fd(bb), backed_block_file_offset(bb)); + break; + case BACKED_BLOCK_FILL: + write_fill_chunk(out, backed_block_len(bb), + backed_block_fill_val(bb)); + break; + } +} + +static int write_all_blocks(struct sparse_file *s, struct output_file *out) { struct backed_block *bb; unsigned int last_block = 0; int64_t pad; - int chunks; - struct output_file *out; - - chunks = sparse_count_chunks(s); - out = open_output_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc); - - if (!out) - return -ENOMEM; for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) { @@ -120,23 +133,7 @@ int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, unsigned int blocks = backed_block_block(bb) - last_block; write_skip_chunk(out, (int64_t)blocks * s->block_size); } - switch (backed_block_type(bb)) { - case BACKED_BLOCK_DATA: - write_data_chunk(out, backed_block_len(bb), backed_block_data(bb)); - break; - case BACKED_BLOCK_FILE: - write_file_chunk(out, backed_block_len(bb), - backed_block_filename(bb), backed_block_file_offset(bb)); - break; - case BACKED_BLOCK_FD: - write_fd_chunk(out, backed_block_len(bb), - backed_block_fd(bb), backed_block_file_offset(bb)); - break; - case BACKED_BLOCK_FILL: - write_fill_chunk(out, backed_block_len(bb), - backed_block_fill_val(bb)); - break; - } + sparse_file_write_block(out, bb); last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), s->block_size); } @@ -147,9 +144,48 @@ int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, write_skip_chunk(out, pad); } + return 0; +} + +int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, + bool crc) +{ + int ret; + int chunks; + struct output_file *out; + + chunks = sparse_count_chunks(s); + out = open_output_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc); + + if (!out) + return -ENOMEM; + + ret = write_all_blocks(s, out); + close_output_file(out); - return 0; + return ret; +} + +int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc, + int (*write)(void *priv, const void *data, int len), void *priv) +{ + int ret; + int chunks; + struct output_file *out; + + chunks = sparse_count_chunks(s); + out = open_output_callback(write, priv, s->block_size, s->len, false, + sparse, chunks, crc); + + if (!out) + return -ENOMEM; + + ret = write_all_blocks(s, out); + + close_output_file(out); + + return ret; } void sparse_file_verbose(struct sparse_file *s) From bdc6d39ed6c09199a5d806f29b71b44cbb27c5c2 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 2 May 2012 15:18:22 -0700 Subject: [PATCH 11/16] libsparse: add function to resparse a file and a utility to use it Add sparse_file_repsarse, which splits chunks in an existing sparse file such that the maximum size of a chunk, plus a header and footer, is smaller than the given size. This will allow multiple smaller sparse files to result in the same data as a large sparse file. Change-Id: I177abdb958a23d5afd394ff265c5b0c6a3ff22fa --- libsparse/Android.mk | 9 +++ libsparse/backed_block.c | 87 ++++++++++++++++++++++ libsparse/backed_block.h | 6 ++ libsparse/include/sparse/sparse.h | 14 ++++ libsparse/simg2img.c | 64 ++++++++++------- libsparse/simg2simg.c | 115 ++++++++++++++++++++++++++++++ libsparse/sparse.c | 99 +++++++++++++++++++++++++ libsparse/sparse_defs.h | 1 + 8 files changed, 369 insertions(+), 26 deletions(-) create mode 100644 libsparse/simg2simg.c diff --git a/libsparse/Android.mk b/libsparse/Android.mk index e83ee1cb0..69b52c387 100644 --- a/libsparse/Android.mk +++ b/libsparse/Android.mk @@ -83,6 +83,15 @@ include $(BUILD_EXECUTABLE) include $(CLEAR_VARS) +LOCAL_SRC_FILES := simg2simg.c +LOCAL_MODULE := simg2simg +LOCAL_MODULE_TAGS := debug +LOCAL_STATIC_LIBRARIES := libsparse libz + +include $(BUILD_HOST_EXECUTABLE) + +include $(CLEAR_VARS) + LOCAL_MODULE := simg_dump.py LOCAL_MODULE_TAGS := debug LOCAL_SRC_FILES := simg_dump.py diff --git a/libsparse/backed_block.c b/libsparse/backed_block.c index 629fc284f..dfb217b30 100644 --- a/libsparse/backed_block.c +++ b/libsparse/backed_block.c @@ -141,6 +141,52 @@ void backed_block_list_destroy(struct backed_block_list *bbl) free(bbl); } +void backed_block_list_move(struct backed_block_list *from, + struct backed_block_list *to, struct backed_block *start, + struct backed_block *end) +{ + struct backed_block *bb; + + if (start == NULL) { + start = from->data_blocks; + } + + if (!end) { + for (end = start; end && end->next; end = end->next) + ; + } + + if (start == NULL || end == NULL) { + return; + } + + from->last_used = NULL; + to->last_used = NULL; + if (from->data_blocks == start) { + from->data_blocks = end->next; + } else { + for (bb = from->data_blocks; bb; bb = bb->next) { + if (bb->next == start) { + bb->next = end->next; + break; + } + } + } + + if (!to->data_blocks) { + to->data_blocks = start; + end->next = NULL; + } else { + for (bb = to->data_blocks; bb; bb = bb->next) { + if (!bb->next || bb->next->block > start->block) { + end->next = bb->next; + bb->next = start; + break; + } + } + } +} + /* may free b */ static int merge_bb(struct backed_block_list *bbl, struct backed_block *a, struct backed_block *b) @@ -311,3 +357,44 @@ int backed_block_add_fd(struct backed_block_list *bbl, int fd, int64_t offset, return queue_bb(bbl, bb); } + +int backed_block_split(struct backed_block_list *bbl, struct backed_block *bb, + unsigned int max_len) +{ + struct backed_block *new_bb; + + max_len = ALIGN_DOWN(max_len, bbl->block_size); + + if (bb->len <= max_len) { + return 0; + } + + new_bb = malloc(sizeof(struct backed_block)); + if (bb == NULL) { + return -ENOMEM; + } + + *new_bb = *bb; + + new_bb->len = bb->len - max_len; + new_bb->block = bb->block + max_len / bbl->block_size; + new_bb->next = bb->next; + bb->next = new_bb; + bb->len = max_len; + + switch (bb->type) { + case BACKED_BLOCK_DATA: + new_bb->data.data = (char *)bb->data.data + max_len; + break; + case BACKED_BLOCK_FILE: + new_bb->file.offset += max_len; + break; + case BACKED_BLOCK_FD: + new_bb->fd.offset += max_len; + break; + case BACKED_BLOCK_FILL: + break; + } + + return 0; +} diff --git a/libsparse/backed_block.h b/libsparse/backed_block.h index 692691712..1a159be1e 100644 --- a/libsparse/backed_block.h +++ b/libsparse/backed_block.h @@ -48,6 +48,8 @@ int backed_block_fd(struct backed_block *bb); int64_t backed_block_file_offset(struct backed_block *bb); uint32_t backed_block_fill_val(struct backed_block *bb); enum backed_block_type backed_block_type(struct backed_block *bb); +int backed_block_split(struct backed_block_list *bbl, struct backed_block *bb, + unsigned int max_len); struct backed_block *backed_block_iter_new(struct backed_block_list *bbl); struct backed_block *backed_block_iter_next(struct backed_block *bb); @@ -55,4 +57,8 @@ struct backed_block *backed_block_iter_next(struct backed_block *bb); struct backed_block_list *backed_block_list_new(unsigned int block_size); void backed_block_list_destroy(struct backed_block_list *bbl); +void backed_block_list_move(struct backed_block_list *from, + struct backed_block_list *to, struct backed_block *start, + struct backed_block *end); + #endif diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h index b2152270b..fe003f61b 100644 --- a/libsparse/include/sparse/sparse.h +++ b/libsparse/include/sparse/sparse.h @@ -227,6 +227,20 @@ struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc); */ struct sparse_file *sparse_file_import_auto(int fd, bool crc); +/** sparse_file_resparse - rechunk an existing sparse file into smaller files + * + * @in_s - sparse file cookie of the existing sparse file + * @max_len - maximum file size + * @out_s - array of sparse file cookies + * @out_s_count - size of out_s array + * + * Splits chunks of an existing sparse file into smaller sparse files such that + * each sparse file is less than max_len. Returns the number of sparse_files + * that would have been written to out_s if out_s were big enough. + */ +int sparse_file_resparse(struct sparse_file *in_s, unsigned int max_len, + struct sparse_file **out_s, int out_s_count); + /** * sparse_file_verbose - set a sparse file cookie to print verbose errors * diff --git a/libsparse/simg2img.c b/libsparse/simg2img.c index ab355838f..95e9b5bea 100644 --- a/libsparse/simg2img.c +++ b/libsparse/simg2img.c @@ -26,50 +26,62 @@ #include #include +#ifndef O_BINARY +#define O_BINARY 0 +#endif + void usage() { - fprintf(stderr, "Usage: simg2img \n"); + fprintf(stderr, "Usage: simg2img \n"); } int main(int argc, char *argv[]) { int in; int out; - unsigned int i; + int i; int ret; struct sparse_file *s; - if (argc != 3) { + if (argc < 3) { usage(); exit(-1); } - if (strcmp(argv[1], "-") == 0) { - in = STDIN_FILENO; - } else { - if ((in = open(argv[1], O_RDONLY)) == 0) { - fprintf(stderr, "Cannot open input file %s\n", argv[1]); - exit(-1); - } - } - - if (strcmp(argv[2], "-") == 0) { - out = STDOUT_FILENO; - } else { - if ((out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, 0666)) == 0) { - fprintf(stderr, "Cannot open output file %s\n", argv[2]); - exit(-1); - } - } - - s = sparse_file_import(in, true, false); - if (!s) { - fprintf(stderr, "Failed to read sparse file\n"); + out = open(argv[argc - 1], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664); + if (out < 0) { + fprintf(stderr, "Cannot open output file %s\n", argv[argc - 1]); exit(-1); } - ret = sparse_file_write(s, out, false, false, false); - close(in); + for (i = 1; i < argc - 1; i++) { + if (strcmp(argv[i], "-") == 0) { + in = STDIN_FILENO; + } else { + in = open(argv[i], O_RDONLY | O_BINARY); + if (in < 0) { + fprintf(stderr, "Cannot open input file %s\n", argv[i]); + exit(-1); + } + } + + s = sparse_file_import(in, true, false); + if (!s) { + fprintf(stderr, "Failed to read sparse file\n"); + exit(-1); + } + + lseek(out, SEEK_SET, 0); + + ret = sparse_file_write(s, out, false, false, false); + if (ret < 0) { + fprintf(stderr, "Cannot write output file\n"); + exit(-1); + } + sparse_file_destroy(s); + close(in); + } + close(out); exit(0); diff --git a/libsparse/simg2simg.c b/libsparse/simg2simg.c new file mode 100644 index 000000000..5f9ccf678 --- /dev/null +++ b/libsparse/simg2simg.c @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE 1 +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +void usage() +{ + fprintf(stderr, "Usage: simg2simg \n"); +} + +int main(int argc, char *argv[]) +{ + int in; + int out; + int i; + int ret; + struct sparse_file *s; + int64_t max_size; + struct sparse_file **out_s; + int files; + char filename[4096]; + + if (argc != 4) { + usage(); + exit(-1); + } + + max_size = atoll(argv[3]); + + in = open(argv[1], O_RDONLY | O_BINARY); + if (in < 0) { + fprintf(stderr, "Cannot open input file %s\n", argv[1]); + exit(-1); + } + + s = sparse_file_import(in, true, false); + if (!s) { + fprintf(stderr, "Failed to import sparse file\n"); + exit(-1); + } + + files = sparse_file_resparse(s, max_size, NULL, 0); + if (files < 0) { + fprintf(stderr, "Failed to resparse\n"); + exit(-1); + } + + out_s = calloc(sizeof(struct sparse_file *), files); + if (!out_s) { + fprintf(stderr, "Failed to allocate sparse file array\n"); + exit(-1); + } + + files = sparse_file_resparse(s, max_size, out_s, files); + if (files < 0) { + fprintf(stderr, "Failed to resparse\n"); + exit(-1); + } + + for (i = 0; i < files; i++) { + ret = snprintf(filename, sizeof(filename), "%s.%d", argv[2], i); + if (ret >= (int)sizeof(filename)) { + fprintf(stderr, "Filename too long\n"); + exit(-1); + } + + out = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664); + if (out < 0) { + fprintf(stderr, "Cannot open output file %s\n", argv[2]); + exit(-1); + } + + ret = sparse_file_write(out_s[i], out, false, true, false); + if (ret) { + fprintf(stderr, "Failed to write sparse file\n"); + exit(-1); + } + close(out); + } + + close(in); + + exit(0); +} diff --git a/libsparse/sparse.c b/libsparse/sparse.c index c560ec93c..77f02fc49 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -24,6 +24,7 @@ #include "output_file.h" #include "backed_block.h" #include "sparse_defs.h" +#include "sparse_format.h" struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len) { @@ -188,6 +189,104 @@ int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc, return ret; } +static int out_counter_write(void *priv, const void *data, int len) +{ + int64_t *count = priv; + *count += len; + return 0; +} + +static struct backed_block *move_chunks_up_to_len(struct sparse_file *from, + struct sparse_file *to, unsigned int len) +{ + int64_t count = 0; + struct output_file *out_counter; + struct backed_block *last_bb = NULL; + struct backed_block *bb; + struct backed_block *start; + int64_t file_len = 0; + + /* + * overhead is sparse file header, initial skip chunk, split chunk, end + * skip chunk, and crc chunk. + */ + int overhead = sizeof(sparse_header_t) + 4 * sizeof(chunk_header_t) + + sizeof(uint32_t); + len -= overhead; + + start = backed_block_iter_new(from->backed_block_list); + out_counter = open_output_callback(out_counter_write, &count, + to->block_size, to->len, false, true, 0, false); + if (!out_counter) { + return NULL; + } + + for (bb = start; bb; bb = backed_block_iter_next(bb)) { + count = 0; + /* will call out_counter_write to update count */ + sparse_file_write_block(out_counter, bb); + if (file_len + count > len) { + /* + * If the remaining available size is more than 1/8th of the + * requested size, split the chunk. Results in sparse files that + * are at least 7/8ths of the requested size + */ + if (!last_bb || (len - file_len > (len / 8))) { + backed_block_split(from->backed_block_list, bb, len - file_len); + last_bb = bb; + } + goto out; + } + file_len += count; + last_bb = bb; + } + +out: + backed_block_list_move(from->backed_block_list, + to->backed_block_list, start, last_bb); + + close_output_file(out_counter); + + return bb; +} + +int sparse_file_resparse(struct sparse_file *in_s, unsigned int max_len, + struct sparse_file **out_s, int out_s_count) +{ + struct backed_block *bb; + unsigned int overhead; + struct sparse_file *s; + struct sparse_file *tmp; + int c = 0; + + tmp = sparse_file_new(in_s->block_size, in_s->len); + if (!tmp) { + return -ENOMEM; + } + + do { + s = sparse_file_new(in_s->block_size, in_s->len); + + bb = move_chunks_up_to_len(in_s, s, max_len); + + if (c < out_s_count) { + out_s[c] = s; + } else { + backed_block_list_move(s->backed_block_list, tmp->backed_block_list, + NULL, NULL); + sparse_file_destroy(s); + } + c++; + } while (bb); + + backed_block_list_move(tmp->backed_block_list, in_s->backed_block_list, + NULL, NULL); + + sparse_file_destroy(tmp); + + return c; +} + void sparse_file_verbose(struct sparse_file *s) { s->verbose = true; diff --git a/libsparse/sparse_defs.h b/libsparse/sparse_defs.h index 9f32d592e..b99cfd584 100644 --- a/libsparse/sparse_defs.h +++ b/libsparse/sparse_defs.h @@ -41,6 +41,7 @@ typedef unsigned char u8; #define DIV_ROUND_UP(x, y) (((x) + (y) - 1)/(y)) #define ALIGN(x, y) ((y) * DIV_ROUND_UP((x), (y))) +#define ALIGN_DOWN(x, y) ((y) * ((x) / (y))) #define error(fmt, args...) do { fprintf(stderr, "error: %s: " fmt "\n", __func__, ## args); } while (0) #define error_errno(s, args...) error(s ": %s", ##args, strerror(errno)) From 317a09e2d47257df5e0972c85f14c2a6ffdbbfd2 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 24 May 2012 17:15:43 -0700 Subject: [PATCH 12/16] libsparse: add sparse_file_len Add sparse_file_len, which will compute the size of data that would be produced if sparse_file_write was called. Useful combined with sparse_file_callback. Change-Id: I1a156d1071760f5559483954a5c62ffc20298703 --- libsparse/include/sparse/sparse.h | 14 ++++++++++++++ libsparse/sparse.c | 24 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h index fe003f61b..17d085ca4 100644 --- a/libsparse/include/sparse/sparse.h +++ b/libsparse/include/sparse/sparse.h @@ -157,6 +157,20 @@ int sparse_file_add_fd(struct sparse_file *s, int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, bool crc); +/** + * sparse_file_len - return the length of a sparse file if written to disk + * + * @s - sparse file cookie + * @sparse - write in the Android sparse file format + * @crc - append a crc chunk + * + * Returns the size a sparse file would be on disk if it were written in the + * specified format. If sparse is true, this is the size of the data in the + * sparse format. If sparse is false, this is the size of the normal + * non-sparse file. + */ +int64_t sparse_file_len(struct sparse_file *s, bool sparse, bool crc); + /** * sparse_file_callback - call a callback for blocks in sparse file * diff --git a/libsparse/sparse.c b/libsparse/sparse.c index 77f02fc49..f04f687b9 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -196,6 +196,30 @@ static int out_counter_write(void *priv, const void *data, int len) return 0; } +int64_t sparse_file_len(struct sparse_file *s, bool sparse, bool crc) +{ + int ret; + int chunks = sparse_count_chunks(s); + int64_t count = 0; + struct output_file *out; + + out = open_output_callback(out_counter_write, &count, + s->block_size, s->len, false, sparse, chunks, crc); + if (!out) { + return -1; + } + + ret = write_all_blocks(s, out); + + close_output_file(out); + + if (ret < 0) { + return -1; + } + + return count; +} + static struct backed_block *move_chunks_up_to_len(struct sparse_file *from, struct sparse_file *to, unsigned int len) { From b43828b247fd4f1e0373584de0504004c69eeac9 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 8 Jun 2012 16:55:35 -0700 Subject: [PATCH 13/16] libsparse: rename symbols that conflict with libext4_utils Until ext4_utils switches to using libsparse, libext4_utils defines some of the same symbols as libsparse. Fastboot links statically against both of them, and there is no easy way to make the symbols hidden, so just rename them in libsparse. Change-Id: Idc2cfe20efe3c3a7fb8233f453a89bbbeb0dcc8b --- libsparse/output_file.c | 6 +++--- libsparse/output_file.h | 6 +++--- libsparse/sparse.c | 16 ++++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/libsparse/output_file.c b/libsparse/output_file.c index dc56149e7..b5ae4191f 100644 --- a/libsparse/output_file.c +++ b/libsparse/output_file.c @@ -541,7 +541,7 @@ static struct sparse_file_ops normal_file_ops = { .write_end_chunk = write_normal_end_chunk, }; -void close_output_file(struct output_file *out) +void output_file_close(struct output_file *out) { int ret; @@ -638,7 +638,7 @@ static struct output_file *output_file_new_normal(void) return &outn->out; } -struct output_file *open_output_callback(int (*write)(void *, const void *, int), +struct output_file *output_file_open_callback(int (*write)(void *, const void *, int), void *priv, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc) { @@ -664,7 +664,7 @@ struct output_file *open_output_callback(int (*write)(void *, const void *, int) return &outc->out; } -struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, +struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc) { int ret; diff --git a/libsparse/output_file.h b/libsparse/output_file.h index 7a9fa2434..474c1fc21 100644 --- a/libsparse/output_file.h +++ b/libsparse/output_file.h @@ -21,9 +21,9 @@ struct output_file; -struct output_file *open_output_fd(int fd, unsigned int block_size, int64_t len, +struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc); -struct output_file *open_output_callback(int (*write)(void *, const void *, int), +struct output_file *output_file_open_callback(int (*write)(void *, const void *, int), void *priv, unsigned int block_size, int64_t len, int gz, int sparse, int chunks, int crc); int write_data_chunk(struct output_file *out, unsigned int len, void *data); @@ -34,7 +34,7 @@ int write_file_chunk(struct output_file *out, unsigned int len, int write_fd_chunk(struct output_file *out, unsigned int len, int fd, int64_t offset); int write_skip_chunk(struct output_file *out, int64_t len); -void close_output_file(struct output_file *out); +void output_file_close(struct output_file *out); int read_all(int fd, void *buf, size_t len); diff --git a/libsparse/sparse.c b/libsparse/sparse.c index f04f687b9..189b4c03e 100644 --- a/libsparse/sparse.c +++ b/libsparse/sparse.c @@ -156,14 +156,14 @@ int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, struct output_file *out; chunks = sparse_count_chunks(s); - out = open_output_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc); + out = output_file_open_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc); if (!out) return -ENOMEM; ret = write_all_blocks(s, out); - close_output_file(out); + output_file_close(out); return ret; } @@ -176,7 +176,7 @@ int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc, struct output_file *out; chunks = sparse_count_chunks(s); - out = open_output_callback(write, priv, s->block_size, s->len, false, + out = output_file_open_callback(write, priv, s->block_size, s->len, false, sparse, chunks, crc); if (!out) @@ -184,7 +184,7 @@ int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc, ret = write_all_blocks(s, out); - close_output_file(out); + output_file_close(out); return ret; } @@ -203,7 +203,7 @@ int64_t sparse_file_len(struct sparse_file *s, bool sparse, bool crc) int64_t count = 0; struct output_file *out; - out = open_output_callback(out_counter_write, &count, + out = output_file_open_callback(out_counter_write, &count, s->block_size, s->len, false, sparse, chunks, crc); if (!out) { return -1; @@ -211,7 +211,7 @@ int64_t sparse_file_len(struct sparse_file *s, bool sparse, bool crc) ret = write_all_blocks(s, out); - close_output_file(out); + output_file_close(out); if (ret < 0) { return -1; @@ -239,7 +239,7 @@ static struct backed_block *move_chunks_up_to_len(struct sparse_file *from, len -= overhead; start = backed_block_iter_new(from->backed_block_list); - out_counter = open_output_callback(out_counter_write, &count, + out_counter = output_file_open_callback(out_counter_write, &count, to->block_size, to->len, false, true, 0, false); if (!out_counter) { return NULL; @@ -269,7 +269,7 @@ out: backed_block_list_move(from->backed_block_list, to->backed_block_list, start, last_bb); - close_output_file(out_counter); + output_file_close(out_counter); return bb; } From 8879f988bac8d4cb46fb82e3d82ad69a9ed89b16 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 22 May 2012 17:53:34 -0700 Subject: [PATCH 14/16] fastboot: use getopt_long Modify the fastboot argument parsing to use getopt_long. This simplifies argument parsing, and permutes all the commands to the end of argv to allow parsing them later. Also moves usb initailization between argument and command processing, to allow commands to query parameters over usb. Change-Id: I883572f52c4190c80ee3b4aa4511ea2061a6b734 --- fastboot/fastboot.c | 108 +++++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 46 deletions(-) diff --git a/fastboot/fastboot.c b/fastboot/fastboot.c index 848cea377..2dc79e955 100644 --- a/fastboot/fastboot.c +++ b/fastboot/fastboot.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -575,62 +576,76 @@ int main(int argc, char **argv) unsigned sz; unsigned page_size = 2048; int status; + int c; - skip(1); - if (argc == 0) { + struct option longopts = { 0, 0, 0, 0 }; + + serial = getenv("ANDROID_SERIAL"); + + while (1) { + c = getopt_long(argc, argv, "wb:n:s:p:c:i:h", &longopts, NULL); + if (c < 0) { + break; + } + + switch (c) { + case 'w': + wants_wipe = 1; + break; + case 'b': + base_addr = strtoul(optarg, 0, 16); + break; + case 'n': + page_size = (unsigned)strtoul(optarg, NULL, 0); + if (!page_size) die("invalid page size"); + break; + case 's': + serial = optarg; + break; + case 'p': + product = optarg; + break; + case 'c': + cmdline = optarg; + break; + case 'i': { + char *endptr = NULL; + unsigned long val; + + val = strtoul(optarg, &endptr, 0); + if (!endptr || *endptr != '\0' || (val & ~0xffff)) + die("invalid vendor id '%s'", optarg); + vendor_id = (unsigned short)val; + break; + } + case 'h': + usage(); + return 1; + case '?': + return 1; + default: + abort(); + } + } + + argc -= optind; + argv += optind; + + if (argc == 0 && !wants_wipe) { usage(); return 1; } if (!strcmp(*argv, "devices")) { + skip(1); list_devices(); return 0; } - if (!strcmp(*argv, "help")) { - usage(); - return 0; - } - - - serial = getenv("ANDROID_SERIAL"); + usb = open_device(); while (argc > 0) { - if(!strcmp(*argv, "-w")) { - wants_wipe = 1; - skip(1); - } else if(!strcmp(*argv, "-b")) { - require(2); - base_addr = strtoul(argv[1], 0, 16); - skip(2); - } else if(!strcmp(*argv, "-n")) { - require(2); - page_size = (unsigned)strtoul(argv[1], NULL, 0); - if (!page_size) die("invalid page size"); - skip(2); - } else if(!strcmp(*argv, "-s")) { - require(2); - serial = argv[1]; - skip(2); - } else if(!strcmp(*argv, "-p")) { - require(2); - product = argv[1]; - skip(2); - } else if(!strcmp(*argv, "-c")) { - require(2); - cmdline = argv[1]; - skip(2); - } else if(!strcmp(*argv, "-i")) { - char *endptr = NULL; - unsigned long val; - - require(2); - val = strtoul(argv[1], &endptr, 0); - if (!endptr || *endptr != '\0' || (val & ~0xffff)) - die("invalid vendor id '%s'", argv[1]); - vendor_id = (unsigned short)val; - skip(2); - } else if(!strcmp(*argv, "getvar")) { + if(!strcmp(*argv, "getvar")) { require(2); fb_queue_display(argv[1], argv[1]); skip(2); @@ -719,6 +734,9 @@ int main(int argc, char **argv) wants_reboot = 1; } else if(!strcmp(*argv, "oem")) { argc = do_oem_command(argc, argv); + } else if (!strcmp(*argv, "help")) { + usage(); + return 0; } else { usage(); return 1; @@ -737,8 +755,6 @@ int main(int argc, char **argv) fb_queue_command("reboot-bootloader", "rebooting into bootloader"); } - usb = open_device(); - status = fb_execute_queue(usb); return (status) ? 1 : 0; } From 80f2d036a9dff894df27961c4aed300f1a5ebbc4 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 24 May 2012 18:24:53 -0700 Subject: [PATCH 15/16] fastboot: add fb_getvar Add an fb_getvar helper that can be used to get values from the target. Change-Id: I0da088fcbc8d40076c7bf5ef6e5bbd97fae61471 --- fastboot/engine.c | 22 ++++++++++++++++------ fastboot/fastboot.h | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/fastboot/engine.c b/fastboot/engine.c index 46b082806..7dc29e4c5 100644 --- a/fastboot/engine.c +++ b/fastboot/engine.c @@ -111,6 +111,20 @@ struct image_data { void generate_ext4_image(struct image_data *image); void cleanup_image(struct image_data *image); +int fb_getvar(struct usb_handle *usb, char *response, const char *fmt, ...) +{ + char cmd[CMD_SIZE] = "getvar:"; + int getvar_len = strlen(cmd); + va_list args; + + response[FB_RESPONSE_SZ] = '\0'; + va_start(args, fmt); + vsnprintf(cmd + getvar_len, sizeof(cmd) - getvar_len, fmt, args); + va_end(args); + cmd[CMD_SIZE - 1] = '\0'; + return fb_command_response(usb, cmd, response); +} + struct generator { char *fs_type; @@ -278,9 +292,7 @@ int fb_format(Action *a, usb_handle *usb, int skip_if_not_supported) unsigned i; char cmd[CMD_SIZE]; - response[FB_RESPONSE_SZ] = '\0'; - snprintf(cmd, sizeof(cmd), "getvar:partition-type:%s", partition); - status = fb_command_response(usb, cmd, response); + status = fb_getvar(usb, response, "partition-type:%s", partition); if (status) { if (skip_if_not_supported) { fprintf(stderr, @@ -312,9 +324,7 @@ int fb_format(Action *a, usb_handle *usb, int skip_if_not_supported) return -1; } - response[FB_RESPONSE_SZ] = '\0'; - snprintf(cmd, sizeof(cmd), "getvar:partition-size:%s", partition); - status = fb_command_response(usb, cmd, response); + status = fb_getvar(usb, response, "partition-size:%s", partition); if (status) { if (skip_if_not_supported) { fprintf(stderr, diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h index 1d3e2b824..a84b0be59 100644 --- a/fastboot/fastboot.h +++ b/fastboot/fastboot.h @@ -41,6 +41,7 @@ char *fb_get_error(void); #define FB_RESPONSE_SZ 64 /* engine.c - high level command queue engine */ +int fb_getvar(struct usb_handle *usb, char *response, const char *fmt, ...); void fb_queue_flash(const char *ptn, void *data, unsigned sz);; void fb_queue_erase(const char *ptn); void fb_queue_format(const char *ptn, int skip_if_not_supported); From f838788e6a4d57634a53eb597ee76a597feffcb5 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 24 May 2012 17:18:41 -0700 Subject: [PATCH 16/16] fastboot: add support for auto-resparsing large files Add support to fastboot for automatically using libsparse to break large files, whether they are in sparse or normal format, into multiple sparse files that can each fit into the target's memory. Allows flashing images that are larger than the size of the available memory on the target. By default, any file over 512MB will be sparsed into 512MB chunks. The limit can be modified with the -m argument, or sparsing can be forced with -S or avoided with -N. If -m is not specified, the target can override the default by implementing getvar:max-download-size Change-Id: I6c59381c3d24475c4f2587ea877200b96971cbd7 --- fastboot/Android.mk | 2 +- fastboot/engine.c | 19 +++- fastboot/fastboot.c | 197 +++++++++++++++++++++++++++++++++++++++- fastboot/fastboot.h | 4 + fastboot/protocol.c | 187 ++++++++++++++++++++++++++++++++------ fastboot/util_windows.c | 45 ++++++--- 6 files changed, 409 insertions(+), 45 deletions(-) diff --git a/fastboot/Android.mk b/fastboot/Android.mk index 089b9bb61..e3261a7f7 100644 --- a/fastboot/Android.mk +++ b/fastboot/Android.mk @@ -48,7 +48,7 @@ ifeq ($(HOST_OS),windows) LOCAL_C_INCLUDES += development/host/windows/usb/api endif -LOCAL_STATIC_LIBRARIES := $(EXTRA_STATIC_LIBS) libzipfile libunz libext4_utils libz +LOCAL_STATIC_LIBRARIES := $(EXTRA_STATIC_LIBS) libzipfile libunz libext4_utils libsparse libz ifneq ($(HOST_OS),windows) ifeq ($(HAVE_SELINUX), true) diff --git a/fastboot/engine.c b/fastboot/engine.c index 7dc29e4c5..93be3decc 100644 --- a/fastboot/engine.c +++ b/fastboot/engine.c @@ -28,7 +28,6 @@ #include "fastboot.h" #include "make_ext4fs.h" -#include "ext4_utils.h" #include #include @@ -77,6 +76,7 @@ char *mkmsg(const char *fmt, ...) #define OP_QUERY 3 #define OP_NOTICE 4 #define OP_FORMAT 5 +#define OP_DOWNLOAD_SPARSE 6 typedef struct Action Action; @@ -382,6 +382,19 @@ void fb_queue_flash(const char *ptn, void *data, unsigned sz) a->msg = mkmsg("writing '%s'", ptn); } +void fb_queue_flash_sparse(const char *ptn, struct sparse_file *s, unsigned sz) +{ + Action *a; + + a = queue_action(OP_DOWNLOAD_SPARSE, ""); + a->data = s; + a->size = 0; + a->msg = mkmsg("sending sparse '%s' (%d KB)", ptn, sz / 1024); + + a = queue_action(OP_COMMAND, "flash:%s", ptn); + a->msg = mkmsg("writing '%s'", ptn); +} + static int match(char *str, const char **value, unsigned count) { const char *val; @@ -580,6 +593,10 @@ int fb_execute_queue(usb_handle *usb) status = fb_format(a, usb, (int)a->data); status = a->func(a, status, status ? fb_get_error() : ""); if (status) break; + } else if (a->op == OP_DOWNLOAD_SPARSE) { + status = fb_download_data_sparse(usb, a->data); + status = a->func(a, status, status ? fb_get_error() : ""); + if (status) break; } else { die("bogus action"); } diff --git a/fastboot/fastboot.c b/fastboot/fastboot.c index 2dc79e955..ff9917300 100644 --- a/fastboot/fastboot.c +++ b/fastboot/fastboot.c @@ -26,9 +26,13 @@ * SUCH DAMAGE. */ +#define _LARGEFILE64_SOURCE + #include #include #include +#include +#include #include #include #include @@ -38,11 +42,20 @@ #include #include +#include + #include +#include #include #include "fastboot.h" +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#define DEFAULT_SPARSE_LIMIT (256 * 1024 * 1024) + char cur_product[FB_RESPONSE_SZ + 1]; void bootimg_set_cmdline(boot_img_hdr *h, const char *cmdline); @@ -59,6 +72,8 @@ static const char *product = 0; static const char *cmdline = 0; static int wipe_data = 0; static unsigned short vendor_id = 0; +static int64_t sparse_limit = -1; +static int64_t target_sparse_limit = -1; static unsigned base_addr = 0x10000000; @@ -117,7 +132,27 @@ char *find_item(const char *item, const char *product) #ifdef _WIN32 void *load_file(const char *fn, unsigned *_sz); +int64_t file_size(const char *fn); #else +#if defined(__APPLE__) && defined(__MACH__) +#define lseek64 lseek +#define off64_t off_t +#endif + +int64_t file_size(const char *fn) +{ + off64_t off; + int fd; + + fd = open(fn, O_RDONLY); + if (fd < 0) return -1; + + off = lseek64(fd, 0, SEEK_END); + close(fd); + + return off; +} + void *load_file(const char *fn, unsigned *_sz) { char *data; @@ -245,6 +280,8 @@ void usage(void) " -i specify a custom USB vendor id\n" " -b specify a custom kernel base address\n" " -n specify the nand page size. default: 2048\n" + " -S [K|M|G] automatically sparse files greater than\n" + " size. default: 256M, 0 to disable\n" ); } @@ -430,6 +467,110 @@ void queue_info_dump(void) fb_queue_notice("--------------------------------------------"); } + +struct sparse_file **load_sparse_files(const char *fname, int max_size) +{ + int fd; + struct sparse_file *s; + int files; + struct sparse_file **out_s; + + fd = open(fname, O_RDONLY | O_BINARY); + if (fd < 0) { + die("cannot open '%s'\n", fname); + } + + s = sparse_file_import_auto(fd, false); + if (!s) { + die("cannot sparse read file '%s'\n", fname); + } + + files = sparse_file_resparse(s, max_size, NULL, 0); + if (files < 0) { + die("Failed to resparse '%s'\n", fname); + } + + out_s = calloc(sizeof(struct sparse_file *), files + 1); + if (!out_s) { + die("Failed to allocate sparse file array\n"); + } + + files = sparse_file_resparse(s, max_size, out_s, files); + if (files < 0) { + die("Failed to resparse '%s'\n", fname); + } + + return out_s; +} + +static int64_t get_target_sparse_limit(struct usb_handle *usb) +{ + int64_t limit = 0; + char response[FB_RESPONSE_SZ + 1]; + int status = fb_getvar(usb, response, "max-download-size"); + + if (!status) { + limit = strtoul(response, NULL, 0); + if (limit > 0) { + fprintf(stderr, "target reported max download size of %lld bytes\n", + limit); + } + } + + return limit; +} + +static int64_t get_sparse_limit(struct usb_handle *usb, int64_t size) +{ + int64_t limit; + + if (sparse_limit == 0) { + return 0; + } else if (sparse_limit > 0) { + limit = sparse_limit; + } else { + if (target_sparse_limit == -1) { + target_sparse_limit = get_target_sparse_limit(usb); + } + if (target_sparse_limit > 0) { + limit = target_sparse_limit; + } else { + limit = DEFAULT_SPARSE_LIMIT; + } + } + + if (size > limit) { + return limit; + } + + return 0; +} + +void do_flash(usb_handle *usb, const char *pname, const char *fname) +{ + int64_t sz64; + void *data; + int64_t limit; + + sz64 = file_size(fname); + limit = get_sparse_limit(usb, sz64); + if (limit) { + struct sparse_file **s = load_sparse_files(fname, limit); + if (s == NULL) { + die("cannot sparse load '%s'\n", fname); + } + while (*s) { + sz64 = sparse_file_len(*s, true, false); + fb_queue_flash_sparse(pname, *s++, sz64); + } + } else { + unsigned int sz; + data = load_file(fname, &sz); + if (data == 0) die("cannot load '%s'\n", fname); + fb_queue_flash(pname, data, sz); + } +} + void do_update_signature(zipfile_t zip, char *fn) { void *data; @@ -567,6 +708,47 @@ int do_oem_command(int argc, char **argv) return 0; } +static int64_t parse_num(const char *arg) +{ + char *endptr; + unsigned long long num; + + num = strtoull(arg, &endptr, 0); + if (endptr == arg) { + return -1; + } + + if (*endptr == 'k' || *endptr == 'K') { + if (num >= (-1ULL) / 1024) { + return -1; + } + num *= 1024LL; + endptr++; + } else if (*endptr == 'm' || *endptr == 'M') { + if (num >= (-1ULL) / (1024 * 1024)) { + return -1; + } + num *= 1024LL * 1024LL; + endptr++; + } else if (*endptr == 'g' || *endptr == 'G') { + if (num >= (-1ULL) / (1024 * 1024 * 1024)) { + return -1; + } + num *= 1024LL * 1024LL * 1024LL; + endptr++; + } + + if (*endptr != '\0') { + return -1; + } + + if (num > INT64_MAX) { + return -1; + } + + return num; +} + int main(int argc, char **argv) { int wants_wipe = 0; @@ -577,13 +759,14 @@ int main(int argc, char **argv) unsigned page_size = 2048; int status; int c; + int r; - struct option longopts = { 0, 0, 0, 0 }; + const struct option longopts = { 0, 0, 0, 0 }; serial = getenv("ANDROID_SERIAL"); while (1) { - c = getopt_long(argc, argv, "wb:n:s:p:c:i:h", &longopts, NULL); + c = getopt_long(argc, argv, "wb:n:s:S:p:c:i:m:h", &longopts, NULL); if (c < 0) { break; } @@ -602,6 +785,12 @@ int main(int argc, char **argv) case 's': serial = optarg; break; + case 'S': + sparse_limit = parse_num(optarg); + if (sparse_limit < 0) { + die("invalid sparse limit"); + } + break; case 'p': product = optarg; break; @@ -702,9 +891,7 @@ int main(int argc, char **argv) skip(2); } if (fname == 0) die("cannot determine image filename for '%s'", pname); - data = load_file(fname, &sz); - if (data == 0) die("cannot load '%s'\n", fname); - fb_queue_flash(pname, data, sz); + do_flash(usb, pname, fname); } else if(!strcmp(*argv, "flash:raw")) { char *pname = argv[1]; char *kname = argv[2]; diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h index a84b0be59..9177932aa 100644 --- a/fastboot/fastboot.h +++ b/fastboot/fastboot.h @@ -31,10 +31,13 @@ #include "usb.h" +struct sparse_file; + /* protocol.c - fastboot protocol */ int fb_command(usb_handle *usb, const char *cmd); int fb_command_response(usb_handle *usb, const char *cmd, char *response); int fb_download_data(usb_handle *usb, const void *data, unsigned size); +int fb_download_data_sparse(usb_handle *usb, struct sparse_file *s); char *fb_get_error(void); #define FB_COMMAND_SZ 64 @@ -43,6 +46,7 @@ char *fb_get_error(void); /* engine.c - high level command queue engine */ int fb_getvar(struct usb_handle *usb, char *response, const char *fmt, ...); void fb_queue_flash(const char *ptn, void *data, unsigned sz);; +void fb_queue_flash_sparse(const char *ptn, struct sparse_file *s, unsigned sz); void fb_queue_erase(const char *ptn); void fb_queue_format(const char *ptn, int skip_if_not_supported); void fb_queue_require(const char *prod, const char *var, int invert, diff --git a/fastboot/protocol.c b/fastboot/protocol.c index e87111398..a0e0fd439 100644 --- a/fastboot/protocol.c +++ b/fastboot/protocol.c @@ -26,11 +26,18 @@ * SUCH DAMAGE. */ +#define min(a, b) \ + ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) +#define round_down(a, b) \ + ({ typeof(a) _a = (a); typeof(b) _b = (b); _a - (_a % _b); }) + #include #include #include #include +#include + #include "fastboot.h" static char ERROR[128]; @@ -40,8 +47,7 @@ char *fb_get_error(void) return ERROR; } -static int check_response(usb_handle *usb, unsigned size, - unsigned data_okay, char *response) +static int check_response(usb_handle *usb, unsigned int size, char *response) { unsigned char status[65]; int r; @@ -82,7 +88,7 @@ static int check_response(usb_handle *usb, unsigned size, return -1; } - if(!memcmp(status, "DATA", 4) && data_okay){ + if(!memcmp(status, "DATA", 4) && size > 0){ unsigned dsize = strtoul((char*) status + 4, 0, 16); if(dsize > size) { strcpy(ERROR, "data size too large"); @@ -100,9 +106,8 @@ static int check_response(usb_handle *usb, unsigned size, return -1; } -static int _command_send(usb_handle *usb, const char *cmd, - const void *data, unsigned size, - char *response) +static int _command_start(usb_handle *usb, const char *cmd, unsigned size, + char *response) { int cmdsize = strlen(cmd); int r; @@ -122,46 +127,81 @@ static int _command_send(usb_handle *usb, const char *cmd, return -1; } - if(data == 0) { - return check_response(usb, size, 0, response); + return check_response(usb, size, response); +} + +static int _command_data(usb_handle *usb, const void *data, unsigned size) +{ + int r; + + r = usb_write(usb, data, size); + if(r < 0) { + sprintf(ERROR, "data transfer failure (%s)", strerror(errno)); + usb_close(usb); + return -1; + } + if(r != ((int) size)) { + sprintf(ERROR, "data transfer failure (short transfer)"); + usb_close(usb); + return -1; } - r = check_response(usb, size, 1, 0); + return r; +} + +static int _command_end(usb_handle *usb) +{ + int r; + r = check_response(usb, 0, 0); if(r < 0) { return -1; } - size = r; + return 0; +} - if(size) { - r = usb_write(usb, data, size); - if(r < 0) { - sprintf(ERROR, "data transfer failure (%s)", strerror(errno)); - usb_close(usb); - return -1; - } - if(r != ((int) size)) { - sprintf(ERROR, "data transfer failure (short transfer)"); - usb_close(usb); - return -1; - } +static int _command_send(usb_handle *usb, const char *cmd, + const void *data, unsigned size, + char *response) +{ + int r; + if (size == 0) { + return -1; } - r = check_response(usb, 0, 0, 0); + r = _command_start(usb, cmd, size, response); + if (r < 0) { + return -1; + } + + r = _command_data(usb, data, size); + if (r < 0) { + return -1; + } + + r = _command_end(usb); if(r < 0) { return -1; - } else { - return size; } + + return size; +} + +static int _command_send_no_data(usb_handle *usb, const char *cmd, + char *response) +{ + int r; + + return _command_start(usb, cmd, 0, response); } int fb_command(usb_handle *usb, const char *cmd) { - return _command_send(usb, cmd, 0, 0, 0); + return _command_send_no_data(usb, cmd, 0); } int fb_command_response(usb_handle *usb, const char *cmd, char *response) { - return _command_send(usb, cmd, 0, 0, response); + return _command_send_no_data(usb, cmd, response); } int fb_download_data(usb_handle *usb, const void *data, unsigned size) @@ -179,3 +219,96 @@ int fb_download_data(usb_handle *usb, const void *data, unsigned size) } } +#define USB_BUF_SIZE 512 +static char usb_buf[USB_BUF_SIZE]; +static int usb_buf_len; + +static int fb_download_data_sparse_write(void *priv, const void *data, int len) +{ + int r; + usb_handle *usb = priv; + int to_write; + const char *ptr = data; + + if (usb_buf_len) { + to_write = min(USB_BUF_SIZE - usb_buf_len, len); + + memcpy(usb_buf + usb_buf_len, ptr, to_write); + usb_buf_len += to_write; + ptr += to_write; + len -= to_write; + } + + if (usb_buf_len == USB_BUF_SIZE) { + r = _command_data(usb, usb_buf, USB_BUF_SIZE); + if (r != USB_BUF_SIZE) { + return -1; + } + usb_buf_len = 0; + } + + if (len > USB_BUF_SIZE) { + if (usb_buf_len > 0) { + sprintf(ERROR, "internal error: usb_buf not empty\n"); + return -1; + } + to_write = round_down(len, USB_BUF_SIZE); + r = _command_data(usb, ptr, to_write); + if (r != to_write) { + return -1; + } + ptr += to_write; + len -= to_write; + } + + if (len > 0) { + if (len > USB_BUF_SIZE) { + sprintf(ERROR, "internal error: too much left for usb_buf\n"); + return -1; + } + memcpy(usb_buf, ptr, len); + usb_buf_len = len; + } + + return 0; +} + +static int fb_download_data_sparse_flush(usb_handle *usb) +{ + int r; + + if (usb_buf_len > 0) { + r = _command_data(usb, usb_buf, usb_buf_len); + if (r != usb_buf_len) { + return -1; + } + usb_buf_len = 0; + } + + return 0; +} + +int fb_download_data_sparse(usb_handle *usb, struct sparse_file *s) +{ + char cmd[64]; + int r; + int size = sparse_file_len(s, true, false); + if (size <= 0) { + return -1; + } + + sprintf(cmd, "download:%08x", size); + r = _command_start(usb, cmd, size, 0); + if (r < 0) { + return -1; + } + + r = sparse_file_callback(s, true, false, fb_download_data_sparse_write, usb); + if (r < 0) { + return -1; + } + + fb_download_data_sparse_flush(usb); + + return _command_end(usb); +} diff --git a/fastboot/util_windows.c b/fastboot/util_windows.c index c3d545cef..9e029fdb8 100644 --- a/fastboot/util_windows.c +++ b/fastboot/util_windows.c @@ -36,6 +36,29 @@ #include +int64_t file_size(const char *fn) +{ + HANDLE file; + char *data; + DWORD sz; + + file = CreateFile( fn, + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + 0, + NULL ); + + if (file == INVALID_HANDLE_VALUE) + return -1; + + sz = GetFileSize( file, NULL ); + CloseHandle( file ); + + return sz; +} + void get_my_path(char exe[PATH_MAX]) { char* r; @@ -52,7 +75,7 @@ void *load_file(const char *fn, unsigned *_sz) { HANDLE file; char *data; - DWORD file_size; + DWORD sz; file = CreateFile( fn, GENERIC_READ, @@ -65,29 +88,29 @@ void *load_file(const char *fn, unsigned *_sz) if (file == INVALID_HANDLE_VALUE) return NULL; - file_size = GetFileSize( file, NULL ); + sz = GetFileSize( file, NULL ); data = NULL; - if (file_size > 0) { - data = (char*) malloc( file_size ); + if (sz > 0) { + data = (char*) malloc( sz ); if (data == NULL) { - fprintf(stderr, "load_file: could not allocate %ld bytes\n", file_size ); - file_size = 0; + fprintf(stderr, "load_file: could not allocate %ld bytes\n", sz ); + sz = 0; } else { DWORD out_bytes; - if ( !ReadFile( file, data, file_size, &out_bytes, NULL ) || - out_bytes != file_size ) + if ( !ReadFile( file, data, sz, &out_bytes, NULL ) || + out_bytes != sz ) { - fprintf(stderr, "load_file: could not read %ld bytes from '%s'\n", file_size, fn); + fprintf(stderr, "load_file: could not read %ld bytes from '%s'\n", sz, fn); free(data); data = NULL; - file_size = 0; + sz = 0; } } } CloseHandle( file ); - *_sz = (unsigned) file_size; + *_sz = (unsigned) sz; return data; }