2 * Copyright (C) 2010 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #define _FILE_OFFSET_BITS 64
18 #define _LARGEFILE64_SOURCE 1
29 #include <sys/types.h>
34 #include "output_file.h"
35 #include "sparse_crc32.h"
36 #include "sparse_format.h"
38 #if defined(__APPLE__) && defined(__MACH__)
40 #define ftruncate64 ftruncate
46 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
48 #define SPARSE_HEADER_MAJOR_VER 1
49 #define SPARSE_HEADER_MINOR_VER 0
50 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
51 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
53 #define container_of(inner, outer_t, elem) \
54 ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
56 struct output_file_ops {
57 int (*open)(struct output_file *, int fd);
58 int (*skip)(struct output_file *, int64_t);
59 int (*pad)(struct output_file *, int64_t);
60 int (*write)(struct output_file *, void *, int);
61 void (*close)(struct output_file *);
64 struct sparse_file_ops {
65 int (*write_data_chunk)(struct output_file *out, unsigned int len,
67 int (*write_fill_chunk)(struct output_file *out, unsigned int len,
69 int (*write_skip_chunk)(struct output_file *out, int64_t len);
70 int (*write_end_chunk)(struct output_file *out);
75 unsigned int chunk_cnt;
77 struct output_file_ops *ops;
78 struct sparse_file_ops *sparse_ops;
80 unsigned int block_size;
87 struct output_file_gz {
88 struct output_file out;
92 #define to_output_file_gz(_o) \
93 container_of((_o), struct output_file_gz, out)
95 struct output_file_normal {
96 struct output_file out;
100 #define to_output_file_normal(_o) \
101 container_of((_o), struct output_file_normal, out)
103 struct output_file_callback {
104 struct output_file out;
106 int (*write)(void *priv, const void *buf, int len);
109 #define to_output_file_callback(_o) \
110 container_of((_o), struct output_file_callback, out)
112 static int file_open(struct output_file *out, int fd)
114 struct output_file_normal *outn = to_output_file_normal(out);
120 static int file_skip(struct output_file *out, int64_t cnt)
123 struct output_file_normal *outn = to_output_file_normal(out);
125 ret = lseek64(outn->fd, cnt, SEEK_CUR);
127 error_errno("lseek64");
133 static int file_pad(struct output_file *out, int64_t len)
136 struct output_file_normal *outn = to_output_file_normal(out);
138 ret = ftruncate64(outn->fd, len);
146 static int file_write(struct output_file *out, void *data, int len)
149 struct output_file_normal *outn = to_output_file_normal(out);
151 ret = write(outn->fd, data, len);
153 error_errno("write");
155 } else if (ret < len) {
156 error("incomplete write");
163 static void file_close(struct output_file *out)
165 struct output_file_normal *outn = to_output_file_normal(out);
170 static struct output_file_ops file_ops = {
178 static int gz_file_open(struct output_file *out, int fd)
180 struct output_file_gz *outgz = to_output_file_gz(out);
182 outgz->gz_fd = gzdopen(fd, "wb9");
184 error_errno("gzopen");
192 static int gz_file_skip(struct output_file *out, int64_t cnt)
195 struct output_file_gz *outgz = to_output_file_gz(out);
197 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
199 error_errno("gzseek");
205 static int gz_file_pad(struct output_file *out, int64_t len)
208 struct output_file_gz *outgz = to_output_file_gz(out);
210 ret = gztell(outgz->gz_fd);
219 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
224 gzwrite(outgz->gz_fd, "", 1);
229 static int gz_file_write(struct output_file *out, void *data, int len)
232 struct output_file_gz *outgz = to_output_file_gz(out);
234 ret = gzwrite(outgz->gz_fd, data, len);
236 error_errno("gzwrite");
238 } else if (ret < len) {
239 error("incomplete gzwrite");
246 static void gz_file_close(struct output_file *out)
248 struct output_file_gz *outgz = to_output_file_gz(out);
250 gzclose(outgz->gz_fd);
254 static struct output_file_ops gz_file_ops = {
255 .open = gz_file_open,
256 .skip = gz_file_skip,
258 .write = gz_file_write,
259 .close = gz_file_close,
262 static int callback_file_open(struct output_file *out __unused, int fd __unused)
267 static int callback_file_skip(struct output_file *out, int64_t off)
269 struct output_file_callback *outc = to_output_file_callback(out);
274 to_write = min(off, (int64_t)INT_MAX);
275 ret = outc->write(outc->priv, NULL, to_write);
285 static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
290 static int callback_file_write(struct output_file *out, void *data, int len)
292 struct output_file_callback *outc = to_output_file_callback(out);
294 return outc->write(outc->priv, data, len);
297 static void callback_file_close(struct output_file *out)
299 struct output_file_callback *outc = to_output_file_callback(out);
304 static struct output_file_ops callback_file_ops = {
305 .open = callback_file_open,
306 .skip = callback_file_skip,
307 .pad = callback_file_pad,
308 .write = callback_file_write,
309 .close = callback_file_close,
312 int read_all(int fd, void *buf, size_t len)
318 while (total < len) {
319 ret = read(fd, ptr, len - total);
334 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
336 chunk_header_t chunk_header;
339 if (skip_len % out->block_size) {
340 error("don't care size %"PRIi64" is not a multiple of the block size %u",
341 skip_len, out->block_size);
345 /* We are skipping data, so emit a don't care chunk. */
346 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
347 chunk_header.reserved1 = 0;
348 chunk_header.chunk_sz = skip_len / out->block_size;
349 chunk_header.total_sz = CHUNK_HEADER_LEN;
350 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
354 out->cur_out_ptr += skip_len;
360 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len,
363 chunk_header_t chunk_header;
364 int rnd_up_len, count;
367 /* Round up the fill length to a multiple of the block size */
368 rnd_up_len = ALIGN(len, out->block_size);
370 /* Finally we can safely emit a chunk of data */
371 chunk_header.chunk_type = CHUNK_TYPE_FILL;
372 chunk_header.reserved1 = 0;
373 chunk_header.chunk_sz = rnd_up_len / out->block_size;
374 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
375 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
379 ret = out->ops->write(out, &fill_val, sizeof(fill_val));
384 count = out->block_size / sizeof(uint32_t);
386 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
389 out->cur_out_ptr += rnd_up_len;
395 static int write_sparse_data_chunk(struct output_file *out, unsigned int len,
398 chunk_header_t chunk_header;
399 int rnd_up_len, zero_len;
402 /* Round up the data length to a multiple of the block size */
403 rnd_up_len = ALIGN(len, out->block_size);
404 zero_len = rnd_up_len - len;
406 /* Finally we can safely emit a chunk of data */
407 chunk_header.chunk_type = CHUNK_TYPE_RAW;
408 chunk_header.reserved1 = 0;
409 chunk_header.chunk_sz = rnd_up_len / out->block_size;
410 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
411 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
415 ret = out->ops->write(out, data, len);
419 ret = out->ops->write(out, out->zero_buf, zero_len);
425 out->crc32 = sparse_crc32(out->crc32, data, len);
427 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
430 out->cur_out_ptr += rnd_up_len;
436 int write_sparse_end_chunk(struct output_file *out)
438 chunk_header_t chunk_header;
442 chunk_header.chunk_type = CHUNK_TYPE_CRC32;
443 chunk_header.reserved1 = 0;
444 chunk_header.chunk_sz = 0;
445 chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
447 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
451 out->ops->write(out, &out->crc32, 4);
462 static struct sparse_file_ops sparse_file_ops = {
463 .write_data_chunk = write_sparse_data_chunk,
464 .write_fill_chunk = write_sparse_fill_chunk,
465 .write_skip_chunk = write_sparse_skip_chunk,
466 .write_end_chunk = write_sparse_end_chunk,
469 static int write_normal_data_chunk(struct output_file *out, unsigned int len,
473 unsigned int rnd_up_len = ALIGN(len, out->block_size);
475 ret = out->ops->write(out, data, len);
480 if (rnd_up_len > len) {
481 ret = out->ops->skip(out, rnd_up_len - len);
487 static int write_normal_fill_chunk(struct output_file *out, unsigned int len,
492 unsigned int write_len;
494 /* Initialize fill_buf with the fill_val */
495 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
496 out->fill_buf[i] = fill_val;
500 write_len = min(len, out->block_size);
501 ret = out->ops->write(out, out->fill_buf, write_len);
512 static int write_normal_skip_chunk(struct output_file *out, int64_t len)
514 return out->ops->skip(out, len);
517 int write_normal_end_chunk(struct output_file *out)
519 return out->ops->pad(out, out->len);
522 static struct sparse_file_ops normal_file_ops = {
523 .write_data_chunk = write_normal_data_chunk,
524 .write_fill_chunk = write_normal_fill_chunk,
525 .write_skip_chunk = write_normal_skip_chunk,
526 .write_end_chunk = write_normal_end_chunk,
529 void output_file_close(struct output_file *out)
531 out->sparse_ops->write_end_chunk(out);
532 out->ops->close(out);
535 static int output_file_init(struct output_file *out, int block_size,
536 int64_t len, bool sparse, int chunks, bool crc)
541 out->block_size = block_size;
542 out->cur_out_ptr = 0ll;
547 out->zero_buf = calloc(block_size, 1);
548 if (!out->zero_buf) {
549 error_errno("malloc zero_buf");
553 out->fill_buf = calloc(block_size, 1);
554 if (!out->fill_buf) {
555 error_errno("malloc fill_buf");
561 out->sparse_ops = &sparse_file_ops;
563 out->sparse_ops = &normal_file_ops;
567 sparse_header_t sparse_header = {
568 .magic = SPARSE_HEADER_MAGIC,
569 .major_version = SPARSE_HEADER_MAJOR_VER,
570 .minor_version = SPARSE_HEADER_MINOR_VER,
571 .file_hdr_sz = SPARSE_HEADER_LEN,
572 .chunk_hdr_sz = CHUNK_HEADER_LEN,
573 .blk_sz = out->block_size,
574 .total_blks = out->len / out->block_size,
575 .total_chunks = chunks,
580 sparse_header.total_chunks++;
583 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
598 static struct output_file *output_file_new_gz(void)
600 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
602 error_errno("malloc struct outgz");
606 outgz->out.ops = &gz_file_ops;
611 static struct output_file *output_file_new_normal(void)
613 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
615 error_errno("malloc struct outn");
619 outn->out.ops = &file_ops;
624 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int),
625 void *priv, unsigned int block_size, int64_t len,
626 int gz __unused, int sparse, int chunks, int crc)
629 struct output_file_callback *outc;
631 outc = calloc(1, sizeof(struct output_file_callback));
633 error_errno("malloc struct outc");
637 outc->out.ops = &callback_file_ops;
641 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
650 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
651 int gz, int sparse, int chunks, int crc)
654 struct output_file *out;
657 out = output_file_new_gz();
659 out = output_file_new_normal();
665 out->ops->open(out, fd);
667 ret = output_file_init(out, block_size, len, sparse, chunks, crc);
676 /* Write a contiguous region of data blocks from a memory buffer */
677 int write_data_chunk(struct output_file *out, unsigned int len, void *data)
679 return out->sparse_ops->write_data_chunk(out, len, data);
682 /* Write a contiguous region of data blocks with a fill value */
683 int write_fill_chunk(struct output_file *out, unsigned int len,
686 return out->sparse_ops->write_fill_chunk(out, len, fill_val);
689 int write_fd_chunk(struct output_file *out, unsigned int len,
690 int fd, int64_t offset)
693 int64_t aligned_offset;
698 aligned_offset = offset & ~(4096 - 1);
699 aligned_diff = offset - aligned_offset;
700 buffer_size = len + aligned_diff;
703 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
705 if (data == MAP_FAILED) {
708 ptr = data + aligned_diff;
711 char *data = malloc(len);
715 pos = lseek64(fd, offset, SEEK_SET);
720 ret = read_all(fd, data, len);
728 ret = out->sparse_ops->write_data_chunk(out, len, ptr);
731 munmap(data, buffer_size);
739 /* Write a contiguous region of data blocks from a file */
740 int write_file_chunk(struct output_file *out, unsigned int len,
741 const char *file, int64_t offset)
745 int file_fd = open(file, O_RDONLY);
750 ret = write_fd_chunk(out, len, file_fd, offset);
757 int write_skip_chunk(struct output_file *out, int64_t len)
759 return out->sparse_ops->write_skip_chunk(out, len);