2 * Copyright (C) 2010 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #define _FILE_OFFSET_BITS 64
18 #define _LARGEFILE64_SOURCE 1
28 #include <sys/types.h>
33 #include "output_file.h"
34 #include "sparse_crc32.h"
35 #include "sparse_format.h"
41 #define ftruncate64 ftruncate
44 #if defined(__APPLE__) && defined(__MACH__)
46 #define ftruncate64 ftruncate
52 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
54 #define SPARSE_HEADER_MAJOR_VER 1
55 #define SPARSE_HEADER_MINOR_VER 0
56 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
57 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
59 #define container_of(inner, outer_t, elem) \
60 ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
62 struct output_file_ops {
63 int (*open)(struct output_file *, int fd);
64 int (*skip)(struct output_file *, int64_t);
65 int (*pad)(struct output_file *, int64_t);
66 int (*write)(struct output_file *, void *, int);
67 void (*close)(struct output_file *);
70 struct sparse_file_ops {
71 int (*write_data_chunk)(struct output_file *out, unsigned int len,
73 int (*write_fill_chunk)(struct output_file *out, unsigned int len,
75 int (*write_skip_chunk)(struct output_file *out, int64_t len);
76 int (*write_end_chunk)(struct output_file *out);
81 unsigned int chunk_cnt;
83 struct output_file_ops *ops;
84 struct sparse_file_ops *sparse_ops;
86 unsigned int block_size;
93 struct output_file_gz {
94 struct output_file out;
98 #define to_output_file_gz(_o) \
99 container_of((_o), struct output_file_gz, out)
101 struct output_file_normal {
102 struct output_file out;
106 #define to_output_file_normal(_o) \
107 container_of((_o), struct output_file_normal, out)
109 struct output_file_callback {
110 struct output_file out;
112 int (*write)(void *priv, const void *buf, int len);
115 #define to_output_file_callback(_o) \
116 container_of((_o), struct output_file_callback, out)
118 static int file_open(struct output_file *out, int fd)
120 struct output_file_normal *outn = to_output_file_normal(out);
126 static int file_skip(struct output_file *out, int64_t cnt)
129 struct output_file_normal *outn = to_output_file_normal(out);
131 ret = lseek64(outn->fd, cnt, SEEK_CUR);
133 error_errno("lseek64");
139 static int file_pad(struct output_file *out, int64_t len)
142 struct output_file_normal *outn = to_output_file_normal(out);
144 ret = ftruncate64(outn->fd, len);
152 static int file_write(struct output_file *out, void *data, int len)
155 struct output_file_normal *outn = to_output_file_normal(out);
157 ret = write(outn->fd, data, len);
159 error_errno("write");
161 } else if (ret < len) {
162 error("incomplete write");
169 static void file_close(struct output_file *out)
171 struct output_file_normal *outn = to_output_file_normal(out);
176 static struct output_file_ops file_ops = {
184 static int gz_file_open(struct output_file *out, int fd)
186 struct output_file_gz *outgz = to_output_file_gz(out);
188 outgz->gz_fd = gzdopen(fd, "wb9");
190 error_errno("gzopen");
198 static int gz_file_skip(struct output_file *out, int64_t cnt)
201 struct output_file_gz *outgz = to_output_file_gz(out);
203 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
205 error_errno("gzseek");
211 static int gz_file_pad(struct output_file *out, int64_t len)
214 struct output_file_gz *outgz = to_output_file_gz(out);
216 ret = gztell(outgz->gz_fd);
225 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
230 gzwrite(outgz->gz_fd, "", 1);
235 static int gz_file_write(struct output_file *out, void *data, int len)
238 struct output_file_gz *outgz = to_output_file_gz(out);
240 ret = gzwrite(outgz->gz_fd, data, len);
242 error_errno("gzwrite");
244 } else if (ret < len) {
245 error("incomplete gzwrite");
252 static void gz_file_close(struct output_file *out)
254 struct output_file_gz *outgz = to_output_file_gz(out);
256 gzclose(outgz->gz_fd);
260 static struct output_file_ops gz_file_ops = {
261 .open = gz_file_open,
262 .skip = gz_file_skip,
264 .write = gz_file_write,
265 .close = gz_file_close,
268 static int callback_file_open(struct output_file *out __unused, int fd __unused)
273 static int callback_file_skip(struct output_file *out, int64_t off)
275 struct output_file_callback *outc = to_output_file_callback(out);
280 to_write = min(off, (int64_t)INT_MAX);
281 ret = outc->write(outc->priv, NULL, to_write);
291 static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
296 static int callback_file_write(struct output_file *out, void *data, int len)
298 struct output_file_callback *outc = to_output_file_callback(out);
300 return outc->write(outc->priv, data, len);
303 static void callback_file_close(struct output_file *out)
305 struct output_file_callback *outc = to_output_file_callback(out);
310 static struct output_file_ops callback_file_ops = {
311 .open = callback_file_open,
312 .skip = callback_file_skip,
313 .pad = callback_file_pad,
314 .write = callback_file_write,
315 .close = callback_file_close,
318 int read_all(int fd, void *buf, size_t len)
324 while (total < len) {
325 ret = read(fd, ptr, len - total);
340 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
342 chunk_header_t chunk_header;
345 if (skip_len % out->block_size) {
346 error("don't care size %"PRIi64" is not a multiple of the block size %u",
347 skip_len, out->block_size);
351 /* We are skipping data, so emit a don't care chunk. */
352 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
353 chunk_header.reserved1 = 0;
354 chunk_header.chunk_sz = skip_len / out->block_size;
355 chunk_header.total_sz = CHUNK_HEADER_LEN;
356 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
360 out->cur_out_ptr += skip_len;
366 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len,
369 chunk_header_t chunk_header;
370 int rnd_up_len, count;
373 /* Round up the fill length to a multiple of the block size */
374 rnd_up_len = ALIGN(len, out->block_size);
376 /* Finally we can safely emit a chunk of data */
377 chunk_header.chunk_type = CHUNK_TYPE_FILL;
378 chunk_header.reserved1 = 0;
379 chunk_header.chunk_sz = rnd_up_len / out->block_size;
380 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
381 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
385 ret = out->ops->write(out, &fill_val, sizeof(fill_val));
390 count = out->block_size / sizeof(uint32_t);
392 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
395 out->cur_out_ptr += rnd_up_len;
401 static int write_sparse_data_chunk(struct output_file *out, unsigned int len,
404 chunk_header_t chunk_header;
405 int rnd_up_len, zero_len;
408 /* Round up the data length to a multiple of the block size */
409 rnd_up_len = ALIGN(len, out->block_size);
410 zero_len = rnd_up_len - len;
412 /* Finally we can safely emit a chunk of data */
413 chunk_header.chunk_type = CHUNK_TYPE_RAW;
414 chunk_header.reserved1 = 0;
415 chunk_header.chunk_sz = rnd_up_len / out->block_size;
416 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
417 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
421 ret = out->ops->write(out, data, len);
425 ret = out->ops->write(out, out->zero_buf, zero_len);
431 out->crc32 = sparse_crc32(out->crc32, data, len);
433 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
436 out->cur_out_ptr += rnd_up_len;
442 int write_sparse_end_chunk(struct output_file *out)
444 chunk_header_t chunk_header;
448 chunk_header.chunk_type = CHUNK_TYPE_CRC32;
449 chunk_header.reserved1 = 0;
450 chunk_header.chunk_sz = 0;
451 chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
453 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
457 out->ops->write(out, &out->crc32, 4);
468 static struct sparse_file_ops sparse_file_ops = {
469 .write_data_chunk = write_sparse_data_chunk,
470 .write_fill_chunk = write_sparse_fill_chunk,
471 .write_skip_chunk = write_sparse_skip_chunk,
472 .write_end_chunk = write_sparse_end_chunk,
475 static int write_normal_data_chunk(struct output_file *out, unsigned int len,
479 unsigned int rnd_up_len = ALIGN(len, out->block_size);
481 ret = out->ops->write(out, data, len);
486 if (rnd_up_len > len) {
487 ret = out->ops->skip(out, rnd_up_len - len);
493 static int write_normal_fill_chunk(struct output_file *out, unsigned int len,
498 unsigned int write_len;
500 /* Initialize fill_buf with the fill_val */
501 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
502 out->fill_buf[i] = fill_val;
506 write_len = min(len, out->block_size);
507 ret = out->ops->write(out, out->fill_buf, write_len);
518 static int write_normal_skip_chunk(struct output_file *out, int64_t len)
520 return out->ops->skip(out, len);
523 int write_normal_end_chunk(struct output_file *out)
525 return out->ops->pad(out, out->len);
528 static struct sparse_file_ops normal_file_ops = {
529 .write_data_chunk = write_normal_data_chunk,
530 .write_fill_chunk = write_normal_fill_chunk,
531 .write_skip_chunk = write_normal_skip_chunk,
532 .write_end_chunk = write_normal_end_chunk,
535 void output_file_close(struct output_file *out)
537 out->sparse_ops->write_end_chunk(out);
538 out->ops->close(out);
541 static int output_file_init(struct output_file *out, int block_size,
542 int64_t len, bool sparse, int chunks, bool crc)
547 out->block_size = block_size;
548 out->cur_out_ptr = 0ll;
553 out->zero_buf = calloc(block_size, 1);
554 if (!out->zero_buf) {
555 error_errno("malloc zero_buf");
559 out->fill_buf = calloc(block_size, 1);
560 if (!out->fill_buf) {
561 error_errno("malloc fill_buf");
567 out->sparse_ops = &sparse_file_ops;
569 out->sparse_ops = &normal_file_ops;
573 sparse_header_t sparse_header = {
574 .magic = SPARSE_HEADER_MAGIC,
575 .major_version = SPARSE_HEADER_MAJOR_VER,
576 .minor_version = SPARSE_HEADER_MINOR_VER,
577 .file_hdr_sz = SPARSE_HEADER_LEN,
578 .chunk_hdr_sz = CHUNK_HEADER_LEN,
579 .blk_sz = out->block_size,
580 .total_blks = out->len / out->block_size,
581 .total_chunks = chunks,
586 sparse_header.total_chunks++;
589 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
604 static struct output_file *output_file_new_gz(void)
606 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
608 error_errno("malloc struct outgz");
612 outgz->out.ops = &gz_file_ops;
617 static struct output_file *output_file_new_normal(void)
619 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
621 error_errno("malloc struct outn");
625 outn->out.ops = &file_ops;
630 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int),
631 void *priv, unsigned int block_size, int64_t len,
632 int gz __unused, int sparse, int chunks, int crc)
635 struct output_file_callback *outc;
637 outc = calloc(1, sizeof(struct output_file_callback));
639 error_errno("malloc struct outc");
643 outc->out.ops = &callback_file_ops;
647 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
656 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
657 int gz, int sparse, int chunks, int crc)
660 struct output_file *out;
663 out = output_file_new_gz();
665 out = output_file_new_normal();
671 out->ops->open(out, fd);
673 ret = output_file_init(out, block_size, len, sparse, chunks, crc);
682 /* Write a contiguous region of data blocks from a memory buffer */
683 int write_data_chunk(struct output_file *out, unsigned int len, void *data)
685 return out->sparse_ops->write_data_chunk(out, len, data);
688 /* Write a contiguous region of data blocks with a fill value */
689 int write_fill_chunk(struct output_file *out, unsigned int len,
692 return out->sparse_ops->write_fill_chunk(out, len, fill_val);
695 int write_fd_chunk(struct output_file *out, unsigned int len,
696 int fd, int64_t offset)
699 int64_t aligned_offset;
704 aligned_offset = offset & ~(4096 - 1);
705 aligned_diff = offset - aligned_offset;
706 buffer_size = len + aligned_diff;
709 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
711 if (data == MAP_FAILED) {
714 ptr = data + aligned_diff;
717 char *data = malloc(len);
721 pos = lseek64(fd, offset, SEEK_SET);
726 ret = read_all(fd, data, len);
734 ret = out->sparse_ops->write_data_chunk(out, len, ptr);
737 munmap(data, buffer_size);
745 /* Write a contiguous region of data blocks from a file */
746 int write_file_chunk(struct output_file *out, unsigned int len,
747 const char *file, int64_t offset)
751 int file_fd = open(file, O_RDONLY | O_BINARY);
756 ret = write_fd_chunk(out, len, file_fd, offset);
763 int write_skip_chunk(struct output_file *out, int64_t len)
765 return out->sparse_ops->write_skip_chunk(out, len);