libsparse/output_file.c

   1 /*
   2  * Copyright (C) 2010 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #define _FILE_OFFSET_BITS 64
  18 #define _LARGEFILE64_SOURCE 1
  19
  20 #include <fcntl.h>
  21 #include <inttypes.h>
  22 #include <limits.h>
  23 #include <stdbool.h>
  24 #include <stddef.h>
  25 #include <stdlib.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <sys/stat.h>
  29 #include <sys/types.h>
  30 #include <unistd.h>
  31 #include <zlib.h>
  32
  33 #include "defs.h"
  34 #include "output_file.h"
  35 #include "sparse_crc32.h"
  36 #include "sparse_format.h"
  37
  38 #if defined(__APPLE__) && defined(__MACH__)
  39 #define lseek64 lseek
  40 #define ftruncate64 ftruncate
  41 #define mmap64 mmap
  42 #define off64_t off_t
  43 #endif
  44
  45 #define min(a, b) \
  46         ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
  47
  48 #define SPARSE_HEADER_MAJOR_VER 1
  49 #define SPARSE_HEADER_MINOR_VER 0
  50 #define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
  51 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
  52
  53 #define container_of(inner, outer_t, elem) \
  54         ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
  55
  56 struct output_file_ops {
  57         int (*open)(struct output_file *, int fd);
  58         int (*skip)(struct output_file *, int64_t);
  59         int (*pad)(struct output_file *, int64_t);
  60         int (*write)(struct output_file *, void *, int);
  61         void (*close)(struct output_file *);
  62 };
  63
  64 struct sparse_file_ops {
  65         int (*write_data_chunk)(struct output_file *out, unsigned int len,
  66                         void *data);
  67         int (*write_fill_chunk)(struct output_file *out, unsigned int len,
  68                         uint32_t fill_val);
  69         int (*write_skip_chunk)(struct output_file *out, int64_t len);
  70         int (*write_end_chunk)(struct output_file *out);
  71 };
  72
  73 struct output_file {
  74         int64_t cur_out_ptr;
  75         unsigned int chunk_cnt;
  76         uint32_t crc32;
  77         struct output_file_ops *ops;
  78         struct sparse_file_ops *sparse_ops;
  79         int use_crc;
  80         unsigned int block_size;
  81         int64_t len;
  82         char *zero_buf;
  83         uint32_t *fill_buf;
  84         char *buf;
  85 };
  86
  87 struct output_file_gz {
  88         struct output_file out;
  89         gzFile gz_fd;
  90 };
  91
  92 #define to_output_file_gz(_o) \
  93         container_of((_o), struct output_file_gz, out)
  94
  95 struct output_file_normal {
  96         struct output_file out;
  97         int fd;
  98 };
  99
 100 #define to_output_file_normal(_o) \
 101         container_of((_o), struct output_file_normal, out)
 102
 103 struct output_file_callback {
 104         struct output_file out;
 105         void *priv;
 106         int (*write)(void *priv, const void *buf, int len);
 107 };
 108
 109 #define to_output_file_callback(_o) \
 110         container_of((_o), struct output_file_callback, out)
 111
 112 static int file_open(struct output_file *out, int fd)
 113 {
 114         struct output_file_normal *outn = to_output_file_normal(out);
 115
 116         outn->fd = fd;
 117         return 0;
 118 }
 119
 120 static int file_skip(struct output_file *out, int64_t cnt)
 121 {
 122         off64_t ret;
 123         struct output_file_normal *outn = to_output_file_normal(out);
 124
 125         ret = lseek64(outn->fd, cnt, SEEK_CUR);
 126         if (ret < 0) {
 127                 error_errno("lseek64");
 128                 return -1;
 129         }
 130         return 0;
 131 }
 132
 133 static int file_pad(struct output_file *out, int64_t len)
 134 {
 135         int ret;
 136         struct output_file_normal *outn = to_output_file_normal(out);
 137
 138         ret = ftruncate64(outn->fd, len);
 139         if (ret < 0) {
 140                 return -errno;
 141         }
 142
 143         return 0;
 144 }
 145
 146 static int file_write(struct output_file *out, void *data, int len)
 147 {
 148         int ret;
 149         struct output_file_normal *outn = to_output_file_normal(out);
 150
 151         ret = write(outn->fd, data, len);
 152         if (ret < 0) {
 153                 error_errno("write");
 154                 return -1;
 155         } else if (ret < len) {
 156                 error("incomplete write");
 157                 return -1;
 158         }
 159
 160         return 0;
 161 }
 162
 163 static void file_close(struct output_file *out)
 164 {
 165         struct output_file_normal *outn = to_output_file_normal(out);
 166
 167         free(outn);
 168 }
 169
 170 static struct output_file_ops file_ops = {
 171         .open = file_open,
 172         .skip = file_skip,
 173         .pad = file_pad,
 174         .write = file_write,
 175         .close = file_close,
 176 };
 177
 178 static int gz_file_open(struct output_file *out, int fd)
 179 {
 180         struct output_file_gz *outgz = to_output_file_gz(out);
 181
 182         outgz->gz_fd = gzdopen(fd, "wb9");
 183         if (!outgz->gz_fd) {
 184                 error_errno("gzopen");
 185                 return -errno;
 186         }
 187
 188         return 0;
 189 }
 190
 191
 192 static int gz_file_skip(struct output_file *out, int64_t cnt)
 193 {
 194         off64_t ret;
 195         struct output_file_gz *outgz = to_output_file_gz(out);
 196
 197         ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
 198         if (ret < 0) {
 199                 error_errno("gzseek");
 200                 return -1;
 201         }
 202         return 0;
 203 }
 204
 205 static int gz_file_pad(struct output_file *out, int64_t len)
 206 {
 207         off64_t ret;
 208         struct output_file_gz *outgz = to_output_file_gz(out);
 209
 210         ret = gztell(outgz->gz_fd);
 211         if (ret < 0) {
 212                 return -1;
 213         }
 214
 215         if (ret >= len) {
 216                 return 0;
 217         }
 218
 219         ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
 220         if (ret < 0) {
 221                 return -1;
 222         }
 223
 224         gzwrite(outgz->gz_fd, "", 1);
 225
 226         return 0;
 227 }
 228
 229 static int gz_file_write(struct output_file *out, void *data, int len)
 230 {
 231         int ret;
 232         struct output_file_gz *outgz = to_output_file_gz(out);
 233
 234         ret = gzwrite(outgz->gz_fd, data, len);
 235         if (ret < 0) {
 236                 error_errno("gzwrite");
 237                 return -1;
 238         } else if (ret < len) {
 239                 error("incomplete gzwrite");
 240                 return -1;
 241         }
 242
 243         return 0;
 244 }
 245
 246 static void gz_file_close(struct output_file *out)
 247 {
 248         struct output_file_gz *outgz = to_output_file_gz(out);
 249
 250         gzclose(outgz->gz_fd);
 251         free(outgz);
 252 }
 253
 254 static struct output_file_ops gz_file_ops = {
 255         .open = gz_file_open,
 256         .skip = gz_file_skip,
 257         .pad = gz_file_pad,
 258         .write = gz_file_write,
 259         .close = gz_file_close,
 260 };
 261
 262 static int callback_file_open(struct output_file *out __unused, int fd __unused)
 263 {
 264         return 0;
 265 }
 266
 267 static int callback_file_skip(struct output_file *out, int64_t off)
 268 {
 269         struct output_file_callback *outc = to_output_file_callback(out);
 270         int to_write;
 271         int ret;
 272
 273         while (off > 0) {
 274                 to_write = min(off, (int64_t)INT_MAX);
 275                 ret = outc->write(outc->priv, NULL, to_write);
 276                 if (ret < 0) {
 277                         return ret;
 278                 }
 279                 off -= to_write;
 280         }
 281
 282         return 0;
 283 }
 284
 285 static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
 286 {
 287         return -1;
 288 }
 289
 290 static int callback_file_write(struct output_file *out, void *data, int len)
 291 {
 292         struct output_file_callback *outc = to_output_file_callback(out);
 293
 294         return outc->write(outc->priv, data, len);
 295 }
 296
 297 static void callback_file_close(struct output_file *out)
 298 {
 299         struct output_file_callback *outc = to_output_file_callback(out);
 300
 301         free(outc);
 302 }
 303
 304 static struct output_file_ops callback_file_ops = {
 305         .open = callback_file_open,
 306         .skip = callback_file_skip,
 307         .pad = callback_file_pad,
 308         .write = callback_file_write,
 309         .close = callback_file_close,
 310 };
 311
 312 int read_all(int fd, void *buf, size_t len)
 313 {
 314         size_t total = 0;
 315         int ret;
 316         char *ptr = buf;
 317
 318         while (total < len) {
 319                 ret = read(fd, ptr, len - total);
 320
 321                 if (ret < 0)
 322                         return -errno;
 323
 324                 if (ret == 0)
 325                         return -EINVAL;
 326
 327                 ptr += ret;
 328                 total += ret;
 329         }
 330
 331         return 0;
 332 }
 333
 334 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
 335 {
 336         chunk_header_t chunk_header;
 337         int ret;
 338
 339         if (skip_len % out->block_size) {
 340                 error("don't care size %"PRIi64" is not a multiple of the block size %u",
 341                                 skip_len, out->block_size);
 342                 return -1;
 343         }
 344
 345         /* We are skipping data, so emit a don't care chunk. */
 346         chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
 347         chunk_header.reserved1 = 0;
 348         chunk_header.chunk_sz = skip_len / out->block_size;
 349         chunk_header.total_sz = CHUNK_HEADER_LEN;
 350         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 351         if (ret < 0)
 352                 return -1;
 353
 354         out->cur_out_ptr += skip_len;
 355         out->chunk_cnt++;
 356
 357         return 0;
 358 }
 359
 360 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len,
 361                 uint32_t fill_val)
 362 {
 363         chunk_header_t chunk_header;
 364         int rnd_up_len, count;
 365         int ret;
 366
 367         /* Round up the fill length to a multiple of the block size */
 368         rnd_up_len = ALIGN(len, out->block_size);
 369
 370         /* Finally we can safely emit a chunk of data */
 371         chunk_header.chunk_type = CHUNK_TYPE_FILL;
 372         chunk_header.reserved1 = 0;
 373         chunk_header.chunk_sz = rnd_up_len / out->block_size;
 374         chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
 375         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 376
 377         if (ret < 0)
 378                 return -1;
 379         ret = out->ops->write(out, &fill_val, sizeof(fill_val));
 380         if (ret < 0)
 381                 return -1;
 382
 383         if (out->use_crc) {
 384                 count = out->block_size / sizeof(uint32_t);
 385                 while (count--)
 386                         out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
 387         }
 388
 389         out->cur_out_ptr += rnd_up_len;
 390         out->chunk_cnt++;
 391
 392         return 0;
 393 }
 394
 395 static int write_sparse_data_chunk(struct output_file *out, unsigned int len,
 396                 void *data)
 397 {
 398         chunk_header_t chunk_header;
 399         int rnd_up_len, zero_len;
 400         int ret;
 401
 402         /* Round up the data length to a multiple of the block size */
 403         rnd_up_len = ALIGN(len, out->block_size);
 404         zero_len = rnd_up_len - len;
 405
 406         /* Finally we can safely emit a chunk of data */
 407         chunk_header.chunk_type = CHUNK_TYPE_RAW;
 408         chunk_header.reserved1 = 0;
 409         chunk_header.chunk_sz = rnd_up_len / out->block_size;
 410         chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
 411         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 412
 413         if (ret < 0)
 414                 return -1;
 415         ret = out->ops->write(out, data, len);
 416         if (ret < 0)
 417                 return -1;
 418         if (zero_len) {
 419                 ret = out->ops->write(out, out->zero_buf, zero_len);
 420                 if (ret < 0)
 421                         return -1;
 422         }
 423
 424         if (out->use_crc) {
 425                 out->crc32 = sparse_crc32(out->crc32, data, len);
 426                 if (zero_len)
 427                         out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
 428         }
 429
 430         out->cur_out_ptr += rnd_up_len;
 431         out->chunk_cnt++;
 432
 433         return 0;
 434 }
 435
 436 int write_sparse_end_chunk(struct output_file *out)
 437 {
 438         chunk_header_t chunk_header;
 439         int ret;
 440
 441         if (out->use_crc) {
 442                 chunk_header.chunk_type = CHUNK_TYPE_CRC32;
 443                 chunk_header.reserved1 = 0;
 444                 chunk_header.chunk_sz = 0;
 445                 chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
 446
 447                 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 448                 if (ret < 0) {
 449                         return ret;
 450                 }
 451                 out->ops->write(out, &out->crc32, 4);
 452                 if (ret < 0) {
 453                         return ret;
 454                 }
 455
 456                 out->chunk_cnt++;
 457         }
 458
 459         return 0;
 460 }
 461
 462 static struct sparse_file_ops sparse_file_ops = {
 463                 .write_data_chunk = write_sparse_data_chunk,
 464                 .write_fill_chunk = write_sparse_fill_chunk,
 465                 .write_skip_chunk = write_sparse_skip_chunk,
 466                 .write_end_chunk = write_sparse_end_chunk,
 467 };
 468
 469 static int write_normal_data_chunk(struct output_file *out, unsigned int len,
 470                 void *data)
 471 {
 472         int ret;
 473         unsigned int rnd_up_len = ALIGN(len, out->block_size);
 474
 475         ret = out->ops->write(out, data, len);
 476         if (ret < 0) {
 477                 return ret;
 478         }
 479
 480         if (rnd_up_len > len) {
 481                 ret = out->ops->skip(out, rnd_up_len - len);
 482         }
 483
 484         return ret;
 485 }
 486
 487 static int write_normal_fill_chunk(struct output_file *out, unsigned int len,
 488                 uint32_t fill_val)
 489 {
 490         int ret;
 491         unsigned int i;
 492         unsigned int write_len;
 493
 494         /* Initialize fill_buf with the fill_val */
 495         for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
 496                 out->fill_buf[i] = fill_val;
 497         }
 498
 499         while (len) {
 500                 write_len = min(len, out->block_size);
 501                 ret = out->ops->write(out, out->fill_buf, write_len);
 502                 if (ret < 0) {
 503                         return ret;
 504                 }
 505
 506                 len -= write_len;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static int write_normal_skip_chunk(struct output_file *out, int64_t len)
 513 {
 514         return out->ops->skip(out, len);
 515 }
 516
 517 int write_normal_end_chunk(struct output_file *out)
 518 {
 519         return out->ops->pad(out, out->len);
 520 }
 521
 522 static struct sparse_file_ops normal_file_ops = {
 523                 .write_data_chunk = write_normal_data_chunk,
 524                 .write_fill_chunk = write_normal_fill_chunk,
 525                 .write_skip_chunk = write_normal_skip_chunk,
 526                 .write_end_chunk = write_normal_end_chunk,
 527 };
 528
 529 void output_file_close(struct output_file *out)
 530 {
 531         out->sparse_ops->write_end_chunk(out);
 532         out->ops->close(out);
 533 }
 534
 535 static int output_file_init(struct output_file *out, int block_size,
 536                 int64_t len, bool sparse, int chunks, bool crc)
 537 {
 538         int ret;
 539
 540         out->len = len;
 541         out->block_size = block_size;
 542         out->cur_out_ptr = 0ll;
 543         out->chunk_cnt = 0;
 544         out->crc32 = 0;
 545         out->use_crc = crc;
 546
 547         out->zero_buf = calloc(block_size, 1);
 548         if (!out->zero_buf) {
 549                 error_errno("malloc zero_buf");
 550                 return -ENOMEM;
 551         }
 552
 553         out->fill_buf = calloc(block_size, 1);
 554         if (!out->fill_buf) {
 555                 error_errno("malloc fill_buf");
 556                 ret = -ENOMEM;
 557                 goto err_fill_buf;
 558         }
 559
 560         if (sparse) {
 561                 out->sparse_ops = &sparse_file_ops;
 562         } else {
 563                 out->sparse_ops = &normal_file_ops;
 564         }
 565
 566         if (sparse) {
 567                 sparse_header_t sparse_header = {
 568                                 .magic = SPARSE_HEADER_MAGIC,
 569                                 .major_version = SPARSE_HEADER_MAJOR_VER,
 570                                 .minor_version = SPARSE_HEADER_MINOR_VER,
 571                                 .file_hdr_sz = SPARSE_HEADER_LEN,
 572                                 .chunk_hdr_sz = CHUNK_HEADER_LEN,
 573                                 .blk_sz = out->block_size,
 574                                 .total_blks = out->len / out->block_size,
 575                                 .total_chunks = chunks,
 576                                 .image_checksum = 0
 577                 };
 578
 579                 if (out->use_crc) {
 580                         sparse_header.total_chunks++;
 581                 }
 582
 583                 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
 584                 if (ret < 0) {
 585                         goto err_write;
 586                 }
 587         }
 588
 589         return 0;
 590
 591 err_write:
 592         free(out->fill_buf);
 593 err_fill_buf:
 594         free(out->zero_buf);
 595         return ret;
 596 }
 597
 598 static struct output_file *output_file_new_gz(void)
 599 {
 600         struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
 601         if (!outgz) {
 602                 error_errno("malloc struct outgz");
 603                 return NULL;
 604         }
 605
 606         outgz->out.ops = &gz_file_ops;
 607
 608         return &outgz->out;
 609 }
 610
 611 static struct output_file *output_file_new_normal(void)
 612 {
 613         struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
 614         if (!outn) {
 615                 error_errno("malloc struct outn");
 616                 return NULL;
 617         }
 618
 619         outn->out.ops = &file_ops;
 620
 621         return &outn->out;
 622 }
 623
 624 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int),
 625                 void *priv, unsigned int block_size, int64_t len,
 626                 int gz __unused, int sparse, int chunks, int crc)
 627 {
 628         int ret;
 629         struct output_file_callback *outc;
 630
 631         outc = calloc(1, sizeof(struct output_file_callback));
 632         if (!outc) {
 633                 error_errno("malloc struct outc");
 634                 return NULL;
 635         }
 636
 637         outc->out.ops = &callback_file_ops;
 638         outc->priv = priv;
 639         outc->write = write;
 640
 641         ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
 642         if (ret < 0) {
 643                 free(outc);
 644                 return NULL;
 645         }
 646
 647         return &outc->out;
 648 }
 649
 650 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
 651                 int gz, int sparse, int chunks, int crc)
 652 {
 653         int ret;
 654         struct output_file *out;
 655
 656         if (gz) {
 657                 out = output_file_new_gz();
 658         } else {
 659                 out = output_file_new_normal();
 660         }
 661         if (!out) {
 662                 return NULL;
 663         }
 664
 665         out->ops->open(out, fd);
 666
 667         ret = output_file_init(out, block_size, len, sparse, chunks, crc);
 668         if (ret < 0) {
 669                 free(out);
 670                 return NULL;
 671         }
 672
 673         return out;
 674 }
 675
 676 /* Write a contiguous region of data blocks from a memory buffer */
 677 int write_data_chunk(struct output_file *out, unsigned int len, void *data)
 678 {
 679         return out->sparse_ops->write_data_chunk(out, len, data);
 680 }
 681
 682 /* Write a contiguous region of data blocks with a fill value */
 683 int write_fill_chunk(struct output_file *out, unsigned int len,
 684                 uint32_t fill_val)
 685 {
 686         return out->sparse_ops->write_fill_chunk(out, len, fill_val);
 687 }
 688
 689 int write_fd_chunk(struct output_file *out, unsigned int len,
 690                 int fd, int64_t offset)
 691 {
 692         int ret;
 693         int64_t aligned_offset;
 694         int aligned_diff;
 695         int buffer_size;
 696         char *ptr;
 697
 698         aligned_offset = offset & ~(4096 - 1);
 699         aligned_diff = offset - aligned_offset;
 700         buffer_size = len + aligned_diff;
 701
 702         char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
 703                         aligned_offset);
 704         if (data == MAP_FAILED) {
 705                 return -errno;
 706         }
 707         ptr = data + aligned_diff;
 708
 709         ret = out->sparse_ops->write_data_chunk(out, len, ptr);
 710
 711         munmap(data, buffer_size);
 712
 713         return ret;
 714 }
 715
 716 /* Write a contiguous region of data blocks from a file */
 717 int write_file_chunk(struct output_file *out, unsigned int len,
 718                 const char *file, int64_t offset)
 719 {
 720         int ret;
 721
 722         int file_fd = open(file, O_RDONLY);
 723         if (file_fd < 0) {
 724                 return -errno;
 725         }
 726
 727         ret = write_fd_chunk(out, len, file_fd, offset);
 728
 729         close(file_fd);
 730
 731         return ret;
 732 }
 733
 734 int write_skip_chunk(struct output_file *out, int64_t len)
 735 {
 736         return out->sparse_ops->write_skip_chunk(out, len);
 737 }