libsparse/output_file.c

   1 /*
   2  * Copyright (C) 2010 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #define _FILE_OFFSET_BITS 64
  18 #define _LARGEFILE64_SOURCE 1
  19
  20 #include <fcntl.h>
  21 #include <inttypes.h>
  22 #include <limits.h>
  23 #include <stdbool.h>
  24 #include <stddef.h>
  25 #include <stdlib.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <sys/stat.h>
  29 #include <sys/types.h>
  30 #include <unistd.h>
  31 #include <zlib.h>
  32
  33 #include "defs.h"
  34 #include "output_file.h"
  35 #include "sparse_crc32.h"
  36 #include "sparse_format.h"
  37
  38 #define min(a, b) \
  39         ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
  40
  41 #define SPARSE_HEADER_MAJOR_VER 1
  42 #define SPARSE_HEADER_MINOR_VER 0
  43 #define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
  44 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
  45
  46 #define container_of(inner, outer_t, elem) \
  47         ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
  48
  49 struct output_file_ops {
  50         int (*open)(struct output_file *, int fd);
  51         int (*skip)(struct output_file *, int64_t);
  52         int (*pad)(struct output_file *, int64_t);
  53         int (*write)(struct output_file *, void *, int);
  54         void (*close)(struct output_file *);
  55 };
  56
  57 struct sparse_file_ops {
  58         int (*write_data_chunk)(struct output_file *out, unsigned int len,
  59                         void *data);
  60         int (*write_fill_chunk)(struct output_file *out, unsigned int len,
  61                         uint32_t fill_val);
  62         int (*write_skip_chunk)(struct output_file *out, int64_t len);
  63         int (*write_end_chunk)(struct output_file *out);
  64 };
  65
  66 struct output_file {
  67         int64_t cur_out_ptr;
  68         unsigned int chunk_cnt;
  69         uint32_t crc32;
  70         struct output_file_ops *ops;
  71         struct sparse_file_ops *sparse_ops;
  72         int use_crc;
  73         unsigned int block_size;
  74         int64_t len;
  75         char *zero_buf;
  76         uint32_t *fill_buf;
  77         char *buf;
  78 };
  79
  80 struct output_file_gz {
  81         struct output_file out;
  82         gzFile gz_fd;
  83 };
  84
  85 #define to_output_file_gz(_o) \
  86         container_of((_o), struct output_file_gz, out)
  87
  88 struct output_file_normal {
  89         struct output_file out;
  90         int fd;
  91 };
  92
  93 #define to_output_file_normal(_o) \
  94         container_of((_o), struct output_file_normal, out)
  95
  96 struct output_file_callback {
  97         struct output_file out;
  98         void *priv;
  99         int (*write)(void *priv, const void *buf, int len);
 100 };
 101
 102 #define to_output_file_callback(_o) \
 103         container_of((_o), struct output_file_callback, out)
 104
 105 static int file_open(struct output_file *out, int fd)
 106 {
 107         struct output_file_normal *outn = to_output_file_normal(out);
 108
 109         outn->fd = fd;
 110         return 0;
 111 }
 112
 113 static int file_skip(struct output_file *out, int64_t cnt)
 114 {
 115         off_t ret;
 116         struct output_file_normal *outn = to_output_file_normal(out);
 117
 118         ret = lseek(outn->fd, cnt, SEEK_CUR);
 119         if (ret < 0) {
 120                 error_errno("lseek");
 121                 return -1;
 122         }
 123         return 0;
 124 }
 125
 126 static int file_pad(struct output_file *out, int64_t len)
 127 {
 128         int ret;
 129         struct output_file_normal *outn = to_output_file_normal(out);
 130
 131         ret = ftruncate(outn->fd, len);
 132         if (ret < 0) {
 133                 return -errno;
 134         }
 135
 136         return 0;
 137 }
 138
 139 static int file_write(struct output_file *out, void *data, int len)
 140 {
 141         int ret;
 142         struct output_file_normal *outn = to_output_file_normal(out);
 143
 144         ret = write(outn->fd, data, len);
 145         if (ret < 0) {
 146                 error_errno("write");
 147                 return -1;
 148         } else if (ret < len) {
 149                 error("incomplete write");
 150                 return -1;
 151         }
 152
 153         return 0;
 154 }
 155
 156 static void file_close(struct output_file *out)
 157 {
 158         struct output_file_normal *outn = to_output_file_normal(out);
 159
 160         free(outn);
 161 }
 162
 163 static struct output_file_ops file_ops = {
 164         .open = file_open,
 165         .skip = file_skip,
 166         .pad = file_pad,
 167         .write = file_write,
 168         .close = file_close,
 169 };
 170
 171 static int gz_file_open(struct output_file *out, int fd)
 172 {
 173         struct output_file_gz *outgz = to_output_file_gz(out);
 174
 175         outgz->gz_fd = gzdopen(fd, "wb9");
 176         if (!outgz->gz_fd) {
 177                 error_errno("gzopen");
 178                 return -errno;
 179         }
 180
 181         return 0;
 182 }
 183
 184
 185 static int gz_file_skip(struct output_file *out, int64_t cnt)
 186 {
 187         off_t ret;
 188         struct output_file_gz *outgz = to_output_file_gz(out);
 189
 190         ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
 191         if (ret < 0) {
 192                 error_errno("gzseek");
 193                 return -1;
 194         }
 195         return 0;
 196 }
 197
 198 static int gz_file_pad(struct output_file *out, int64_t len)
 199 {
 200         off_t ret;
 201         struct output_file_gz *outgz = to_output_file_gz(out);
 202
 203         ret = gztell(outgz->gz_fd);
 204         if (ret < 0) {
 205                 return -1;
 206         }
 207
 208         if (ret >= len) {
 209                 return 0;
 210         }
 211
 212         ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
 213         if (ret < 0) {
 214                 return -1;
 215         }
 216
 217         gzwrite(outgz->gz_fd, "", 1);
 218
 219         return 0;
 220 }
 221
 222 static int gz_file_write(struct output_file *out, void *data, int len)
 223 {
 224         int ret;
 225         struct output_file_gz *outgz = to_output_file_gz(out);
 226
 227         ret = gzwrite(outgz->gz_fd, data, len);
 228         if (ret < 0) {
 229                 error_errno("gzwrite");
 230                 return -1;
 231         } else if (ret < len) {
 232                 error("incomplete gzwrite");
 233                 return -1;
 234         }
 235
 236         return 0;
 237 }
 238
 239 static void gz_file_close(struct output_file *out)
 240 {
 241         struct output_file_gz *outgz = to_output_file_gz(out);
 242
 243         gzclose(outgz->gz_fd);
 244         free(outgz);
 245 }
 246
 247 static struct output_file_ops gz_file_ops = {
 248         .open = gz_file_open,
 249         .skip = gz_file_skip,
 250         .pad = gz_file_pad,
 251         .write = gz_file_write,
 252         .close = gz_file_close,
 253 };
 254
 255 static int callback_file_open(struct output_file *out __unused, int fd __unused)
 256 {
 257         return 0;
 258 }
 259
 260 static int callback_file_skip(struct output_file *out, int64_t off)
 261 {
 262         struct output_file_callback *outc = to_output_file_callback(out);
 263         int to_write;
 264         int ret;
 265
 266         while (off > 0) {
 267                 to_write = min(off, (int64_t)INT_MAX);
 268                 ret = outc->write(outc->priv, NULL, to_write);
 269                 if (ret < 0) {
 270                         return ret;
 271                 }
 272                 off -= to_write;
 273         }
 274
 275         return 0;
 276 }
 277
 278 static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
 279 {
 280         return -1;
 281 }
 282
 283 static int callback_file_write(struct output_file *out, void *data, int len)
 284 {
 285         struct output_file_callback *outc = to_output_file_callback(out);
 286
 287         return outc->write(outc->priv, data, len);
 288 }
 289
 290 static void callback_file_close(struct output_file *out)
 291 {
 292         struct output_file_callback *outc = to_output_file_callback(out);
 293
 294         free(outc);
 295 }
 296
 297 static struct output_file_ops callback_file_ops = {
 298         .open = callback_file_open,
 299         .skip = callback_file_skip,
 300         .pad = callback_file_pad,
 301         .write = callback_file_write,
 302         .close = callback_file_close,
 303 };
 304
 305 int read_all(int fd, void *buf, size_t len)
 306 {
 307         size_t total = 0;
 308         int ret;
 309         char *ptr = buf;
 310
 311         while (total < len) {
 312                 ret = read(fd, ptr, len - total);
 313
 314                 if (ret < 0)
 315                         return -errno;
 316
 317                 if (ret == 0)
 318                         return -EINVAL;
 319
 320                 ptr += ret;
 321                 total += ret;
 322         }
 323
 324         return 0;
 325 }
 326
 327 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
 328 {
 329         chunk_header_t chunk_header;
 330         int ret;
 331
 332         if (skip_len % out->block_size) {
 333                 error("don't care size %"PRIi64" is not a multiple of the block size %u",
 334                                 skip_len, out->block_size);
 335                 return -1;
 336         }
 337
 338         /* We are skipping data, so emit a don't care chunk. */
 339         chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
 340         chunk_header.reserved1 = 0;
 341         chunk_header.chunk_sz = skip_len / out->block_size;
 342         chunk_header.total_sz = CHUNK_HEADER_LEN;
 343         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 344         if (ret < 0)
 345                 return -1;
 346
 347         out->cur_out_ptr += skip_len;
 348         out->chunk_cnt++;
 349
 350         return 0;
 351 }
 352
 353 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len,
 354                 uint32_t fill_val)
 355 {
 356         chunk_header_t chunk_header;
 357         int rnd_up_len, count;
 358         int ret;
 359
 360         /* Round up the fill length to a multiple of the block size */
 361         rnd_up_len = ALIGN(len, out->block_size);
 362
 363         /* Finally we can safely emit a chunk of data */
 364         chunk_header.chunk_type = CHUNK_TYPE_FILL;
 365         chunk_header.reserved1 = 0;
 366         chunk_header.chunk_sz = rnd_up_len / out->block_size;
 367         chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
 368         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 369
 370         if (ret < 0)
 371                 return -1;
 372         ret = out->ops->write(out, &fill_val, sizeof(fill_val));
 373         if (ret < 0)
 374                 return -1;
 375
 376         if (out->use_crc) {
 377                 count = out->block_size / sizeof(uint32_t);
 378                 while (count--)
 379                         out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
 380         }
 381
 382         out->cur_out_ptr += rnd_up_len;
 383         out->chunk_cnt++;
 384
 385         return 0;
 386 }
 387
 388 static int write_sparse_data_chunk(struct output_file *out, unsigned int len,
 389                 void *data)
 390 {
 391         chunk_header_t chunk_header;
 392         int rnd_up_len, zero_len;
 393         int ret;
 394
 395         /* Round up the data length to a multiple of the block size */
 396         rnd_up_len = ALIGN(len, out->block_size);
 397         zero_len = rnd_up_len - len;
 398
 399         /* Finally we can safely emit a chunk of data */
 400         chunk_header.chunk_type = CHUNK_TYPE_RAW;
 401         chunk_header.reserved1 = 0;
 402         chunk_header.chunk_sz = rnd_up_len / out->block_size;
 403         chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
 404         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 405
 406         if (ret < 0)
 407                 return -1;
 408         ret = out->ops->write(out, data, len);
 409         if (ret < 0)
 410                 return -1;
 411         if (zero_len) {
 412                 ret = out->ops->write(out, out->zero_buf, zero_len);
 413                 if (ret < 0)
 414                         return -1;
 415         }
 416
 417         if (out->use_crc) {
 418                 out->crc32 = sparse_crc32(out->crc32, data, len);
 419                 if (zero_len)
 420                         out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
 421         }
 422
 423         out->cur_out_ptr += rnd_up_len;
 424         out->chunk_cnt++;
 425
 426         return 0;
 427 }
 428
 429 int write_sparse_end_chunk(struct output_file *out)
 430 {
 431         chunk_header_t chunk_header;
 432         int ret;
 433
 434         if (out->use_crc) {
 435                 chunk_header.chunk_type = CHUNK_TYPE_CRC32;
 436                 chunk_header.reserved1 = 0;
 437                 chunk_header.chunk_sz = 0;
 438                 chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
 439
 440                 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 441                 if (ret < 0) {
 442                         return ret;
 443                 }
 444                 out->ops->write(out, &out->crc32, 4);
 445                 if (ret < 0) {
 446                         return ret;
 447                 }
 448
 449                 out->chunk_cnt++;
 450         }
 451
 452         return 0;
 453 }
 454
 455 static struct sparse_file_ops sparse_file_ops = {
 456                 .write_data_chunk = write_sparse_data_chunk,
 457                 .write_fill_chunk = write_sparse_fill_chunk,
 458                 .write_skip_chunk = write_sparse_skip_chunk,
 459                 .write_end_chunk = write_sparse_end_chunk,
 460 };
 461
 462 static int write_normal_data_chunk(struct output_file *out, unsigned int len,
 463                 void *data)
 464 {
 465         int ret;
 466         unsigned int rnd_up_len = ALIGN(len, out->block_size);
 467
 468         ret = out->ops->write(out, data, len);
 469         if (ret < 0) {
 470                 return ret;
 471         }
 472
 473         if (rnd_up_len > len) {
 474                 ret = out->ops->skip(out, rnd_up_len - len);
 475         }
 476
 477         return ret;
 478 }
 479
 480 static int write_normal_fill_chunk(struct output_file *out, unsigned int len,
 481                 uint32_t fill_val)
 482 {
 483         int ret;
 484         unsigned int i;
 485         unsigned int write_len;
 486
 487         /* Initialize fill_buf with the fill_val */
 488         for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
 489                 out->fill_buf[i] = fill_val;
 490         }
 491
 492         while (len) {
 493                 write_len = min(len, out->block_size);
 494                 ret = out->ops->write(out, out->fill_buf, write_len);
 495                 if (ret < 0) {
 496                         return ret;
 497                 }
 498
 499                 len -= write_len;
 500         }
 501
 502         return 0;
 503 }
 504
 505 static int write_normal_skip_chunk(struct output_file *out, int64_t len)
 506 {
 507         return out->ops->skip(out, len);
 508 }
 509
 510 int write_normal_end_chunk(struct output_file *out)
 511 {
 512         return out->ops->pad(out, out->len);
 513 }
 514
 515 static struct sparse_file_ops normal_file_ops = {
 516                 .write_data_chunk = write_normal_data_chunk,
 517                 .write_fill_chunk = write_normal_fill_chunk,
 518                 .write_skip_chunk = write_normal_skip_chunk,
 519                 .write_end_chunk = write_normal_end_chunk,
 520 };
 521
 522 void output_file_close(struct output_file *out)
 523 {
 524         out->sparse_ops->write_end_chunk(out);
 525         out->ops->close(out);
 526 }
 527
 528 static int output_file_init(struct output_file *out, int block_size,
 529                 int64_t len, bool sparse, int chunks, bool crc)
 530 {
 531         int ret;
 532
 533         out->len = len;
 534         out->block_size = block_size;
 535         out->cur_out_ptr = 0ll;
 536         out->chunk_cnt = 0;
 537         out->crc32 = 0;
 538         out->use_crc = crc;
 539
 540         out->zero_buf = calloc(block_size, 1);
 541         if (!out->zero_buf) {
 542                 error_errno("malloc zero_buf");
 543                 return -ENOMEM;
 544         }
 545
 546         out->fill_buf = calloc(block_size, 1);
 547         if (!out->fill_buf) {
 548                 error_errno("malloc fill_buf");
 549                 ret = -ENOMEM;
 550                 goto err_fill_buf;
 551         }
 552
 553         if (sparse) {
 554                 out->sparse_ops = &sparse_file_ops;
 555         } else {
 556                 out->sparse_ops = &normal_file_ops;
 557         }
 558
 559         if (sparse) {
 560                 sparse_header_t sparse_header = {
 561                                 .magic = SPARSE_HEADER_MAGIC,
 562                                 .major_version = SPARSE_HEADER_MAJOR_VER,
 563                                 .minor_version = SPARSE_HEADER_MINOR_VER,
 564                                 .file_hdr_sz = SPARSE_HEADER_LEN,
 565                                 .chunk_hdr_sz = CHUNK_HEADER_LEN,
 566                                 .blk_sz = out->block_size,
 567                                 .total_blks = out->len / out->block_size,
 568                                 .total_chunks = chunks,
 569                                 .image_checksum = 0
 570                 };
 571
 572                 if (out->use_crc) {
 573                         sparse_header.total_chunks++;
 574                 }
 575
 576                 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
 577                 if (ret < 0) {
 578                         goto err_write;
 579                 }
 580         }
 581
 582         return 0;
 583
 584 err_write:
 585         free(out->fill_buf);
 586 err_fill_buf:
 587         free(out->zero_buf);
 588         return ret;
 589 }
 590
 591 static struct output_file *output_file_new_gz(void)
 592 {
 593         struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
 594         if (!outgz) {
 595                 error_errno("malloc struct outgz");
 596                 return NULL;
 597         }
 598
 599         outgz->out.ops = &gz_file_ops;
 600
 601         return &outgz->out;
 602 }
 603
 604 static struct output_file *output_file_new_normal(void)
 605 {
 606         struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
 607         if (!outn) {
 608                 error_errno("malloc struct outn");
 609                 return NULL;
 610         }
 611
 612         outn->out.ops = &file_ops;
 613
 614         return &outn->out;
 615 }
 616
 617 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int),
 618                 void *priv, unsigned int block_size, int64_t len,
 619                 int gz __unused, int sparse, int chunks, int crc)
 620 {
 621         int ret;
 622         struct output_file_callback *outc;
 623
 624         outc = calloc(1, sizeof(struct output_file_callback));
 625         if (!outc) {
 626                 error_errno("malloc struct outc");
 627                 return NULL;
 628         }
 629
 630         outc->out.ops = &callback_file_ops;
 631         outc->priv = priv;
 632         outc->write = write;
 633
 634         ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
 635         if (ret < 0) {
 636                 free(outc);
 637                 return NULL;
 638         }
 639
 640         return &outc->out;
 641 }
 642
 643 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
 644                 int gz, int sparse, int chunks, int crc)
 645 {
 646         int ret;
 647         struct output_file *out;
 648
 649         if (gz) {
 650                 out = output_file_new_gz();
 651         } else {
 652                 out = output_file_new_normal();
 653         }
 654         if (!out) {
 655                 return NULL;
 656         }
 657
 658         out->ops->open(out, fd);
 659
 660         ret = output_file_init(out, block_size, len, sparse, chunks, crc);
 661         if (ret < 0) {
 662                 free(out);
 663                 return NULL;
 664         }
 665
 666         return out;
 667 }
 668
 669 /* Write a contiguous region of data blocks from a memory buffer */
 670 int write_data_chunk(struct output_file *out, unsigned int len, void *data)
 671 {
 672         return out->sparse_ops->write_data_chunk(out, len, data);
 673 }
 674
 675 /* Write a contiguous region of data blocks with a fill value */
 676 int write_fill_chunk(struct output_file *out, unsigned int len,
 677                 uint32_t fill_val)
 678 {
 679         return out->sparse_ops->write_fill_chunk(out, len, fill_val);
 680 }
 681
 682 int write_fd_chunk(struct output_file *out, unsigned int len,
 683                 int fd, int64_t offset)
 684 {
 685         int ret;
 686         int64_t aligned_offset;
 687         int aligned_diff;
 688         int buffer_size;
 689         char *ptr;
 690
 691         aligned_offset = offset & ~(4096 - 1);
 692         aligned_diff = offset - aligned_offset;
 693         buffer_size = len + aligned_diff;
 694
 695         char *data = mmap(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
 696                         aligned_offset);
 697         if (data == MAP_FAILED) {
 698                 return -errno;
 699         }
 700         ptr = data + aligned_diff;
 701
 702         ret = out->sparse_ops->write_data_chunk(out, len, ptr);
 703
 704         munmap(data, buffer_size);
 705
 706         return ret;
 707 }
 708
 709 /* Write a contiguous region of data blocks from a file */
 710 int write_file_chunk(struct output_file *out, unsigned int len,
 711                 const char *file, int64_t offset)
 712 {
 713         int ret;
 714
 715         int file_fd = open(file, O_RDONLY);
 716         if (file_fd < 0) {
 717                 return -errno;
 718         }
 719
 720         ret = write_fd_chunk(out, len, file_fd, offset);
 721
 722         close(file_fd);
 723
 724         return ret;
 725 }
 726
 727 int write_skip_chunk(struct output_file *out, int64_t len)
 728 {
 729         return out->sparse_ops->write_skip_chunk(out, len);
 730 }