libsparse/sparse_read.c

   1 /*
   2  * Copyright (C) 2012 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #define _GNU_SOURCE
  18 #define _FILE_OFFSET_BITS 64
  19 #define _LARGEFILE64_SOURCE 1
  20
  21 #include <fcntl.h>
  22 #include <stdarg.h>
  23 #include <stdbool.h>
  24 #include <stdint.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <string.h>
  28 #include <unistd.h>
  29
  30 #include <sparse/sparse.h>
  31
  32 #include "defs.h"
  33 #include "output_file.h"
  34 #include "sparse_crc32.h"
  35 #include "sparse_file.h"
  36 #include "sparse_format.h"
  37
  38 #if defined(__APPLE__) && defined(__MACH__)
  39 #define lseek64 lseek
  40 #define off64_t off_t
  41 #endif
  42
  43 #define SPARSE_HEADER_MAJOR_VER 1
  44 #define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
  45 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
  46
  47 #define COPY_BUF_SIZE (1024U*1024U)
  48 static char *copybuf;
  49
  50 #define min(a, b) \
  51         ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
  52
  53 static void verbose_error(bool verbose, int err, const char *fmt, ...)
  54 {
  55         char *s = "";
  56         char *at = "";
  57         if (fmt) {
  58                 va_list argp;
  59                 int size;
  60
  61                 va_start(argp, fmt);
  62                 size = vsnprintf(NULL, 0, fmt, argp);
  63                 va_end(argp);
  64
  65                 if (size < 0) {
  66                         return;
  67                 }
  68
  69                 at = malloc(size + 1);
  70                 if (at == NULL) {
  71                         return;
  72                 }
  73
  74                 va_start(argp, fmt);
  75                 vsnprintf(at, size, fmt, argp);
  76                 va_end(argp);
  77                 at[size] = 0;
  78                 s = " at ";
  79         }
  80         if (verbose) {
  81                 if (err == -EOVERFLOW) {
  82                         sparse_print_verbose("EOF while reading file%s%s\n", s, at);
  83                 } else if (err == -EINVAL) {
  84                         sparse_print_verbose("Invalid sparse file format%s%s\n", s, at);
  85                 } else if (err == -ENOMEM) {
  86                         sparse_print_verbose("Failed allocation while reading file%s%s\n",
  87                                         s, at);
  88                 } else {
  89                         sparse_print_verbose("Unknown error %d%s%s\n", err, s, at);
  90                 }
  91         }
  92         if (fmt) {
  93                 free(at);
  94         }
  95 }
  96
  97 static int process_raw_chunk(struct sparse_file *s, unsigned int chunk_size,
  98                 int fd, int64_t offset, unsigned int blocks, unsigned int block,
  99                 uint32_t *crc32)
 100 {
 101         int ret;
 102         int chunk;
 103         unsigned int len = blocks * s->block_size;
 104
 105         if (chunk_size % s->block_size != 0) {
 106                 return -EINVAL;
 107         }
 108
 109         if (chunk_size / s->block_size != blocks) {
 110                 return -EINVAL;
 111         }
 112
 113         ret = sparse_file_add_fd(s, fd, offset, len, block);
 114         if (ret < 0) {
 115                 return ret;
 116         }
 117
 118         if (crc32) {
 119                 while (len) {
 120                         chunk = min(len, COPY_BUF_SIZE);
 121                         ret = read_all(fd, copybuf, chunk);
 122                         if (ret < 0) {
 123                                 return ret;
 124                         }
 125                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 126                         len -= chunk;
 127                 }
 128         } else {
 129                 lseek64(fd, len, SEEK_CUR);
 130         }
 131
 132         return 0;
 133 }
 134
 135 static int process_fill_chunk(struct sparse_file *s, unsigned int chunk_size,
 136                 int fd, unsigned int blocks, unsigned int block, uint32_t *crc32)
 137 {
 138         int ret;
 139         int chunk;
 140         int64_t len = (int64_t)blocks * s->block_size;
 141         uint32_t fill_val;
 142         uint32_t *fillbuf;
 143         unsigned int i;
 144
 145         if (chunk_size != sizeof(fill_val)) {
 146                 return -EINVAL;
 147         }
 148
 149         ret = read_all(fd, &fill_val, sizeof(fill_val));
 150         if (ret < 0) {
 151                 return ret;
 152         }
 153
 154         ret = sparse_file_add_fill(s, fill_val, len, block);
 155         if (ret < 0) {
 156                 return ret;
 157         }
 158
 159         if (crc32) {
 160                 /* Fill copy_buf with the fill value */
 161                 fillbuf = (uint32_t *)copybuf;
 162                 for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
 163                         fillbuf[i] = fill_val;
 164                 }
 165
 166                 while (len) {
 167                         chunk = min(len, COPY_BUF_SIZE);
 168                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 169                         len -= chunk;
 170                 }
 171         }
 172
 173         return 0;
 174 }
 175
 176 static int process_skip_chunk(struct sparse_file *s, unsigned int chunk_size,
 177                 int fd __unused, unsigned int blocks,
 178                 unsigned int block __unused, uint32_t *crc32)
 179 {
 180         if (chunk_size != 0) {
 181                 return -EINVAL;
 182         }
 183
 184         if (crc32) {
 185                 int64_t len = (int64_t)blocks * s->block_size;
 186                 memset(copybuf, 0, COPY_BUF_SIZE);
 187
 188                 while (len) {
 189                         int chunk = min(len, COPY_BUF_SIZE);
 190                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 191                         len -= chunk;
 192                 }
 193         }
 194
 195         return 0;
 196 }
 197
 198 static int process_crc32_chunk(int fd, unsigned int chunk_size, uint32_t crc32)
 199 {
 200         uint32_t file_crc32;
 201         int ret;
 202
 203         if (chunk_size != sizeof(file_crc32)) {
 204                 return -EINVAL;
 205         }
 206
 207         ret = read_all(fd, &file_crc32, sizeof(file_crc32));
 208         if (ret < 0) {
 209                 return ret;
 210         }
 211
 212         if (file_crc32 != crc32) {
 213                 return -EINVAL;
 214         }
 215
 216         return 0;
 217 }
 218
 219 static int process_chunk(struct sparse_file *s, int fd, off64_t offset,
 220                 unsigned int chunk_hdr_sz, chunk_header_t *chunk_header,
 221                 unsigned int cur_block, uint32_t *crc_ptr)
 222 {
 223         int ret;
 224         unsigned int chunk_data_size;
 225
 226         chunk_data_size = chunk_header->total_sz - chunk_hdr_sz;
 227
 228         switch (chunk_header->chunk_type) {
 229                 case CHUNK_TYPE_RAW:
 230                         ret = process_raw_chunk(s, chunk_data_size, fd, offset,
 231                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 232                         if (ret < 0) {
 233                                 verbose_error(s->verbose, ret, "data block at %lld", offset);
 234                                 return ret;
 235                         }
 236                         return chunk_header->chunk_sz;
 237                 case CHUNK_TYPE_FILL:
 238                         ret = process_fill_chunk(s, chunk_data_size, fd,
 239                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 240                         if (ret < 0) {
 241                                 verbose_error(s->verbose, ret, "fill block at %lld", offset);
 242                                 return ret;
 243                         }
 244                         return chunk_header->chunk_sz;
 245                 case CHUNK_TYPE_DONT_CARE:
 246                         ret = process_skip_chunk(s, chunk_data_size, fd,
 247                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 248                         if (chunk_data_size != 0) {
 249                                 if (ret < 0) {
 250                                         verbose_error(s->verbose, ret, "skip block at %lld", offset);
 251                                         return ret;
 252                                 }
 253                         }
 254                         return chunk_header->chunk_sz;
 255                 case CHUNK_TYPE_CRC32:
 256                         ret = process_crc32_chunk(fd, chunk_data_size, *crc_ptr);
 257                         if (ret < 0) {
 258                                 verbose_error(s->verbose, -EINVAL, "crc block at %lld",
 259                                                 offset);
 260                                 return ret;
 261                         }
 262                         return 0;
 263                 default:
 264                         verbose_error(s->verbose, -EINVAL, "unknown block %04X at %lld",
 265                                         chunk_header->chunk_type, offset);
 266         }
 267
 268         return 0;
 269 }
 270
 271 static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc)
 272 {
 273         int ret;
 274         unsigned int i;
 275         sparse_header_t sparse_header;
 276         chunk_header_t chunk_header;
 277         uint32_t crc32 = 0;
 278         uint32_t *crc_ptr = 0;
 279         unsigned int cur_block = 0;
 280         off64_t offset;
 281
 282         if (!copybuf) {
 283                 copybuf = malloc(COPY_BUF_SIZE);
 284         }
 285
 286         if (!copybuf) {
 287                 return -ENOMEM;
 288         }
 289
 290         if (crc) {
 291                 crc_ptr = &crc32;
 292         }
 293
 294         ret = read_all(fd, &sparse_header, sizeof(sparse_header));
 295         if (ret < 0) {
 296                 return ret;
 297         }
 298
 299         if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
 300                 return -EINVAL;
 301         }
 302
 303         if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
 304                 return -EINVAL;
 305         }
 306
 307         if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
 308                 return -EINVAL;
 309         }
 310
 311         if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) {
 312                 return -EINVAL;
 313         }
 314
 315         if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
 316                 /* Skip the remaining bytes in a header that is longer than
 317                  * we expected.
 318                  */
 319                 lseek64(fd, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR);
 320         }
 321
 322         for (i = 0; i < sparse_header.total_chunks; i++) {
 323                 ret = read_all(fd, &chunk_header, sizeof(chunk_header));
 324                 if (ret < 0) {
 325                         return ret;
 326                 }
 327
 328                 if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
 329                         /* Skip the remaining bytes in a header that is longer than
 330                          * we expected.
 331                          */
 332                         lseek64(fd, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR);
 333                 }
 334
 335                 offset = lseek64(fd, 0, SEEK_CUR);
 336
 337                 ret = process_chunk(s, fd, offset, sparse_header.chunk_hdr_sz, &chunk_header,
 338                                 cur_block, crc_ptr);
 339                 if (ret < 0) {
 340                         return ret;
 341                 }
 342
 343                 cur_block += ret;
 344         }
 345
 346         if (sparse_header.total_blks != cur_block) {
 347                 return -EINVAL;
 348         }
 349
 350         return 0;
 351 }
 352
 353 static int sparse_file_read_normal(struct sparse_file *s, int fd)
 354 {
 355         int ret;
 356         uint32_t *buf = malloc(s->block_size);
 357         unsigned int block = 0;
 358         int64_t remain = s->len;
 359         int64_t offset = 0;
 360         unsigned int to_read;
 361         unsigned int i;
 362         bool sparse_block;
 363
 364         if (!buf) {
 365                 return -ENOMEM;
 366         }
 367
 368         while (remain > 0) {
 369                 to_read = min(remain, s->block_size);
 370                 ret = read_all(fd, buf, to_read);
 371                 if (ret < 0) {
 372                         error("failed to read sparse file");
 373                         return ret;
 374                 }
 375
 376                 if (to_read == s->block_size) {
 377                         sparse_block = true;
 378                         for (i = 1; i < s->block_size / sizeof(uint32_t); i++) {
 379                                 if (buf[0] != buf[i]) {
 380                                         sparse_block = false;
 381                                         break;
 382                                 }
 383                         }
 384                 } else {
 385                         sparse_block = false;
 386                 }
 387
 388                 if (sparse_block) {
 389                         /* TODO: add flag to use skip instead of fill for buf[0] == 0 */
 390                         sparse_file_add_fill(s, buf[0], to_read, block);
 391                 } else {
 392                         sparse_file_add_fd(s, fd, offset, to_read, block);
 393                 }
 394
 395                 remain -= to_read;
 396                 offset += to_read;
 397                 block++;
 398         }
 399
 400         return 0;
 401 }
 402
 403 int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
 404 {
 405         if (crc && !sparse) {
 406                 return -EINVAL;
 407         }
 408
 409         if (sparse) {
 410                 return sparse_file_read_sparse(s, fd, crc);
 411         } else {
 412                 return sparse_file_read_normal(s, fd);
 413         }
 414 }
 415
 416 struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc)
 417 {
 418         int ret;
 419         sparse_header_t sparse_header;
 420         int64_t len;
 421         struct sparse_file *s;
 422
 423         ret = read_all(fd, &sparse_header, sizeof(sparse_header));
 424         if (ret < 0) {
 425                 verbose_error(verbose, ret, "header");
 426                 return NULL;
 427         }
 428
 429         if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
 430                 verbose_error(verbose, -EINVAL, "header magic");
 431                 return NULL;
 432         }
 433
 434         if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
 435                 verbose_error(verbose, -EINVAL, "header major version");
 436                 return NULL;
 437         }
 438
 439         if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
 440                 return NULL;
 441         }
 442
 443         if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) {
 444                 return NULL;
 445         }
 446
 447         len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz;
 448         s = sparse_file_new(sparse_header.blk_sz, len);
 449         if (!s) {
 450                 verbose_error(verbose, -EINVAL, NULL);
 451                 return NULL;
 452         }
 453
 454         ret = lseek64(fd, 0, SEEK_SET);
 455         if (ret < 0) {
 456                 verbose_error(verbose, ret, "seeking");
 457                 sparse_file_destroy(s);
 458                 return NULL;
 459         }
 460
 461         s->verbose = verbose;
 462
 463         ret = sparse_file_read(s, fd, true, crc);
 464         if (ret < 0) {
 465                 sparse_file_destroy(s);
 466                 return NULL;
 467         }
 468
 469         return s;
 470 }
 471
 472 struct sparse_file *sparse_file_import_auto(int fd, bool crc, bool verbose)
 473 {
 474         struct sparse_file *s;
 475         int64_t len;
 476         int ret;
 477
 478         s = sparse_file_import(fd, verbose, crc);
 479         if (s) {
 480                 return s;
 481         }
 482
 483         len = lseek64(fd, 0, SEEK_END);
 484         if (len < 0) {
 485                 return NULL;
 486         }
 487
 488         lseek64(fd, 0, SEEK_SET);
 489
 490         s = sparse_file_new(4096, len);
 491         if (!s) {
 492                 return NULL;
 493         }
 494
 495         ret = sparse_file_read_normal(s, fd);
 496         if (ret < 0) {
 497                 sparse_file_destroy(s);
 498                 return NULL;
 499         }
 500
 501         return s;
 502 }