Support tabs as seaprators when parsing fs config
[project/make_ext4fs.git] / make_ext4fs.c
1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 #include "make_ext4fs.h"
18 #include "ext4_utils.h"
19 #include "allocate.h"
20 #include "contents.h"
21 #include "uuid.h"
22 #include "wipe.h"
23
24 #include <sparse/sparse.h>
25
26 #include <assert.h>
27 #include <dirent.h>
28 #include <fcntl.h>
29 #include <inttypes.h>
30 #include <libgen.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35 #include <sys/stat.h>
36 #include <sys/types.h>
37
38 /* TODO: Not implemented:
39    Allocating blocks in the same block group as the file inode
40    Hash or binary tree directories
41  */
42
43 static int filter_dot(const struct dirent *d)
44 {
45         return (strcmp(d->d_name, "..") && strcmp(d->d_name, "."));
46 }
47
48 static u32 build_default_directory_structure(const char *dir_path)
49 {
50         u32 inode;
51         u32 root_inode;
52         struct dentry dentries = {
53                         .filename = "lost+found",
54                         .file_type = EXT4_FT_DIR,
55                         .mode = S_IRWXU,
56                         .uid = 0,
57                         .gid = 0,
58                         .mtime = 0,
59         };
60         root_inode = make_directory(0, 1, &dentries, 1);
61         inode = make_directory(root_inode, 0, NULL, 0);
62         *dentries.inode = inode;
63         inode_set_permissions(inode, dentries.mode,
64                 dentries.uid, dentries.gid, dentries.mtime);
65
66         return root_inode;
67 }
68
69 #ifndef USE_MINGW
70 /* Read a local directory and create the same tree in the generated filesystem.
71    Calls itself recursively with each directory in the given directory.
72    full_path is an absolute or relative path, with a trailing slash, to the
73    directory on disk that should be copied, or NULL if this is a directory
74    that does not exist on disk (e.g. lost+found).
75    dir_path is an absolute path, with trailing slash, to the same directory
76    if the image were mounted at the specified mount point */
77 static u32 build_directory_structure(const char *full_path, const char *dir_path,
78                 u32 dir_inode, fs_config_func_t fs_config_func,
79                 int verbose, time_t fixed_time)
80 {
81         int entries = 0;
82         struct dentry *dentries;
83         struct dirent **namelist = NULL;
84         struct stat stat;
85         int ret;
86         int i;
87         u32 inode;
88         u32 entry_inode;
89         u32 dirs = 0;
90         bool needs_lost_and_found = false;
91
92         if (full_path) {
93                 entries = scandir(full_path, &namelist, filter_dot, (void*)alphasort);
94                 if (entries < 0) {
95 #ifdef __GLIBC__
96                         /* The scandir function implemented in glibc has a bug that makes it
97                            erroneously fail with ENOMEM under certain circumstances.
98                            As a workaround we can retry the scandir call with the same arguments.
99                            GLIBC BZ: https://sourceware.org/bugzilla/show_bug.cgi?id=17804 */
100                         if (errno == ENOMEM)
101                                 entries = scandir(full_path, &namelist, filter_dot, (void*)alphasort);
102 #endif
103                         if (entries < 0) {
104                                 error_errno("scandir");
105                                 return EXT4_ALLOCATE_FAILED;
106                         }
107                 }
108         }
109
110         if (dir_inode == 0) {
111                 /* root directory, check if lost+found already exists */
112                 for (i = 0; i < entries; i++)
113                         if (strcmp(namelist[i]->d_name, "lost+found") == 0)
114                                 break;
115                 if (i == entries)
116                         needs_lost_and_found = true;
117         }
118
119         dentries = calloc(entries, sizeof(struct dentry));
120         if (dentries == NULL)
121                 critical_error_errno("malloc");
122
123         for (i = 0; i < entries; i++) {
124                 dentries[i].filename = strdup(namelist[i]->d_name);
125                 if (dentries[i].filename == NULL)
126                         critical_error_errno("strdup");
127
128                 asprintf(&dentries[i].path, "%s%s", dir_path, namelist[i]->d_name);
129                 asprintf(&dentries[i].full_path, "%s%s", full_path, namelist[i]->d_name);
130
131                 free(namelist[i]);
132
133                 ret = lstat(dentries[i].full_path, &stat);
134                 if (ret < 0) {
135                         error_errno("lstat");
136                         i--;
137                         entries--;
138                         continue;
139                 }
140
141                 dentries[i].size = stat.st_size;
142                 dentries[i].mode = stat.st_mode & (S_ISUID|S_ISGID|S_ISVTX|S_IRWXU|S_IRWXG|S_IRWXO);
143                 if (fixed_time == -1) {
144                         dentries[i].mtime = stat.st_mtime;
145                 } else {
146                         dentries[i].mtime = fixed_time;
147                 }
148                 uint64_t capabilities;
149                 if (fs_config_func != NULL) {
150                         unsigned int mode = 0;
151                         unsigned int uid = 0;
152                         unsigned int gid = 0;
153                         int dir = S_ISDIR(stat.st_mode);
154                         if (fs_config_func(dentries[i].path, dir, &uid, &gid, &mode, &capabilities)) {
155                                 dentries[i].mode = mode;
156                                 dentries[i].uid = uid;
157                                 dentries[i].gid = gid;
158                                 dentries[i].capabilities = capabilities;
159                         }
160                 }
161
162                 if (S_ISREG(stat.st_mode)) {
163                         dentries[i].file_type = EXT4_FT_REG_FILE;
164                 } else if (S_ISDIR(stat.st_mode)) {
165                         dentries[i].file_type = EXT4_FT_DIR;
166                         dirs++;
167                 } else if (S_ISCHR(stat.st_mode)) {
168                         dentries[i].file_type = EXT4_FT_CHRDEV;
169                 } else if (S_ISBLK(stat.st_mode)) {
170                         dentries[i].file_type = EXT4_FT_BLKDEV;
171                 } else if (S_ISFIFO(stat.st_mode)) {
172                         dentries[i].file_type = EXT4_FT_FIFO;
173                 } else if (S_ISSOCK(stat.st_mode)) {
174                         dentries[i].file_type = EXT4_FT_SOCK;
175                 } else if (S_ISLNK(stat.st_mode)) {
176                         dentries[i].file_type = EXT4_FT_SYMLINK;
177                         dentries[i].link = calloc(info.block_size, 1);
178                         readlink(dentries[i].full_path, dentries[i].link, info.block_size - 1);
179                 } else {
180                         error("unknown file type on %s", dentries[i].path);
181                         i--;
182                         entries--;
183                 }
184         }
185         free(namelist);
186
187         if (needs_lost_and_found) {
188                 /* insert a lost+found directory at the beginning of the dentries */
189                 struct dentry *tmp = calloc(entries + 1, sizeof(struct dentry));
190                 memset(tmp, 0, sizeof(struct dentry));
191                 memcpy(tmp + 1, dentries, entries * sizeof(struct dentry));
192                 dentries = tmp;
193
194                 dentries[0].filename = strdup("lost+found");
195                 asprintf(&dentries[0].path, "%slost+found", dir_path);
196                 dentries[0].full_path = NULL;
197                 dentries[0].size = 0;
198                 dentries[0].mode = S_IRWXU;
199                 dentries[0].file_type = EXT4_FT_DIR;
200                 dentries[0].uid = 0;
201                 dentries[0].gid = 0;
202                 entries++;
203                 dirs++;
204         }
205
206         inode = make_directory(dir_inode, entries, dentries, dirs);
207
208         for (i = 0; i < entries; i++) {
209                 if (dentries[i].file_type == EXT4_FT_REG_FILE) {
210                         entry_inode = make_file(dentries[i].full_path, dentries[i].size);
211                 } else if (dentries[i].file_type == EXT4_FT_DIR) {
212                         char *subdir_full_path = NULL;
213                         char *subdir_dir_path;
214                         if (dentries[i].full_path) {
215                                 ret = asprintf(&subdir_full_path, "%s/", dentries[i].full_path);
216                                 if (ret < 0)
217                                         critical_error_errno("asprintf");
218                         }
219                         ret = asprintf(&subdir_dir_path, "%s/", dentries[i].path);
220                         if (ret < 0)
221                                 critical_error_errno("asprintf");
222                         entry_inode = build_directory_structure(subdir_full_path,
223                                         subdir_dir_path, inode, fs_config_func, verbose, fixed_time);
224                         free(subdir_full_path);
225                         free(subdir_dir_path);
226                 } else if (dentries[i].file_type == EXT4_FT_SYMLINK) {
227                         entry_inode = make_link(dentries[i].link);
228                 } else if (dentries[i].file_type == EXT4_FT_CHRDEV ||
229                            dentries[i].file_type == EXT4_FT_BLKDEV ||
230                            dentries[i].file_type == EXT4_FT_SOCK ||
231                            dentries[i].file_type == EXT4_FT_FIFO) {
232                         entry_inode = make_special(dentries[i].full_path);
233                 } else {
234                         error("unknown file type on %s", dentries[i].path);
235                         entry_inode = 0;
236                 }
237                 *dentries[i].inode = entry_inode;
238
239                 ret = inode_set_permissions(entry_inode, dentries[i].mode,
240                         dentries[i].uid, dentries[i].gid,
241                         dentries[i].mtime);
242                 if (ret)
243                         error("failed to set permissions on %s\n", dentries[i].path);
244
245                 /*
246                  * It's important to call inode_set_selinux() before
247                  * inode_set_capabilities(). Extended attributes need to
248                  * be stored sorted order, and we guarantee this by making
249                  * the calls in the proper order.
250                  * Please see xattr_assert_sane() in contents.c
251                  */
252                 ret = inode_set_selinux(entry_inode, dentries[i].secon);
253                 if (ret)
254                         error("failed to set SELinux context on %s\n", dentries[i].path);
255                 ret = inode_set_capabilities(entry_inode, dentries[i].capabilities);
256                 if (ret)
257                         error("failed to set capability on %s\n", dentries[i].path);
258
259                 free(dentries[i].path);
260                 free(dentries[i].full_path);
261                 free(dentries[i].link);
262                 free((void *)dentries[i].filename);
263                 free(dentries[i].secon);
264         }
265
266         free(dentries);
267         return inode;
268 }
269 #endif
270
271 static u32 compute_block_size()
272 {
273         return 4096;
274 }
275
276 static u32 compute_journal_blocks()
277 {
278         u32 journal_blocks = DIV_ROUND_UP(info.len, info.block_size) / 64;
279         if (journal_blocks < 1024)
280                 journal_blocks = 1024;
281         if (journal_blocks > 32768)
282                 journal_blocks = 32768;
283         return journal_blocks;
284 }
285
286 static u32 compute_blocks_per_group()
287 {
288         return info.block_size * 8;
289 }
290
291 static u32 compute_inodes()
292 {
293         return DIV_ROUND_UP(info.len, info.block_size) / 4;
294 }
295
296 static u32 compute_inodes_per_group()
297 {
298         u32 blocks = DIV_ROUND_UP(info.len, info.block_size);
299         u32 block_groups = DIV_ROUND_UP(blocks, info.blocks_per_group);
300         u32 inodes = DIV_ROUND_UP(info.inodes, block_groups);
301         inodes = EXT4_ALIGN(inodes, (info.block_size / info.inode_size));
302
303         /* After properly rounding up the number of inodes/group,
304          * make sure to update the total inodes field in the info struct.
305          */
306         info.inodes = inodes * block_groups;
307
308         return inodes;
309 }
310
311 static u32 compute_bg_desc_reserve_blocks()
312 {
313         u32 blocks = DIV_ROUND_UP(info.len, info.block_size);
314         u32 block_groups = DIV_ROUND_UP(blocks, info.blocks_per_group);
315         u32 bg_desc_blocks = DIV_ROUND_UP(block_groups * sizeof(struct ext2_group_desc),
316                         info.block_size);
317
318         u32 bg_desc_reserve_blocks =
319                         DIV_ROUND_UP(block_groups * 1024 * sizeof(struct ext2_group_desc),
320                                         info.block_size) - bg_desc_blocks;
321
322         if (bg_desc_reserve_blocks > info.block_size / sizeof(u32))
323                 bg_desc_reserve_blocks = info.block_size / sizeof(u32);
324
325         return bg_desc_reserve_blocks;
326 }
327
328 void reset_ext4fs_info() {
329         // Reset all the global data structures used by make_ext4fs so it
330         // can be called again.
331         memset(&info, 0, sizeof(info));
332         memset(&aux_info, 0, sizeof(aux_info));
333
334         if (ext4_sparse_file) {
335                 sparse_file_destroy(ext4_sparse_file);
336                 ext4_sparse_file = NULL;
337         }
338 }
339
340 int make_ext4fs_sparse_fd(int fd, long long len,
341                                 const char *mountpoint)
342 {
343         reset_ext4fs_info();
344         info.len = len;
345
346         return make_ext4fs_internal(fd, NULL, mountpoint, NULL, 0, 1, 0, 0, 0, -1, NULL);
347 }
348
349 int make_ext4fs(const char *filename, long long len,
350                                 const char *mountpoint)
351 {
352         int fd;
353         int status;
354
355         reset_ext4fs_info();
356         info.len = len;
357
358         fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644);
359         if (fd < 0) {
360                 error_errno("open");
361                 return EXIT_FAILURE;
362         }
363
364         status = make_ext4fs_internal(fd, NULL, mountpoint, NULL, 0, 0, 0, 1, 0, -1, NULL);
365         close(fd);
366
367         return status;
368 }
369
370 /* return a newly-malloc'd string that is a copy of str.  The new string
371    is guaranteed to have a trailing slash.  If absolute is true, the new string
372    is also guaranteed to have a leading slash.
373 */
374 static char *canonicalize_slashes(const char *str, bool absolute)
375 {
376         char *ret;
377         int len = strlen(str);
378         int newlen = len;
379         char *ptr;
380
381         if (len == 0) {
382                 if (absolute)
383                         return strdup("/");
384                 else
385                         return strdup("");
386         }
387
388         if (str[0] != '/' && absolute) {
389                 newlen++;
390         }
391         if (str[len - 1] != '/') {
392                 newlen++;
393         }
394         ret = malloc(newlen + 1);
395         if (!ret) {
396                 critical_error("malloc");
397         }
398
399         ptr = ret;
400         if (str[0] != '/' && absolute) {
401                 *ptr++ = '/';
402         }
403
404         strcpy(ptr, str);
405         ptr += len;
406
407         if (str[len - 1] != '/') {
408                 *ptr++ = '/';
409         }
410
411         if (ptr != ret + newlen) {
412                 critical_error("assertion failed\n");
413         }
414
415         *ptr = '\0';
416
417         return ret;
418 }
419
420 static char *canonicalize_abs_slashes(const char *str)
421 {
422         return canonicalize_slashes(str, true);
423 }
424
425 static char *canonicalize_rel_slashes(const char *str)
426 {
427         return canonicalize_slashes(str, false);
428 }
429
430 int make_ext4fs_internal(int fd, const char *_directory,
431                                                  const char *_mountpoint, fs_config_func_t fs_config_func, int gzip,
432                                                  int sparse, int crc, int wipe,
433                                                  int verbose, time_t fixed_time,
434                                                  FILE* block_list_file)
435 {
436         u32 root_inode_num;
437         u16 root_mode;
438         char *mountpoint;
439         char *directory = NULL;
440
441         if (setjmp(setjmp_env))
442                 return EXIT_FAILURE; /* Handle a call to longjmp() */
443
444         if (_mountpoint == NULL) {
445                 mountpoint = strdup("");
446         } else {
447                 mountpoint = canonicalize_abs_slashes(_mountpoint);
448         }
449
450         if (_directory) {
451                 directory = canonicalize_rel_slashes(_directory);
452         }
453
454         if (info.len <= 0)
455                 info.len = get_file_size(fd);
456
457         if (info.len <= 0) {
458                 fprintf(stderr, "Need size of filesystem\n");
459                 return EXIT_FAILURE;
460         }
461
462         if (info.block_size <= 0)
463                 info.block_size = compute_block_size();
464
465         /* Round down the filesystem length to be a multiple of the block size */
466         info.len &= ~((u64)info.block_size - 1);
467
468         if (info.journal_blocks == 0)
469                 info.journal_blocks = compute_journal_blocks();
470
471         if (info.no_journal == 0)
472                 info.feat_compat = EXT4_FEATURE_COMPAT_HAS_JOURNAL;
473         else
474                 info.journal_blocks = 0;
475
476         if (info.blocks_per_group <= 0)
477                 info.blocks_per_group = compute_blocks_per_group();
478
479         if (info.inodes <= 0)
480                 info.inodes = compute_inodes();
481
482         if (info.inode_size <= 0)
483                 info.inode_size = 256;
484
485         if (info.label == NULL)
486                 info.label = "";
487
488         info.inodes_per_group = compute_inodes_per_group();
489
490         info.feat_compat |=
491                         EXT4_FEATURE_COMPAT_RESIZE_INODE |
492                         EXT4_FEATURE_COMPAT_EXT_ATTR;
493
494         info.feat_ro_compat |=
495                         EXT4_FEATURE_RO_COMPAT_SPARSE_SUPER |
496                         EXT4_FEATURE_RO_COMPAT_LARGE_FILE |
497                         EXT4_FEATURE_RO_COMPAT_GDT_CSUM;
498
499         info.feat_incompat |=
500                         EXT4_FEATURE_INCOMPAT_EXTENTS |
501                         EXT4_FEATURE_INCOMPAT_FILETYPE;
502
503
504         info.bg_desc_reserve_blocks = compute_bg_desc_reserve_blocks();
505
506         printf("Creating filesystem with parameters:\n");
507         printf("    Size: %"PRIu64"\n", info.len);
508         printf("    Block size: %d\n", info.block_size);
509         printf("    Blocks per group: %d\n", info.blocks_per_group);
510         printf("    Inodes per group: %d\n", info.inodes_per_group);
511         printf("    Inode size: %d\n", info.inode_size);
512         printf("    Journal blocks: %d\n", info.journal_blocks);
513         printf("    Label: %s\n", info.label);
514
515         ext4_create_fs_aux_info();
516
517         printf("    Blocks: %"PRIu64"\n", aux_info.len_blocks);
518         printf("    Block groups: %d\n", aux_info.groups);
519         printf("    Reserved block group size: %d\n", info.bg_desc_reserve_blocks);
520
521         ext4_sparse_file = sparse_file_new(info.block_size, info.len);
522
523         block_allocator_init();
524
525         ext4_fill_in_sb();
526
527         if (reserve_inodes(0, 10) == EXT4_ALLOCATE_FAILED)
528                 error("failed to reserve first 10 inodes");
529
530         if (info.feat_compat & EXT4_FEATURE_COMPAT_HAS_JOURNAL)
531                 ext4_create_journal_inode();
532
533         if (info.feat_compat & EXT4_FEATURE_COMPAT_RESIZE_INODE)
534                 ext4_create_resize_inode();
535
536 #ifdef USE_MINGW
537         // Windows needs only 'create an empty fs image' functionality
538         assert(!directory);
539         root_inode_num = build_default_directory_structure(mountpoint);
540 #else
541         if (directory)
542                 root_inode_num = build_directory_structure(directory, mountpoint, 0,
543                         fs_config_func, verbose, fixed_time);
544         else
545                 root_inode_num = build_default_directory_structure(mountpoint);
546 #endif
547
548         root_mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
549         inode_set_permissions(root_inode_num, root_mode, 0, 0, 0);
550
551         ext4_update_free();
552
553         ext4_queue_sb();
554
555         if (block_list_file) {
556                 size_t dirlen = directory ? strlen(directory) : 0;
557                 struct block_allocation* p = get_saved_allocation_chain();
558                 while (p) {
559                         if (directory && strncmp(p->filename, directory, dirlen) == 0) {
560                                 // substitute mountpoint for the leading directory in the filename, in the output file
561                                 fprintf(block_list_file, "%s%s", mountpoint, p->filename + dirlen);
562                         } else {
563                                 fprintf(block_list_file, "%s", p->filename);
564                         }
565                         print_blocks(block_list_file, p);
566                         struct block_allocation* pn = p->next;
567                         free_alloc(p);
568                         p = pn;
569                 }
570         }
571
572         printf("Created filesystem with %d/%d inodes and %d/%d blocks\n",
573                         aux_info.sb->s_inodes_count - aux_info.sb->s_free_inodes_count,
574                         aux_info.sb->s_inodes_count,
575                         aux_info.sb->s_blocks_count_lo - aux_info.sb->s_free_blocks_count_lo,
576                         aux_info.sb->s_blocks_count_lo);
577
578         if (wipe && WIPE_IS_SUPPORTED) {
579                 wipe_block_device(fd, info.len);
580         }
581
582         write_ext4_image(fd, gzip, sparse, crc);
583
584         sparse_file_destroy(ext4_sparse_file);
585         ext4_sparse_file = NULL;
586
587         free(mountpoint);
588         free(directory);
589
590         return 0;
591 }