diff --git a/src/import.c b/src/import.c index f07307a..a21f24d 100644 --- a/src/import.c +++ b/src/import.c @@ -1932,8 +1932,7 @@ pthread_t thread_start(ldb_importation_config_t * job, pthread_t * tlist) } bool process_sectors(ldb_importation_config_t * job, pthread_t * tlist) { - DIR *dir; - struct dirent *ent; + struct dirent **ents; /*Process one file with multiples sectors inside*/ if (*job->csv_path) @@ -1953,25 +1952,25 @@ bool process_sectors(ldb_importation_config_t * job, pthread_t * tlist) { } } - /*Process a directory with one sector per file*/ - if ((dir = opendir(job->path)) != NULL) + /*Process a directory with one sector per file, in alphabetical order + (readdir returns the filesystem's order, e.g. hash order on ext4)*/ + int n = scandir(job->path, &ents, NULL, alphasort); + if (n >= 0) { - while ((ent = readdir(dir)) != NULL) + for (int i = 0; i < n; i++) { - if (!strcmp(ent->d_name,".") || !strcmp(ent->d_name,"..")) - continue; - - if (ent->d_type == DT_REG) + if (ents[i]->d_type == DT_REG) { - snprintf(job->csv_path, LDB_MAX_PATH, "%s/%s", job->path, ent->d_name); + snprintf(job->csv_path, LDB_MAX_PATH, "%s/%s", job->path, ents[i]->d_name); pthread_t tid = thread_start(job, tlist); if (tid == 0) ldb_import(job); - } + } + free(ents[i]); } - closedir(dir); - } + free(ents); + } else { log_info("Cannot open directory: %s\n", job->path); diff --git a/test/test_kb_crc64 b/test/test_kb_crc64 index 0f4e5e5..7610d65 100644 --- a/test/test_kb_crc64 +++ b/test/test_kb_crc64 @@ -268,6 +268,28 @@ test_16_collate_dedup_ignores_stale_slot_bytes() { rm -rf "$tmpdir" /var/lib/ldb/test_kbdup /usr/local/etc/scanoss/ldb/test_kbdup.conf /var/log/scanoss/ldb/test_kbdup.log } +#The files of a table folder are imported in alphabetical order, whatever order +#the filesystem lists them in (readdir: creation order or hash order), so an +#interrupted import leaves a clear boundary. The files are created in a mixed +#order and imported one at a time (THREADS=1): the log shows the start order. +test_17_import_files_in_alphabetical_order() { + rm -rf /var/lib/ldb/test_kborder /usr/local/etc/scanoss/ldb/test_kborder.conf /var/log/scanoss/ldb/test_kborder.log + tmpdir=$(mktemp -d) + mkdir -p "$tmpdir/t1" + for s in 80 ff 00 10 0a; do + printf '%s00000000000001,record_%s\n' "$s" "$s" > "$tmpdir/t1/$s.csv" + done + + echo "bulk insert test_kborder from $tmpdir with (KEYS=1,KEY_SIZE=8,FIELDS=2,SORT=0,THREADS=1,FILE_DEL=0,VALIDATE_VERSION=0)" | ../ldb -q + + order=$(grep -oE "t1/[0-9a-f]{2}\.csv: 1 records imported" /var/log/scanoss/ldb/test_kborder.log | cut -c4-5 | paste -sd' ') + assert_equals "00 0a 10 80 ff" "$order" "the files of the table were not imported in alphabetical order" + result=$(echo "select from test_kborder/t1 key 0a00000000000001 csv hex 8" | ../ldb -q) + assert_equals "0a00000000000001,record_0a" "$result" + + rm -rf "$tmpdir" /var/lib/ldb/test_kborder /usr/local/etc/scanoss/ldb/test_kborder.conf /var/log/scanoss/ldb/test_kborder.log +} + setup_suite () { echo 'bulk insert test_kb from source_crc64/mined with (KEY_SIZE=8)' | ../ldb } @@ -278,4 +300,5 @@ teardown_suite () { # test_14 leaves an ~86 MB sector behind when it fails before its own cleanup rm -rf /var/lib/ldb/test_kbcap /usr/local/etc/scanoss/ldb/test_kbcap.conf /var/log/scanoss/ldb/test_kbcap.log rm -rf /var/lib/ldb/test_kbdup /usr/local/etc/scanoss/ldb/test_kbdup.conf /var/log/scanoss/ldb/test_kbdup.log + rm -rf /var/lib/ldb/test_kborder /usr/local/etc/scanoss/ldb/test_kborder.conf /var/log/scanoss/ldb/test_kborder.log } \ No newline at end of file