Skip to content

Fix output-dir-flat collisions with --rm #4855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xiaobai0529 wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Fix output-dir-flat collisions with --rm #4855

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 35 additions & 9 deletions programs/fileio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1142,11 +1142,13 @@ static void FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* co


/* FIO_checkFilenameCollisions() :
* Checks for and warns if there are any files that would have the same output path
* Checks for files that would have the same output path.
* Returns non-zero if a collision is fatal, i.e. when --rm is enabled.
*/
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles, int removeSrcFile) {
const char **filenameTableSorted, *prevElem, *filename;
unsigned u;
int collision = 0;

filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
if (!filenameTableSorted) {
Expand All @@ -1156,6 +1158,11 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {

for (u = 0; u < nbFiles; ++u) {
filename = strrchr(filenameTable[u], PATH_SEP);
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
{ const char* const slash = strrchr(filenameTable[u], '/');
if (slash > filename) filename = slash;
}
#endif
if (filename == NULL) {
filenameTableSorted[u] = filenameTable[u];
} else {
Expand All @@ -1166,14 +1173,25 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
prevElem = filenameTableSorted[0];
for (u = 1; u < nbFiles; ++u) {
if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
DISPLAYLEVEL(2, "WARNING: Two files have same filename: %s\n", prevElem);
int const sameName =
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
_stricmp(prevElem, filenameTableSorted[u]) == 0;
#else
strcmp(prevElem, filenameTableSorted[u]) == 0;
#endif
if (sameName) {
if (removeSrcFile) {
DISPLAYLEVEL(1, "ERROR: Two files have same filename: %s\n", prevElem);
collision = 1;
} else {
DISPLAYLEVEL(2, "WARNING: Two files have same filename: %s\n", prevElem);
}
}
prevElem = filenameTableSorted[u];
}

free((void*)filenameTableSorted);
return 0;
return collision;
}

static const char*
Expand Down Expand Up @@ -2512,6 +2530,12 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);

if (outDirName && FIO_checkFilenameCollisions(inFileNamesTable, (unsigned)fCtx->nbFilesTotal,
prefs->removeSrcFile)) {
FIO_freeCResources(&ress);
return 1;
}

for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) {
const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx];
const char* dstFileName = NULL;
Expand All @@ -2533,8 +2557,6 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
error |= status;
}

if (outDirName)
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
}

if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
Expand Down Expand Up @@ -3367,6 +3389,12 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);

if (outDirName && FIO_checkFilenameCollisions(srcNamesTable, (unsigned)fCtx->nbFilesTotal,
prefs->removeSrcFile)) {
FIO_freeDResources(ress);
return 1;
}

for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */
const char* const srcFileName = srcNamesTable[fCtx->currFileIdx];
const char* dstFileName = NULL;
Expand All @@ -3386,8 +3414,6 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
if (!status) fCtx->nbFilesProcessed++;
error |= status;
}
if (outDirName)
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
}

if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
Expand Down
5 changes: 3 additions & 2 deletions programs/fileio.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,10 @@ int FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
const char* dictFileName);

/* FIO_checkFilenameCollisions() :
* Checks for and warns if there are any files that would have the same output path
* Checks for files that would have the same output path.
* Returns non-zero if a collision is fatal, i.e. when --rm is enabled.
*/
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles, int removeSrcFile);



Expand Down
40 changes: 40 additions & 0 deletions tests/cli-tests/basic/output_dir_collision.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/sh

set -e

mkdir d1 d2 out
printf 'first\n' > d1/x
printf 'second\n' > d2/x

zstd -f --rm --output-dir-flat out d1/x d2/x && die "Collision with --rm should fail!"
test -f d1/x
test -f d2/x
test ! -e out/x.zst

zstd -f --output-dir-flat out d1/x d2/x
test -f d1/x
test -f d2/x
test -f out/x.zst

mkdir dz1 dz2 dout
printf 'first\n' | zstd -q > dz1/x.zst
printf 'second\n' | zstd -q > dz2/x.zst

zstd -d -f --rm --output-dir-flat dout dz1/x.zst dz2/x.zst && die "Decompression collision with --rm should fail!"
test -f dz1/x.zst
test -f dz2/x.zst
test ! -e dout/x

zstd -d -f --output-dir-flat dout dz1/x.zst dz2/x.zst
test -f dz1/x.zst
test -f dz2/x.zst
test -f dout/x

if [ "${OS:-}" = "Windows_NT" ]; then
printf 'first case\n' > d1/A
printf 'second case\n' > d2/a
zstd -f --rm --output-dir-flat out d1/A d2/a && die "Case collision with --rm should fail!"
test -f d1/A
test -f d2/a
fi
exit 0
1 change: 1 addition & 0 deletions tests/cli-tests/basic/output_dir_collision.sh.exit
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0