diff --git a/opendj-cli/src/main/java/com/forgerock/opendj/cli/ArgumentParser.java b/opendj-cli/src/main/java/com/forgerock/opendj/cli/ArgumentParser.java index 548cb3c0f1..043ab8a005 100644 --- a/opendj-cli/src/main/java/com/forgerock/opendj/cli/ArgumentParser.java +++ b/opendj-cli/src/main/java/com/forgerock/opendj/cli/ArgumentParser.java @@ -23,10 +23,18 @@ import static com.forgerock.opendj.cli.Utils.*; import static com.forgerock.opendj.util.StaticUtils.*; +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.File; -import java.io.FileInputStream; +import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; +import java.io.StringReader; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -411,10 +419,7 @@ Properties checkExternalProperties() throws ArgumentException { try { final Properties argumentProperties = new Properties(); final String scriptName = getScriptName(); - final Properties p = new Properties(); - try (final FileInputStream fis = new FileInputStream(propertiesFilePath)) { - p.load(fis); - } + final Properties p = loadPropertiesFile(propertiesFilePath); for (final Enumeration e = p.propertyNames(); e.hasMoreElements();) { final String currentPropertyName = (String) e.nextElement(); @@ -440,6 +445,26 @@ Properties checkExternalProperties() throws ArgumentException { } } + /** + * Reads a properties file as UTF-8, or as ISO-8859-1, which earlier releases used, when it is + * not valid UTF-8. A backslash keeps its meaning in the properties format and must be doubled. + */ + private static Properties loadPropertiesFile(final String path) throws IOException { + final byte[] content = Files.readAllBytes(Paths.get(path)); + // A UTF-8 byte order mark is skipped before decoding, so that the ISO-8859-1 fallback skips it too. + final int bom = content.length >= 3 && content[0] == (byte) 0xEF && content[1] == (byte) 0xBB + && content[2] == (byte) 0xBF ? 3 : 0; + String text; + try { + text = UTF_8.newDecoder().decode(ByteBuffer.wrap(content, bom, content.length - bom)).toString(); + } catch (final CharacterCodingException e) { + text = new String(content, bom, content.length - bom, ISO_8859_1); + } + final Properties properties = new Properties(); + properties.load(new StringReader(text)); + return properties; + } + /** * Retrieves the argument with the specified long identifier. * @@ -1268,10 +1293,8 @@ public void parseArguments(final String[] rawArguments, final String propertiesF final boolean requirePropertiesFile) throws ArgumentException { Properties argumentProperties = null; - try (final FileInputStream fis = new FileInputStream(propertiesFile)) { - final Properties p = new Properties(); - p.load(fis); - argumentProperties = p; + try { + argumentProperties = loadPropertiesFile(propertiesFile); } catch (final Exception e) { if (requirePropertiesFile) { final LocalizableMessage message = diff --git a/opendj-cli/src/main/java/com/forgerock/opendj/cli/CommandBuilder.java b/opendj-cli/src/main/java/com/forgerock/opendj/cli/CommandBuilder.java index 33335196cc..e409ddd426 100644 --- a/opendj-cli/src/main/java/com/forgerock/opendj/cli/CommandBuilder.java +++ b/opendj-cli/src/main/java/com/forgerock/opendj/cli/CommandBuilder.java @@ -13,6 +13,7 @@ * * Copyright 2008-2009 Sun Microsystems, Inc. * Portions Copyright 2014-2016 ForgeRock AS. + * Portions Copyright 2026 3A Systems, LLC. */ package com.forgerock.opendj.cli; @@ -240,8 +241,18 @@ public boolean isObfuscated(final Argument argument) { * @return The transformed value. */ public static String escapeValue(String value) { + return escapeValue(value, OperatingSystem.isUnix()); + } + + private static void appendBackslashes(final StringBuilder b, final int count) { + for (int i = 0; i < count; i++) { + b.append('\\'); + } + } + + static String escapeValue(String value, boolean unix) { final StringBuilder b = new StringBuilder(); - if (OperatingSystem.isUnix()) { + if (unix) { for (int i = 0; i < value.length(); i++) { final char c = value.charAt(i); if (CHARSTOESCAPE.contains(c)) { @@ -250,7 +261,25 @@ public static String escapeValue(String value) { b.append(c); } } else { - b.append('"').append(value).append('"'); + // The C runtime takes backslashes literally unless they precede a double quote: there + // each pair gives one backslash and an odd one escapes the quote. So the backslashes + // before a quote, or before the closing quote, are doubled and the quote is escaped. + b.append('"'); + int backslashes = 0; + for (int i = 0; i < value.length(); i++) { + final char c = value.charAt(i); + if (c == '\\') { + backslashes++; + } else { + if (c == '"') { + appendBackslashes(b, backslashes + 1); + } + backslashes = 0; + } + b.append(c); + } + appendBackslashes(b, backslashes); + b.append('"'); } return b.toString(); } diff --git a/opendj-cli/src/test/java/com/forgerock/opendj/cli/ArgumentParserPropertiesFileTestCase.java b/opendj-cli/src/test/java/com/forgerock/opendj/cli/ArgumentParserPropertiesFileTestCase.java new file mode 100644 index 0000000000..190b31e410 --- /dev/null +++ b/opendj-cli/src/test/java/com/forgerock/opendj/cli/ArgumentParserPropertiesFileTestCase.java @@ -0,0 +1,103 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions copyright [year] [name of copyright owner]". + * + * Copyright 2026 3A Systems, LLC. + */ +package com.forgerock.opendj.cli; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.fest.assertions.Assertions.assertThat; + +import java.io.File; +import java.nio.file.Files; + +import org.forgerock.i18n.LocalizableMessage; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** Reading argument defaults from a {@code tools.properties} file. */ +@SuppressWarnings("javadoc") +public final class ArgumentParserPropertiesFileTestCase extends CliTestCase { + + /** Both ways a tool reads the file: through --propertiesFilePath and through an explicit path. */ + @DataProvider + public Object[][] readers() { + return new Object[][] { { true }, { false } }; + } + + @Test(dataProvider = "readers") + public void testUtf8FileKeepsNonAsciiCharacters(boolean viaArgument) throws Exception { + final File file = write("basedn=ou=Jörg Ж,dc=example,dc=com\n".getBytes(UTF_8)); + assertThat(baseDN(file, viaArgument)).isEqualTo("ou=Jörg Ж,dc=example,dc=com"); + } + + /** Some Windows editors start a UTF-8 file with a byte order mark. */ + @Test(dataProvider = "readers") + public void testUtf8ByteOrderMarkIsSkipped(boolean viaArgument) throws Exception { + final File file = write("\uFEFFbasedn=dc=example,dc=com\n".getBytes(UTF_8)); + assertThat(baseDN(file, viaArgument)).isEqualTo("dc=example,dc=com"); + } + + /** A file that starts as UTF-8 with a byte order mark but later gets a Latin-1 byte. */ + @Test(dataProvider = "readers") + public void testByteOrderMarkIsSkippedInIso88591Fallback(boolean viaArgument) throws Exception { + final byte[] bom = { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF }; + final byte[] text = "basedn=dc=example,dc=com\nx=ö\n".getBytes(ISO_8859_1); + final byte[] content = new byte[bom.length + text.length]; + System.arraycopy(bom, 0, content, 0, bom.length); + System.arraycopy(text, 0, content, bom.length, text.length); + assertThat(baseDN(write(content), viaArgument)).isEqualTo("dc=example,dc=com"); + } + + /** Files written for earlier releases, which read them as ISO-8859-1, keep working. */ + @Test(dataProvider = "readers") + public void testIso88591FileIsStillAccepted(boolean viaArgument) throws Exception { + final File file = write("basedn=ou=Jörg,dc=example,dc=com\n".getBytes(ISO_8859_1)); + assertThat(baseDN(file, viaArgument)).isEqualTo("ou=Jörg,dc=example,dc=com"); + } + + /** The file keeps the properties format: a backslash in a value must be doubled. */ + @Test(dataProvider = "readers") + public void testDoubledBackslashIsOneBackslash(boolean viaArgument) throws Exception { + final File file = write("basedn=cn=a\\\\,b,dc=example,dc=com\n".getBytes(UTF_8)); + assertThat(baseDN(file, viaArgument)).isEqualTo("cn=a\\,b,dc=example,dc=com"); + } + + private static String baseDN(File file, boolean viaArgument) throws Exception { + final ArgumentParser parser = + new ArgumentParser(ArgumentParserPropertiesFileTestCase.class.getName(), + LocalizableMessage.raw("test"), false); + final StringArgument baseDN = StringArgument.builder("baseDN") + .valuePlaceholder(LocalizableMessage.raw("{baseDN}")) + .description(LocalizableMessage.raw("base DN")) + .buildAndAddToParser(parser); + if (viaArgument) { + final StringArgument propertiesFile = + CommonArguments.propertiesFileArgument(); + parser.addArgument(propertiesFile); + parser.setFilePropertiesArgument(propertiesFile); + parser.parseArguments(new String[] { "--" + propertiesFile.getLongIdentifier(), file.getPath() }); + } else { + parser.parseArguments(new String[0], file.getPath(), true); + } + return baseDN.getValue(); + } + + private static File write(byte[] content) throws Exception { + final File file = File.createTempFile("tools", ".properties"); + file.deleteOnExit(); + Files.write(file.toPath(), content); + return file; + } +} diff --git a/opendj-cli/src/test/java/com/forgerock/opendj/cli/CommandBuilderTestCase.java b/opendj-cli/src/test/java/com/forgerock/opendj/cli/CommandBuilderTestCase.java new file mode 100644 index 0000000000..f4bc4c1526 --- /dev/null +++ b/opendj-cli/src/test/java/com/forgerock/opendj/cli/CommandBuilderTestCase.java @@ -0,0 +1,60 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions copyright [year] [name of copyright owner]". + * + * Copyright 2026 3A Systems, LLC. + */ +package com.forgerock.opendj.cli; + +import static org.fest.assertions.Assertions.assertThat; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** Quoting of the values in the equivalent command lines that the tools print. */ +@SuppressWarnings("javadoc") +public final class CommandBuilderTestCase extends CliTestCase { + + /** + * Values and their Windows form. A Windows program splits its command line by the rules of + * the Microsoft C runtime: backslashes are literal unless they precede a double quote, where + * each pair gives one backslash and an odd one escapes the quote. + */ + @DataProvider + public Object[][] windowsValues() { + return new Object[][] { + { "cn=a\\,b,dc=x", "\"cn=a\\,b,dc=x\"" }, + { "o=My Company", "\"o=My Company\"" }, + { "(targetattr=\"cn\")(version 3.0; acl \"x\"; allow (read) userdn=\"ldap:///self\";)", + "\"(targetattr=\\\"cn\\\")(version 3.0; acl \\\"x\\\"; allow (read) userdn=\\\"ldap:///self\\\";)\"" }, + { "abc\\", "\"abc\\\\\"" }, + { "a\\\"b", "\"a\\\\\\\"b\"" }, + { "", "\"\"" }, + }; + } + + @Test(dataProvider = "windowsValues") + public void testWindowsValueIsQuotedForTheCRuntime(String value, String expected) { + assertThat(CommandBuilder.escapeValue(value, false)).isEqualTo(expected); + } + + @Test + public void testUnixValueIsBackslashEscaped() { + assertThat(CommandBuilder.escapeValue("cn=a\\,b \"x\"", true)).isEqualTo("cn=a\\\\,b\\ \\\"x\\\""); + } + + /** A dsconfig batch line takes an unescaped single quote as an opening quote. */ + @Test + public void testUnixApostropheIsEscaped() { + assertThat(CommandBuilder.escapeValue("description:O'Brien", true)).isEqualTo("description:O\\'Brien"); + } +} diff --git a/opendj-config/src/main/java/org/forgerock/opendj/config/dsconfig/DSConfig.java b/opendj-config/src/main/java/org/forgerock/opendj/config/dsconfig/DSConfig.java index 07d83f14bc..27b049f23b 100644 --- a/opendj-config/src/main/java/org/forgerock/opendj/config/dsconfig/DSConfig.java +++ b/opendj-config/src/main/java/org/forgerock/opendj/config/dsconfig/DSConfig.java @@ -1394,40 +1394,38 @@ private void handleBatch(String[] args) { try (BufferedReader bReader = batchCommandsReader()) { List initialArgs = removeBatchArgs(args); - // Split the CLI string into arguments array - String command = ""; - String line; - while ((line = bReader.readLine()) != null) { - if (line.isEmpty() || line.startsWith("#")) { - // Empty line or comment - continue; - } - // command split in several line support - if (line.endsWith("\\")) { - // command is split into several lines - command += line.substring(0, line.length() - 1); - continue; - } - - command += line; - command = command.trim(); - printlnNoWrap(LocalizableMessage.raw(command)); + String command; + while ((command = nextBatchCommand(bReader)) != null) { + // Only the echo is trimmed: an escaped blank at the end of the command is kept. + printlnNoWrap(LocalizableMessage.raw(command.trim())); - // Append initial arguments to the file line - final String[] allArgsArray = buildCommandArgs(initialArgs, command); - int exitCode = main(allArgsArray, getOutputStream(), getErrorStream()); + final int exitCode = runBatchCommand(initialArgs, command, getOutputStream(), getErrorStream()); if (exitCode != ReturnCode.SUCCESS.get()) { System.exit(filterExitCode(exitCode)); } println(); - // reset command - command = ""; } } catch (IOException ex) { errPrintln(ERR_DSCFG_ERROR_READING_BATCH_FILE.get(ex)); } } + /** + * Runs one command of a batch with the initial arguments appended, and returns its exit code. + * A command that cannot be split into arguments is reported on the error stream: a batch + * always runs with --no-prompt, so that is where dsconfig writes its errors. + */ + static int runBatchCommand(final List initialArgs, final String command, final PrintStream out, + final PrintStream err) { + try { + // Append initial arguments to the file line + return main(buildCommandArgs(initialArgs, command), out, err); + } catch (final ArgumentException e) { + err.println(wrapText(e.getMessageObject(), MAX_LINE_WIDTH)); + return ReturnCode.ERROR_USER_DATA.get(); + } + } + private BufferedReader batchCommandsReader() throws FileNotFoundException { if (batchArgument.isPresent()) { return new BufferedReader(new InputStreamReader(System.in)); @@ -1440,7 +1438,38 @@ private BufferedReader batchCommandsReader() throws FileNotFoundException { } } - private String[] buildCommandArgs(List initialArgs, String batchCommand) { + /** + * Reads the next command of a batch, or returns {@code null} at its end. Lines that are empty + * or hold only blanks, and comments, are skipped, also inside a command that continues over + * several lines, and a line that ends in an unescaped backslash continues on the next line. A + * command whose last line continues still runs at the end of the batch. + */ + static String nextBatchCommand(final BufferedReader reader) throws IOException { + final StringBuilder command = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + if (line.trim().isEmpty() || line.startsWith("#")) { + // Empty or blank line, or comment + continue; + } + if (!continuesOnNextLine(line)) { + return command.append(line).toString(); + } + command.append(line, 0, line.length() - 1); + } + return command.toString().trim().isEmpty() ? null : command.toString(); + } + + /** Whether a line ends in a backslash that is not itself escaped by a backslash. */ + private static boolean continuesOnNextLine(final String line) { + int backslashes = 0; + for (int i = line.length() - 1; i >= 0 && line.charAt(i) == '\\'; i--) { + backslashes++; + } + return backslashes % 2 == 1; + } + + private static String[] buildCommandArgs(List initialArgs, String batchCommand) throws ArgumentException { final Collection commandArgs = toCommandArgs(batchCommand); final int length = commandArgs.size() + initialArgs.size(); final List allArguments = new ArrayList<>(length); @@ -1449,46 +1478,64 @@ private String[] buildCommandArgs(List initialArgs, String batchCommand) return allArguments.toArray(new String[length]); } - static Collection toCommandArgs(String command) { - Collection commandArgs = new ArrayList<>(); - StringBuilder builder = new StringBuilder(); - boolean inQuotes = false; + /** + * Splits a batch line into arguments the way a POSIX shell does, without any expansion: a + * backslash outside quotes escapes the next char, single quotes keep everything literally, and + * inside double quotes a backslash escapes only {@code "}, {@code \}, {@code $} and {@code `}. + * + * @throws ArgumentException + * If a quote is not closed. + */ + static Collection toCommandArgs(String command) throws ArgumentException { + final Collection commandArgs = new ArrayList<>(); + final StringBuilder builder = new StringBuilder(); + // A word that holds only quotes is still an (empty) argument. + boolean inWord = false; for (int i = 0; i < command.length(); i++) { final char c = command.charAt(i); switch (c) { - default: - builder.append(c); - break; + case ' ': + case '\t': + if (inWord) { + commandArgs.add(builder.toString()); + builder.setLength(0); + inWord = false; + } + continue; case '\\': - builder.append(command.charAt(++i)); + // A trailing backslash has nothing to escape and is kept. + builder.append(i + 1 < command.length() ? command.charAt(++i) : c); break; - case '"': - if (inQuotes) { - builder = newArgumentString(commandArgs, builder); - inQuotes = false; - } else { - inQuotes = true; + case '\'': + final int end = command.indexOf('\'', i + 1); + if (end < 0) { + throw new ArgumentException(ERR_DSCFG_ERROR_BATCH_UNCLOSED_QUOTE.get(command.trim())); } + builder.append(command, i + 1, end); + i = end; break; - case ' ': - if (inQuotes) { - builder.append(c); - } else { - builder = newArgumentString(commandArgs, builder); + case '"': + for (i++; i < command.length() && command.charAt(i) != '"'; i++) { + if (command.charAt(i) == '\\' && i + 1 < command.length() + && "\"\\$`".indexOf(command.charAt(i + 1)) >= 0) { + i++; + } + builder.append(command.charAt(i)); + } + if (i == command.length()) { + throw new ArgumentException(ERR_DSCFG_ERROR_BATCH_UNCLOSED_QUOTE.get(command.trim())); } break; + default: + builder.append(c); + break; } + inWord = true; } - newArgumentString(commandArgs, builder); - return commandArgs; - } - - private static StringBuilder newArgumentString(Collection commandArgs, StringBuilder stringBuilder) { - if (stringBuilder.length() > 0) { - commandArgs.add(stringBuilder.toString()); - stringBuilder = new StringBuilder(); + if (inWord) { + commandArgs.add(builder.toString()); } - return stringBuilder; + return commandArgs; } private List removeBatchArgs(String[] args) { diff --git a/opendj-config/src/main/resources/com/forgerock/opendj/dsconfig/dsconfig.properties b/opendj-config/src/main/resources/com/forgerock/opendj/dsconfig/dsconfig.properties index 3d4aa65105..9ed5d0d3b6 100644 --- a/opendj-config/src/main/resources/com/forgerock/opendj/dsconfig/dsconfig.properties +++ b/opendj-config/src/main/resources/com/forgerock/opendj/dsconfig/dsconfig.properties @@ -388,6 +388,8 @@ INFO_DSCFG_TYPE_PROMPT_SINGLE_249=>>>> There is only one type of %s available: " Are you sure that this is the correct one? ERR_DSCFG_ERROR_READING_BATCH_FILE_250=An error occurred \ while attempting to read the batch file : %s +ERR_DSCFG_ERROR_BATCH_UNCLOSED_QUOTE_1197=The following batch command has a \ + quote that is not closed: %s INFO_DSCFG_SESSION_START_TIME_MESSAGE_251=%s session start date: %s INFO_DSCFG_EQUIVALENT_COMMAND_LINE_SESSION_OPERATION_NUMBER_252=Session \ operation number: %d diff --git a/opendj-config/src/test/java/org/forgerock/opendj/config/dsconfig/DSConfigParseTest.java b/opendj-config/src/test/java/org/forgerock/opendj/config/dsconfig/DSConfigParseTest.java index 6cd877f06e..7e8e51fb87 100644 --- a/opendj-config/src/test/java/org/forgerock/opendj/config/dsconfig/DSConfigParseTest.java +++ b/opendj-config/src/test/java/org/forgerock/opendj/config/dsconfig/DSConfigParseTest.java @@ -12,15 +12,26 @@ * information: "Portions Copyright [year] [name of copyright owner]". * * Portions copyright 2016 ForgeRock AS. + * Portions Copyright 2026 3A Systems, LLC. */ package org.forgerock.opendj.config.dsconfig; +import com.forgerock.opendj.cli.ArgumentException; +import com.forgerock.opendj.cli.ReturnCode; + import org.forgerock.testng.ForgeRockTestCase; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.List; @Test(groups = { "precommit", "config" }) public class DSConfigParseTest extends ForgeRockTestCase { @@ -45,4 +56,131 @@ public void testEscapeSequenceInCommandArgument(String arg, String value) throws Collection cmdLine = DSConfig.toCommandArgs(arg); Assert.assertEquals(cmdLine.iterator().next(), value); } + + /** Batch lines and the arguments a POSIX shell splits them into. */ + @DataProvider + public Object[][] shellQuoting() { + return new Object[][] { + // Inside double quotes a backslash is kept unless it precedes ", \, $ or `. + { "--set base-dn:\"cn=a\\,b,dc=x\"", args("--set", "base-dn:cn=a\\,b,dc=x") }, + { "--set \"base-dn:cn=a\\2Cb,dc=x\"", args("--set", "base-dn:cn=a\\2Cb,dc=x") }, + { "\"a\\\"b\\\\c\\$d\\`e\"", args("a\"b\\c$d`e") }, + // Single quotes keep everything literally, a backslash included. + { "--set 'base-dn:o=My Company'", args("--set", "base-dn:o=My Company") }, + { "--set 'base-dn:cn=a\\,b,dc=x'", args("--set", "base-dn:cn=a\\,b,dc=x") }, + { "'say \"hi\"' \"it's\"", args("say \"hi\"", "it's") }, + // Quoted and unquoted parts of one word make a single argument. + { "base-dn:\"o=My \"'Company'", args("base-dn:o=My Company") }, + { "\"a\"b c", args("ab", "c") }, + // An empty quoted word is an empty argument. + { "--set description:\"\" \"\"", args("--set", "description:", "") }, + // Tabs separate arguments as spaces do. + { "a\tb c", args("a", "b", "c") }, + // A trailing backslash has nothing to escape and is kept. + { "a b\\", args("a", "b\\") }, + }; + } + + @Test(dataProvider = "shellQuoting") + public void testShellQuotingInCommandLine(String line, List expected) throws Exception { + Assert.assertEquals(new ArrayList<>(DSConfig.toCommandArgs(line)), expected); + } + + /** Batch lines with a quote that is not closed: a POSIX shell rejects them. */ + @DataProvider + public Object[][] unclosedQuotes() { + return new Object[][] { + // Before single quotes were supported, this line gave three arguments. + { "--set description:O'Brien --advanced" }, + { "--set 'base-dn:o=My Company" }, + { "--set \"base-dn:o=My Company" }, + { "--set \"base-dn:o=My Company\\\"" }, + }; + } + + @Test(dataProvider = "unclosedQuotes", expectedExceptions = ArgumentException.class, + expectedExceptionsMessageRegExp = "The following batch command has a quote that is not closed: .*") + public void testUnclosedQuoteIsRejected(String line) throws Exception { + DSConfig.toCommandArgs(line); + } + + @Test + public void testBatchCommandWithUnclosedQuoteFails() { + final ByteArrayOutputStream err = new ByteArrayOutputStream(); + + final int exitCode = DSConfig.runBatchCommand(args("--noPropertiesFile", "-n"), "set-x --set 'a", + new PrintStream(new ByteArrayOutputStream()), new PrintStream(err)); + + Assert.assertEquals(exitCode, ReturnCode.ERROR_USER_DATA.get()); + Assert.assertTrue(text(err).contains("The following batch command has a quote that is not closed: " + + "set-x --set 'a"), text(err)); + } + + @Test + public void testBatchCommandKeepsEscapedTrailingBlank() { + // dsconfig fails on the first argument, before it reads --no-prompt, so it still writes + // the error to the output stream. + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + final PrintStream stream = new PrintStream(output); + + // dsconfig has no such subcommand: the error quotes the argument it was given. + final int exitCode = DSConfig.runBatchCommand(args("--noPropertiesFile", "-n"), "set-x\\ ", stream, stream); + + Assert.assertEquals(exitCode, ReturnCode.CONFLICTING_ARGS.get()); + Assert.assertTrue(text(output).contains("The provided argument \"set-x \" is not recognized"), text(output)); + } + + /** The output with every run of blanks and line breaks as a single space, as the console wraps lines. */ + private static String text(ByteArrayOutputStream out) { + return new String(out.toByteArray()).replaceAll("\\s+", " "); + } + + /** Batch files and the arguments of each command they hold. */ + @DataProvider + public Object[][] batchFiles() { + return new Object[][] { + { "# comment\n\nset-x --foo bar\nset-y\n", commands(args("set-x", "--foo", "bar"), args("set-y")) }, + // A line of blanks is skipped as an empty line is. + { "set-x\n \nset-y\n", commands(args("set-x"), args("set-y")) }, + { "set-x\n \t\n", commands(args("set-x")) }, + // Empty lines and comments are skipped inside a continued command too, so a line of a + // long command can be commented out. + { "set-x \\\n# --foo bar \\\n --baz qux\n", commands(args("set-x", "--baz", "qux")) }, + { "set-x \\\n\n --baz qux\n", commands(args("set-x", "--baz", "qux")) }, + // A line that ends in a backslash continues on the next line. + { "set-x \\\n --foo bar\n", commands(args("set-x", "--foo", "bar")) }, + { "set-x --foo ab\\\ncd\n", commands(args("set-x", "--foo", "abcd")) }, + // An escaped backslash at the end of a line does not continue it, as dsconfig + // --commandFilePath writes a value that ends in a backslash on UNIX. + { "set-x --bindPassword pa\\\\\nset-y --foo bar\n", + commands(args("set-x", "--bindPassword", "pa\\"), args("set-y", "--foo", "bar")) }, + { "set-x --foo a\\\\\\\nb\n", commands(args("set-x", "--foo", "a\\b")) }, + // An escaped blank at the end of a line is kept. + { "set-x --set description:value\\ \n", commands(args("set-x", "--set", "description:value ")) }, + // A command still runs when its last line continues at the end of the file. + { "set-x --foo bar \\", commands(args("set-x", "--foo", "bar")) }, + { "set-x\n \\\n", commands(args("set-x")) }, + }; + } + + @Test(dataProvider = "batchFiles") + public void testBatchFileCommands(String batch, List> expected) throws Exception { + final List> actual = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new StringReader(batch))) { + String command; + while ((command = DSConfig.nextBatchCommand(reader)) != null) { + actual.add(new ArrayList<>(DSConfig.toCommandArgs(command))); + } + } + Assert.assertEquals(actual, expected); + } + + private static List args(String... args) { + return Arrays.asList(args); + } + + @SafeVarargs + private static List> commands(List... commands) { + return Arrays.asList(commands); + } } diff --git a/opendj-core/src/main/java/org/forgerock/opendj/ldap/LDAPUrl.java b/opendj-core/src/main/java/org/forgerock/opendj/ldap/LDAPUrl.java index 9bc4947199..48677cf9c0 100644 --- a/opendj-core/src/main/java/org/forgerock/opendj/ldap/LDAPUrl.java +++ b/opendj-core/src/main/java/org/forgerock/opendj/ldap/LDAPUrl.java @@ -17,6 +17,10 @@ */ package org.forgerock.opendj.ldap; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -234,32 +238,43 @@ private static void percentDecoder(final String urlString, final int index, fina final StringBuilder decoded) { Reject.ifNull(s); Reject.ifNull(decoded); - decoded.append(s); - int srcPos = 0, dstPos = 0; - - while (srcPos < decoded.length()) { - if (decoded.charAt(srcPos) != '%') { - if (srcPos != dstPos) { - decoded.setCharAt(dstPos, decoded.charAt(srcPos)); - } + final ByteArrayOutputStream octets = new ByteArrayOutputStream(); + int srcPos = 0; + while (srcPos < s.length()) { + if (s.charAt(srcPos) != '%') { + decoded.append(s.charAt(srcPos)); srcPos++; - dstPos++; continue; } - if (srcPos + 2 >= decoded.length()) { - // A percent sign must be followed by two hexadecimal digits. - final LocalizableMessage msg = - ERR_LDAPURL_INVALID_HEX_BYTE.get(urlString, index + srcPos + 1); - throw new LocalizedIllegalArgumentException(msg); + // Collect a run of percent-encoded octets and decode it as a whole. + octets.reset(); + while (srcPos < s.length() && s.charAt(srcPos) == '%') { + if (srcPos + 2 >= s.length()) { + // A percent sign must be followed by two hexadecimal digits. + final LocalizableMessage msg = + ERR_LDAPURL_INVALID_HEX_BYTE.get(urlString, index + srcPos + 1); + throw new LocalizedIllegalArgumentException(msg); + } + int i = decodeHex(urlString, index + srcPos + 1, s.charAt(srcPos + 1)) << 4; + int j = decodeHex(urlString, index + srcPos + 2, s.charAt(srcPos + 2)); + octets.write(i | j); + srcPos += 3; } - int i = decodeHex(urlString, index + srcPos + 1, decoded.charAt(srcPos + 1)) << 4; - int j = decodeHex(urlString, index + srcPos + 2, decoded.charAt(srcPos + 2)); - decoded.setCharAt(dstPos, (char) (i | j)); - dstPos++; - srcPos += 3; + decoded.append(decodeOctets(octets.toByteArray())); + } + } + + /** + * Decodes percent-encoded octets as UTF-8 (RFC 4516 section 2.1). Octets that are not UTF-8 + * give one char each, as this class decoded them before. + */ + private static String decodeOctets(final byte[] octets) { + try { + return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(octets)).toString(); + } catch (final CharacterCodingException e) { + return new String(octets, StandardCharsets.ISO_8859_1); } - decoded.setLength(dstPos); } /** @@ -273,14 +288,20 @@ private static void percentDecoder(final String urlString, final int index, fina */ private static void percentEncoder(final String urlElement, final StringBuilder encodedBuffer) { Reject.ifNull(urlElement); - for (int count = 0; count < urlElement.length(); count++) { - final char c = urlElement.charAt(count); - if (VALID_CHARS.contains(c)) { - encodedBuffer.append(c); + for (int count = 0; count < urlElement.length();) { + final int c = urlElement.codePointAt(count); + final int next = count + Character.charCount(c); + if (c < Character.MIN_SUPPLEMENTARY_CODE_POINT && VALID_CHARS.contains((char) c)) { + encodedBuffer.append((char) c); } else { - encodedBuffer.append(PERCENT_ENCODING_CHAR); - encodedBuffer.append(Integer.toHexString(c)); + // Each octet of the UTF-8 encoding, as two hex digits. + for (final byte b : urlElement.substring(count, next).getBytes(StandardCharsets.UTF_8)) { + encodedBuffer.append(PERCENT_ENCODING_CHAR); + encodedBuffer.append(Character.forDigit((b >> 4) & 0xF, 16)); + encodedBuffer.append(Character.forDigit(b & 0xF, 16)); + } } + count = next; } } diff --git a/opendj-core/src/test/java/org/forgerock/opendj/ldap/LDAPUrlTestCase.java b/opendj-core/src/test/java/org/forgerock/opendj/ldap/LDAPUrlTestCase.java index 1c3020e400..2bd8e5218d 100644 --- a/opendj-core/src/test/java/org/forgerock/opendj/ldap/LDAPUrlTestCase.java +++ b/opendj-core/src/test/java/org/forgerock/opendj/ldap/LDAPUrlTestCase.java @@ -18,6 +18,7 @@ package org.forgerock.opendj.ldap; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; import org.forgerock.i18n.LocalizedIllegalArgumentException; @@ -211,4 +212,68 @@ public Object[][] truncatedPercentUrls() { public void testTruncatedPercentEncoding(final String url) throws Exception { LDAPUrl.valueOf(url); } + + /** + * Percent-encoded octets are the UTF-8 encoding of the characters (RFC 4516 section 2.1). + * + * @return The URLs and the DN and filter they hold. + */ + @DataProvider + public Object[][] utf8PercentEncodedUrls() { + return new Object[][] { + { "ldap:///cn=J%C3%B6rg,dc=x", "cn=Jörg,dc=x", "(objectClass=*)" }, + { "ldap:///cn=J%c3%b6rg%20%D0%96,dc=x", "cn=Jörg Ж,dc=x", "(objectClass=*)" }, + { "ldap:///cn=%F0%9F%98%80,dc=x", "cn=😀,dc=x", "(objectClass=*)" }, + { "ldap:///cn=Jörg,dc=x??sub?(cn=J%C3%B6rg)", "cn=Jörg,dc=x", "(cn=Jörg)" }, + // An octet sequence that is not UTF-8 is read one char per octet, as before. + { "ldap:///cn=J%f6rg,dc=x", "cn=Jörg,dc=x", "(objectClass=*)" }, + }; + } + + /** + * Tests that percent-encoded octets are decoded as UTF-8. + * + * @param url + * The URL to decode. + * @param dn + * The DN the URL holds. + * @param filter + * The filter the URL holds. + */ + @Test(dataProvider = "utf8PercentEncodedUrls") + public void testPercentDecodingIsUtf8(final String url, final String dn, final String filter) { + final LDAPUrl ldapUrl = LDAPUrl.valueOf(url); + assertEquals(ldapUrl.getName(), DN.valueOf(dn)); + assertEquals(ldapUrl.getFilter().toString(), Filter.valueOf(filter).toString()); + } + + /** + * Tests that characters outside the allowed set are encoded as the percent-encoded octets of + * their UTF-8 encoding, and that the result decodes to the same URL. + */ + @Test + public void testPercentEncodingIsUtf8() { + final LDAPUrl url = new LDAPUrl(false, "h", 389, DN.valueOf("cn=Jörg Ж 😀,dc=x"), + SearchScope.WHOLE_SUBTREE, Filter.equality("cn", "Jörg")); + assertTrue(url.toString().startsWith("ldap://h:389/cn=J%c3%b6rg%20%d0%96%20%f0%9f%98%80,dc=x??sub?"), + url.toString()); + final LDAPUrl decoded = LDAPUrl.valueOf(url.toString()); + assertEquals(decoded.getName(), url.getName()); + assertEquals(decoded.getFilter().toString(), url.getFilter().toString()); + assertEquals(decoded, url); + } + + /** + * An octet below 0x10 is encoded with two hex digits: with one, {@code a%01b} and {@code a%1b} + * would both read {@code a%1b}. The filter is percent-encoded for {@code equals()}, and an + * attribute description reaches it unchecked. + */ + @Test + public void testPercentEncodingPadsOctetsBelow0x10() { + final LDAPUrl url1 = new LDAPUrl(false, "h", 389, DN.valueOf("dc=x"), + SearchScope.WHOLE_SUBTREE, Filter.equality("a\u0001b", "x")); + final LDAPUrl url2 = new LDAPUrl(false, "h", 389, DN.valueOf("dc=x"), + SearchScope.WHOLE_SUBTREE, Filter.equality("a\u001b", "x")); + assertNotEquals(url1, url2); + } } diff --git a/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-admin-tools.adoc b/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-admin-tools.adoc index caa19432fc..2e758be543 100644 --- a/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-admin-tools.adoc +++ b/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-admin-tools.adoc @@ -12,7 +12,7 @@ information: "Portions copyright [year] [name of copyright owner]". Copyright 2017 ForgeRock AS. - Portions Copyright 2024 3A Systems LLC. + Portions Copyright 2024-2026 3A Systems LLC. //// :figure-caption!: @@ -223,6 +223,15 @@ You can prepare `dsconfig` batch scripts by running the command with the `--comm + Alternatively, you can read commands from standard input by using the `--batch` option. ++ +The `dsconfig` command splits each batch line into arguments as a POSIX shell does, without expanding variables. Outside quotes, a backslash escapes the next character. Single quotes keep everything as typed. Inside double quotes, a backslash escapes only a double quote, a backslash, a dollar sign, or a backquote, and is kept before any other character. For example, `--set base-dn:"cn=a\,b,dc=example,dc=com"` and `--set 'base-dn:cn=a\,b,dc=example,dc=com'` both set the DN `cn=a\,b,dc=example,dc=com`. + ++ +A single quote is a quote character, as in a shell, and no longer a literal character as in earlier releases: write `+O\'Brien+` or `+"O'Brien"+` for a value that holds one. A line with a quote that is not closed is rejected. A line that ends in a backslash continues on the next line, unless that backslash is itself escaped by another backslash. Empty lines, lines of blanks, and lines that start with `#` are skipped, also inside a command that continues over several lines, so a line of a long command can be commented out. + ++ +On Windows, the equivalent non-interactive command that `dsconfig` prints quotes each value by the rules of the C runtime, which splits the command line of the Java program. It does not escape the characters that `cmd.exe` interprets itself, such as `&`, `|`, `<`, and `>`, so when a value holds one of them, such as an ACI with an `&` filter, the command does not run in `cmd.exe` as printed. + + For details see xref:../reference/admin-tools-ref.adoc#dsconfig-1[dsconfig(1)] in the __Reference__. diff --git a/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-production.adoc b/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-production.adoc index 1b09c1309c..98cc89e863 100644 --- a/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-production.adoc +++ b/opendj-doc-generated-ref/src/main/asciidoc/admin-guide/chap-production.adoc @@ -12,7 +12,7 @@ information: "Portions copyright [year] [name of copyright owner]". Copyright 2017 ForgeRock AS. - Portions Copyright 2024 3A Systems LLC. + Portions Copyright 2024-2026 3A Systems LLC. //// :figure-caption!: @@ -170,7 +170,7 @@ userPassword: {SSHA512}WiYWHyAa612EZwCMY7uGwN/WYp2Ne7EmV0QTPX5g6RrTKi8jZX3u5rBIW ---- Notice that the password appears neither in the shell history, nor in the terminal session. -When using scripts where the password cannot be supplied interactively, passwords can be read from files. For example, the `--bindPasswordFile file` option takes a file that should be readable only by the user running the command. It is also possible to set passwords in the `tools.properties` file for the user. This file is located in the user's home directory, on UNIX `~/.opendj/tools.properties`, and on Windows typically `C:\Documents and Settings\username\.opendj\tools.properties`, though the location can depend on the Java runtime environment used. Here as well, make sure that the file is readable only by the user. Alternatively, use other approaches that work with scripts such as Java properties or environment variables, depending on what method is most secure in production. +When using scripts where the password cannot be supplied interactively, passwords can be read from files. For example, the `--bindPasswordFile file` option takes a file that should be readable only by the user running the command. It is also possible to set passwords in the `tools.properties` file for the user. This file is located in the user's home directory, on UNIX `~/.opendj/tools.properties`, and on Windows typically `C:\Documents and Settings\username\.opendj\tools.properties`, though the location can depend on the Java runtime environment used. The file uses the Java properties format, so double each backslash in a password. Here as well, make sure that the file is readable only by the user. Alternatively, use other approaches that work with scripts such as Java properties or environment variables, depending on what method is most secure in production. [#production-password-policy] diff --git a/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_description-dsconfig.adoc b/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_description-dsconfig.adoc index 25d696a23c..05ba0f4653 100644 --- a/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_description-dsconfig.adoc +++ b/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_description-dsconfig.adoc @@ -13,7 +13,7 @@ information: "Portions Copyright [year] [name of copyright owner]". Copyright 2015-2016 ForgeRock AS. - Portions Copyright 2024 3A Systems LLC. + Portions Copyright 2024-2026 3A Systems LLC. //// @@ -23,6 +23,12 @@ When you pass connection information, subcommands, and additional options to `ds You can prepare `dsconfig` batch scripts by running the tool with the `--commandFilePath` option in interactive mode, then reading from the batch file with the `--batchFilePath` option in script mode. Batch files can be useful when you have many `dsconfig` commands to run and want to avoid starting the JVM for each command. Alternatively, you can read commands from standard input by using the `--batch` option. +The `dsconfig` command splits each batch line into arguments as a POSIX shell does, without expanding variables. Outside quotes, a backslash escapes the next character. Single quotes keep everything as typed. Inside double quotes, a backslash escapes only a double quote, a backslash, a dollar sign, or a backquote, and is kept before any other character. For example, `--set base-dn:"cn=a\,b,dc=example,dc=com"` and `--set 'base-dn:cn=a\,b,dc=example,dc=com'` both set the DN `cn=a\,b,dc=example,dc=com`. + +A single quote is a quote character, as in a shell, and no longer a literal character as in earlier releases: write `+O\'Brien+` or `+"O'Brien"+` for a value that holds one. A line with a quote that is not closed is rejected. A line that ends in a backslash continues on the next line, unless that backslash is itself escaped by another backslash. Empty lines, lines of blanks, and lines that start with `#` are skipped, also inside a command that continues over several lines, so a line of a long command can be commented out. + +On Windows, the equivalent non-interactive command that `dsconfig` prints quotes each value by the rules of the C runtime, which splits the command line of the Java program. It does not escape the characters that `cmd.exe` interprets itself, such as `&`, `|`, `<`, and `>`, so when a value holds one of them, such as an ACI with an `&` filter, the command does not run in `cmd.exe` as printed. + The `dsconfig` command categorizes directory server configuration into __components__, also called __managed objects__. Actual components often inherit from a parent component type. For example, one component is a Connection Handler. An LDAP Connection Handler is a type of Connection Handler. You configure the LDAP Connection Handler component to specify how OpenDJ directory server handles LDAP connections coming from client applications. Configuration components have __properties__. For example, the LDAP Connection Handler component has properties such as `listen-port` and `allow-start-tls`. You can set the component's `listen-port` property to `389` to use the default LDAP port number. You can set the component's `allow-start-tls` property to `true` to permit LDAP client applications to use StartTLS. Much of the configuration you do with `dsconfig` involves setting component properties. diff --git a/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_files.adoc b/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_files.adoc index fee48a9751..9011df6fcc 100644 --- a/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_files.adoc +++ b/opendj-doc-generated-ref/src/main/asciidoc/man-pages/_files.adoc @@ -13,7 +13,7 @@ information: "Portions Copyright [year] [name of copyright owner]". Copyright 2015 ForgeRock AS. - Portions Copyright 2024 3A Systems LLC. + Portions Copyright 2024-2026 3A Systems LLC. //// @@ -35,3 +35,5 @@ ldapsearch.port=1389 ---- The location on Windows is `%UserProfile%/.opendj/tools.properties`. + +The file is read as UTF-8, or as ISO-8859-1 if it is not valid UTF-8. It uses the Java properties format, where a backslash starts an escape sequence. Double each backslash in a value, for example `bindDN=cn=a\\,b,dc=example,dc=com` for the DN `cn=a\,b,dc=example,dc=com`. diff --git a/opendj-doc-generated-ref/src/main/asciidoc/server-dev-guide/chap-ldap-operations.adoc b/opendj-doc-generated-ref/src/main/asciidoc/server-dev-guide/chap-ldap-operations.adoc index effc7ce00c..30c336a2db 100644 --- a/opendj-doc-generated-ref/src/main/asciidoc/server-dev-guide/chap-ldap-operations.adoc +++ b/opendj-doc-generated-ref/src/main/asciidoc/server-dev-guide/chap-ldap-operations.adoc @@ -12,7 +12,7 @@ information: "Portions copyright [year] [name of copyright owner]". Copyright 2017 ForgeRock AS. - Portions Copyright 2024 3A Systems LLC. + Portions Copyright 2024-2026 3A Systems LLC. //// :figure-caption!: @@ -1596,6 +1596,8 @@ ldapsearch.port=1389 ---- The location on Windows is `%UserProfile%/.opendj/tools.properties`. +The file is read as UTF-8, or as ISO-8859-1 if it is not valid UTF-8. It uses the Java properties format, where a backslash starts an escape sequence. Double each backslash in a value, for example `bindDN=cn=a\\,b,dc=example,dc=com` for the DN `cn=a\,b,dc=example,dc=com`. + [#client-auth] === Authenticating To the Directory Server diff --git a/opendj-server-legacy/resource/config/tools.properties b/opendj-server-legacy/resource/config/tools.properties index e03239c1d3..0290eb8d05 100644 --- a/opendj-server-legacy/resource/config/tools.properties +++ b/opendj-server-legacy/resource/config/tools.properties @@ -11,6 +11,11 @@ # information: "Portions Copyright [year] [name of copyright owner]". # # Copyright 2008 Sun Microsystems, Inc. +# Portions Copyright 2026 3A Systems, LLC. + +# This file is read as UTF-8, or as ISO-8859-1 if it is not valid UTF-8. +# A backslash starts an escape sequence, so double each backslash in a +# value: bindDN=cn=a\\,b,dc=example,dc=com sets cn=a\,b,dc=example,dc=com. # Default argument values. These arguments will be the # default values for all OpenDS client tools. Defaults