001/*
002 * Copyright 2020-2022 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2020-2022 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2020-2022 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldif;
037
038
039
040import java.io.BufferedReader;
041import java.io.File;
042import java.io.FileInputStream;
043import java.io.FileOutputStream;
044import java.io.IOException;
045import java.io.InputStream;
046import java.io.InputStreamReader;
047import java.io.OutputStream;
048import java.util.ArrayList;
049import java.util.Arrays;
050import java.util.Collections;
051import java.util.HashSet;
052import java.util.Iterator;
053import java.util.LinkedHashMap;
054import java.util.LinkedHashSet;
055import java.util.List;
056import java.util.Map;
057import java.util.Set;
058import java.util.TreeMap;
059import java.util.concurrent.atomic.AtomicReference;
060import java.util.zip.GZIPOutputStream;
061
062import com.unboundid.ldap.listener.SearchEntryParer;
063import com.unboundid.ldap.sdk.DN;
064import com.unboundid.ldap.sdk.Entry;
065import com.unboundid.ldap.sdk.Filter;
066import com.unboundid.ldap.sdk.InternalSDKHelper;
067import com.unboundid.ldap.sdk.LDAPException;
068import com.unboundid.ldap.sdk.LDAPURL;
069import com.unboundid.ldap.sdk.ResultCode;
070import com.unboundid.ldap.sdk.SearchResultEntry;
071import com.unboundid.ldap.sdk.SearchScope;
072import com.unboundid.ldap.sdk.Version;
073import com.unboundid.ldap.sdk.schema.EntryValidator;
074import com.unboundid.ldap.sdk.schema.Schema;
075import com.unboundid.ldap.sdk.unboundidds.tools.ColumnBasedLDAPResultWriter;
076import com.unboundid.ldap.sdk.unboundidds.tools.DNsOnlyLDAPResultWriter;
077import com.unboundid.ldap.sdk.unboundidds.tools.JSONLDAPResultWriter;
078import com.unboundid.ldap.sdk.unboundidds.tools.LDAPResultWriter;
079import com.unboundid.ldap.sdk.unboundidds.tools.LDIFLDAPResultWriter;
080import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
081import com.unboundid.ldap.sdk.unboundidds.tools.ValuesOnlyLDAPResultWriter;
082import com.unboundid.util.CommandLineTool;
083import com.unboundid.util.Debug;
084import com.unboundid.util.NotNull;
085import com.unboundid.util.Nullable;
086import com.unboundid.util.ObjectPair;
087import com.unboundid.util.OutputFormat;
088import com.unboundid.util.PassphraseEncryptedOutputStream;
089import com.unboundid.util.StaticUtils;
090import com.unboundid.util.ThreadSafety;
091import com.unboundid.util.ThreadSafetyLevel;
092import com.unboundid.util.args.ArgumentException;
093import com.unboundid.util.args.ArgumentParser;
094import com.unboundid.util.args.BooleanArgument;
095import com.unboundid.util.args.DNArgument;
096import com.unboundid.util.args.FileArgument;
097import com.unboundid.util.args.IntegerArgument;
098import com.unboundid.util.args.ScopeArgument;
099import com.unboundid.util.args.StringArgument;
100
101import static com.unboundid.ldif.LDIFMessages.*;
102
103
104
105/**
106 * This class provides a command-line tool that can be used to search for
107 * entries matching a given set of criteria in an LDIF file.
108 */
109@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
110public final class LDIFSearch
111       extends CommandLineTool
112{
113  /**
114   * The server root directory for the Ping Identity Directory Server (or
115   * related Ping Identity server product) that contains this tool, if
116   * applicable.
117   */
118  @Nullable private static final File PING_SERVER_ROOT =
119       InternalSDKHelper.getPingIdentityServerRoot();
120
121
122
123  /**
124   * Indicates whether the tool is running as part of a Ping Identity Directory
125   * Server (or related Ping Identity Server Product) installation.
126   */
127  private static final boolean PING_SERVER_AVAILABLE =
128       (PING_SERVER_ROOT != null);
129
130
131
132  /**
133   * The column at which to wrap long lines.
134   */
135  private static final int WRAP_COLUMN = StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
136
137
138
139  // The argument parser for this tool.
140  @Nullable private volatile ArgumentParser parser;
141
142  // The completion message for this tool.
143  @NotNull private final AtomicReference<String> completionMessage;
144
145  // Indicates whether the LDIF encryption passphrase file has been read.
146  private volatile boolean ldifEncryptionPassphraseFileRead;
147
148  // Encryption passphrases used thus far.
149  @NotNull private final List<char[]> inputEncryptionPassphrases;
150
151  // The list of LDAP URLs to use when processing searches, mapped to the
152  // corresponding search entry parers.
153  @NotNull private final List<LDAPURL> searchURLs;
154
155  // The LDAP result writer for this tool.
156  @NotNull private volatile LDAPResultWriter resultWriter;
157
158  // The command-line arguments supported by this tool.
159  @Nullable private BooleanArgument checkSchema;
160  @Nullable private BooleanArgument compressOutput;
161  @Nullable private BooleanArgument doNotWrap;
162  @Nullable private BooleanArgument encryptOutput;
163  @Nullable private BooleanArgument isCompressed;
164  @Nullable private BooleanArgument overwriteExistingOutputFile;
165  @Nullable private BooleanArgument separateOutputFilePerSearch;
166  @Nullable private BooleanArgument stripTrailingSpaces;
167  @Nullable private DNArgument baseDN;
168  @Nullable private FileArgument filterFile;
169  @Nullable private FileArgument ldapURLFile;
170  @Nullable private FileArgument ldifEncryptionPassphraseFile;
171  @Nullable private FileArgument ldifFile;
172  @Nullable private FileArgument outputFile;
173  @Nullable private FileArgument outputEncryptionPassphraseFile;
174  @Nullable private FileArgument schemaPath;
175  @Nullable private IntegerArgument sizeLimit;
176  @Nullable private IntegerArgument timeLimitSeconds;
177  @Nullable private IntegerArgument wrapColumn;
178  @Nullable private ScopeArgument scope;
179  @Nullable private StringArgument outputFormat = null;
180
181
182
183  /**
184   * Invokes this tool with the provided set of command-line arguments.
185   *
186   * @param  args  The set of arguments provided to this tool.  It may be
187   *               empty but must not be {@code null}.
188   */
189  public static void main(@NotNull final String... args)
190  {
191    final ResultCode resultCode = main(System.out, System.err, args);
192    if (resultCode != ResultCode.SUCCESS)
193    {
194      System.exit(resultCode.intValue());
195    }
196  }
197
198
199
200  /**
201   * Invokes this tool with the provided set of command-line arguments, using
202   * the given output and error streams.
203   *
204   * @param  out   The output stream to use for standard output.  It may be
205   *               {@code null} if standard output should be suppressed.
206   * @param  err   The output stream to use for standard error.  It may be
207   *               {@code null} if standard error should be suppressed.
208   * @param  args  The set of arguments provided to this tool.  It may be
209   *               empty but must not be {@code null}.
210   *
211   * @return  A result code indicating the status of processing.  Any result
212   *          code other than {@link ResultCode#SUCCESS} should be considered
213   *          an error.
214   */
215  @NotNull()
216  public static ResultCode main(@Nullable final OutputStream out,
217                                @Nullable final OutputStream err,
218                                @NotNull final String... args)
219  {
220    final LDIFSearch tool = new LDIFSearch(out, err);
221    return tool.runTool(args);
222  }
223
224
225
226  /**
227   * Creates a new instance of this tool with the provided output and error
228   * streams.
229   *
230   * @param  out  The output stream to use for standard output.  It may be
231   *              {@code null} if standard output should be suppressed.
232   * @param  err  The output stream to use for standard error.  It may be
233   *              {@code null} if standard error should be suppressed.
234   */
235  public LDIFSearch(@Nullable final OutputStream out,
236                    @Nullable final OutputStream err)
237  {
238    super(out, err);
239
240    resultWriter = new LDIFLDAPResultWriter(getOut(), WRAP_COLUMN);
241
242    parser = null;
243    completionMessage = new AtomicReference<>();
244    inputEncryptionPassphrases = new ArrayList<>(5);
245    searchURLs = new ArrayList<>();
246    ldifEncryptionPassphraseFileRead = false;
247
248    checkSchema = null;
249    compressOutput = null;
250    doNotWrap = null;
251    encryptOutput = null;
252    isCompressed = null;
253    overwriteExistingOutputFile = null;
254    separateOutputFilePerSearch = null;
255    stripTrailingSpaces = null;
256    baseDN = null;
257    filterFile = null;
258    ldapURLFile = null;
259    ldifEncryptionPassphraseFile = null;
260    ldifFile = null;
261    outputFile = null;
262    outputFormat = null;
263    outputEncryptionPassphraseFile = null;
264    schemaPath = null;
265    sizeLimit = null;
266    timeLimitSeconds = null;
267    wrapColumn = null;
268    scope = null;
269  }
270
271
272
273  /**
274   * {@inheritDoc}
275   */
276  @Override()
277  @NotNull()
278  public String getToolName()
279  {
280    return "ldifsearch";
281  }
282
283
284
285  /**
286   * {@inheritDoc}
287   */
288  @Override()
289  @NotNull()
290  public String getToolDescription()
291  {
292    return INFO_LDIFSEARCH_TOOL_DESCRIPTION.get();
293  }
294
295
296
297  /**
298   * {@inheritDoc}
299   */
300  @Override()
301  @NotNull()
302  public String getToolVersion()
303  {
304    return Version.NUMERIC_VERSION_STRING;
305  }
306
307
308
309  /**
310   * {@inheritDoc}
311   */
312  @Override()
313  public int getMinTrailingArguments()
314  {
315    return 0;
316  }
317
318
319
320  /**
321   * {@inheritDoc}
322   */
323  @Override()
324  public int getMaxTrailingArguments()
325  {
326    return -1;
327  }
328
329
330
331  /**
332   * {@inheritDoc}
333   */
334  @Override()
335  @NotNull()
336  public String getTrailingArgumentsPlaceholder()
337  {
338    return INFO_LDIFSEARCH_TRAILING_ARGS_PLACEHOLDER.get();
339  }
340
341
342
343  /**
344   * {@inheritDoc}
345   */
346  @Override()
347  public boolean supportsInteractiveMode()
348  {
349    return true;
350  }
351
352
353
354  /**
355   * {@inheritDoc}
356   */
357  @Override()
358  public boolean defaultsToInteractiveMode()
359  {
360    return true;
361  }
362
363
364
365  /**
366   * {@inheritDoc}
367   */
368  @Override()
369  public boolean supportsPropertiesFile()
370  {
371    return true;
372  }
373
374
375
376  /**
377   * {@inheritDoc}
378   */
379  @Override()
380  @Nullable()
381  protected String getToolCompletionMessage()
382  {
383    return completionMessage.get();
384  }
385
386
387
388  /**
389   * {@inheritDoc}
390   */
391  @Override()
392  public void addToolArguments(@NotNull final ArgumentParser parser)
393         throws ArgumentException
394  {
395    this.parser = parser;
396
397
398    ldifFile = new FileArgument('l', "ldifFile", true, 0, null,
399         INFO_LDIFSEARCH_ARG_DESC_LDIF_FILE.get(), true, true, true, false);
400    ldifFile.addLongIdentifier("ldif-file", true);
401    ldifFile.addLongIdentifier("inputFile", true);
402    ldifFile.addLongIdentifier("input-file", true);
403    ldifFile.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_INPUT.get());
404    parser.addArgument(ldifFile);
405
406
407    final String ldifPWDesc;
408    if (PING_SERVER_AVAILABLE)
409    {
410      ldifPWDesc = INFO_LDIFSEARCH_ARG_DESC_LDIF_PW_FILE_PING_SERVER.get();
411    }
412    else
413    {
414      ldifPWDesc = INFO_LDIFSEARCH_ARG_DESC_LDIF_PW_FILE_STANDALONE.get();
415    }
416    ldifEncryptionPassphraseFile = new FileArgument(null,
417         "ldifEncryptionPassphraseFile", false, 1, null, ldifPWDesc, true,
418         true, true, false);
419    ldifEncryptionPassphraseFile.addLongIdentifier(
420         "ldif-encryption-passphrase-file", true);
421    ldifEncryptionPassphraseFile.addLongIdentifier("ldifPassphraseFile", true);
422    ldifEncryptionPassphraseFile.addLongIdentifier("ldif-passphrase-file",
423         true);
424    ldifEncryptionPassphraseFile.addLongIdentifier("ldifEncryptionPasswordFile",
425         true);
426    ldifEncryptionPassphraseFile.addLongIdentifier(
427         "ldif-encryption-password-file", true);
428    ldifEncryptionPassphraseFile.addLongIdentifier("ldifPasswordFile", true);
429    ldifEncryptionPassphraseFile.addLongIdentifier("ldif-password-file", true);
430    ldifEncryptionPassphraseFile.addLongIdentifier(
431         "inputEncryptionPassphraseFile", true);
432    ldifEncryptionPassphraseFile.addLongIdentifier(
433         "input-encryption-passphrase-file", true);
434    ldifEncryptionPassphraseFile.addLongIdentifier("inputPassphraseFile", true);
435    ldifEncryptionPassphraseFile.addLongIdentifier("input-passphrase-file",
436         true);
437    ldifEncryptionPassphraseFile.addLongIdentifier(
438         "inputEncryptionPasswordFile", true);
439    ldifEncryptionPassphraseFile.addLongIdentifier(
440         "input-encryption-password-file", true);
441    ldifEncryptionPassphraseFile.addLongIdentifier("inputPasswordFile", true);
442    ldifEncryptionPassphraseFile.addLongIdentifier("input-password-file", true);
443    ldifEncryptionPassphraseFile.setArgumentGroupName(
444         INFO_LDIFSEARCH_ARG_GROUP_INPUT.get());
445    parser.addArgument(ldifEncryptionPassphraseFile);
446
447
448    stripTrailingSpaces = new BooleanArgument(null, "stripTrailingSpaces", 1,
449         INFO_LDIFSEARCH_ARG_DESC_STRIP_TRAILING_SPACES.get());
450    stripTrailingSpaces.addLongIdentifier("strip-trailing-spaces", true);
451    stripTrailingSpaces.addLongIdentifier("ignoreTrailingSpaces", true);
452    stripTrailingSpaces.addLongIdentifier("ignore-trailing-spaces", true);
453    stripTrailingSpaces.setArgumentGroupName(
454         INFO_LDIFSEARCH_ARG_GROUP_INPUT.get());
455    parser.addArgument(stripTrailingSpaces);
456
457
458    final String schemaPathDesc;
459    if (PING_SERVER_AVAILABLE)
460    {
461      schemaPathDesc = INFO_LDIFSEARCH_ARG_DESC_SCHEMA_PATH_PING_SERVER.get();
462    }
463    else
464    {
465      schemaPathDesc = INFO_LDIFSEARCH_ARG_DESC_SCHEMA_PATH_STANDALONE.get();
466    }
467    schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
468         schemaPathDesc, true, true, false, false);
469    schemaPath.addLongIdentifier("schema-path", true);
470    schemaPath.addLongIdentifier("schemaFile", true);
471    schemaPath.addLongIdentifier("schema-file", true);
472    schemaPath.addLongIdentifier("schemaDirectory", true);
473    schemaPath.addLongIdentifier("schema-directory", true);
474    schemaPath.addLongIdentifier("schema", true);
475    schemaPath.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_INPUT.get());
476    parser.addArgument(schemaPath);
477
478
479    checkSchema = new BooleanArgument(null, "checkSchema", 1,
480         INFO_LDIFSEARCH_ARG_DESC_CHECK_SCHEMA.get());
481    checkSchema.addLongIdentifier("check-schema", true);
482    checkSchema.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_INPUT.get());
483    parser.addArgument(checkSchema);
484
485
486    isCompressed = new BooleanArgument(null, "isCompressed", 1,
487         INFO_LDIFSEARCH_ARG_DESC_IS_COMPRESSED.get());
488    isCompressed.addLongIdentifier("is-compressed", true);
489    isCompressed.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_INPUT.get());
490    isCompressed.setHidden(true);
491    parser.addArgument(isCompressed);
492
493
494    outputFile = new FileArgument('o', "outputFile", false, 1, null,
495         INFO_LDIFSEARCH_ARG_DESC_OUTPUT_FILE.get(), false, true, true, false);
496    outputFile.addLongIdentifier("output-file", true);
497    outputFile.addLongIdentifier("outputLDIF", true);
498    outputFile.addLongIdentifier("output-ldif", true);
499    outputFile.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
500    parser.addArgument(outputFile);
501
502
503    separateOutputFilePerSearch = new BooleanArgument(null,
504         "separateOutputFilePerSearch", 1,
505         INFO_LDIFSEARCH_ARG_DESC_SEPARATE_OUTPUT_FILES.get());
506    separateOutputFilePerSearch.addLongIdentifier(
507         "separate-output-file-per-search", true);
508    separateOutputFilePerSearch.addLongIdentifier("separateOutputFiles", true);
509    separateOutputFilePerSearch.addLongIdentifier("separate-output-files",
510         true);
511    separateOutputFilePerSearch.setArgumentGroupName(
512         INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
513    parser.addArgument(separateOutputFilePerSearch);
514
515
516    compressOutput = new BooleanArgument(null, "compressOutput", 1,
517         INFO_LDIFSEARCH_ARG_DESC_COMPRESS_OUTPUT.get());
518    compressOutput.addLongIdentifier("compress-output", true);
519    compressOutput.addLongIdentifier("compressLDIF", true);
520    compressOutput.addLongIdentifier("compress-ldif", true);
521    compressOutput.addLongIdentifier("compress", true);
522    compressOutput.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
523    parser.addArgument(compressOutput);
524
525
526    encryptOutput = new BooleanArgument(null, "encryptOutput", 1,
527         INFO_LDIFSEARCH_ARG_DESC_ENCRYPT_OUTPUT.get());
528    encryptOutput.addLongIdentifier("encrypt-output", true);
529    encryptOutput.addLongIdentifier("encryptLDIF", true);
530    encryptOutput.addLongIdentifier("encrypt-ldif", true);
531    encryptOutput.addLongIdentifier("encrypt", true);
532    encryptOutput.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
533    parser.addArgument(encryptOutput);
534
535
536    outputEncryptionPassphraseFile = new FileArgument(null,
537         "outputEncryptionPassphraseFile", false, 1, null,
538         INFO_LDIFSEARCH_ARG_DESC_OUTPUT_PW_FILE.get(), true, true, true,
539         false);
540    outputEncryptionPassphraseFile.addLongIdentifier(
541         "output-encryption-passphrase-file", true);
542    outputEncryptionPassphraseFile.addLongIdentifier("outputPassphraseFile",
543         true);
544    outputEncryptionPassphraseFile.addLongIdentifier("output-passphrase-file",
545         true);
546    outputEncryptionPassphraseFile.addLongIdentifier(
547         "outputEncryptionPasswordFile", true);
548    outputEncryptionPassphraseFile.addLongIdentifier(
549         "output-encryption-password-file", true);
550    outputEncryptionPassphraseFile.addLongIdentifier("outputPasswordFile",
551         true);
552    outputEncryptionPassphraseFile.addLongIdentifier("output-password-file",
553         true);
554    outputEncryptionPassphraseFile.addLongIdentifier(
555         "outputEncryptionPasswordFile", true);
556    outputEncryptionPassphraseFile.addLongIdentifier(
557         "output-encryption-password-file", true);
558    outputEncryptionPassphraseFile.addLongIdentifier("outputPasswordFile",
559         true);
560    outputEncryptionPassphraseFile.addLongIdentifier("output-password-file",
561         true);
562    outputEncryptionPassphraseFile.setArgumentGroupName(
563         INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
564    parser.addArgument(outputEncryptionPassphraseFile);
565
566
567    overwriteExistingOutputFile = new BooleanArgument('O',
568         "overwriteExistingOutputFile", 1,
569         INFO_LDIFSEARCH_ARG_DESC_OVERWRITE_EXISTING.get());
570    overwriteExistingOutputFile.addLongIdentifier(
571         "overwrite-existing-output-file", true);
572    overwriteExistingOutputFile.addLongIdentifier(
573         "overwriteExistingOutputFiles", true);
574    overwriteExistingOutputFile.addLongIdentifier(
575         "overwrite-existing-output-files", true);
576    overwriteExistingOutputFile.addLongIdentifier("overwriteExistingOutput",
577         true);
578    overwriteExistingOutputFile.addLongIdentifier("overwrite-existing-output",
579         true);
580    overwriteExistingOutputFile.addLongIdentifier("overwriteExisting", true);
581    overwriteExistingOutputFile.addLongIdentifier("overwrite-existing", true);
582    overwriteExistingOutputFile.addLongIdentifier("overwrite", true);
583    overwriteExistingOutputFile.setArgumentGroupName(
584         INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
585    parser.addArgument(overwriteExistingOutputFile);
586
587
588    final Set<String> outputFormatAllowedValues = StaticUtils.setOf("ldif",
589         "json", "csv", "multi-valued-csv", "tab-delimited",
590         "multi-valued-tab-delimited", "dns-only", "values-only");
591    outputFormat = new StringArgument(null, "outputFormat", false, 1,
592         "{ldif|json|csv|multi-valued-csv|tab-delimited|" +
593              "multi-valued-tab-delimited|dns-only|values-only}",
594         INFO_LDIFSEARCH_ARG_DESC_OUTPUT_FORMAT.get(),
595         outputFormatAllowedValues, "ldif");
596    outputFormat.addLongIdentifier("output-format", true);
597    outputFormat.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
598    parser.addArgument(outputFormat);
599
600
601    wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
602         INFO_LDIFSEARCH_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
603    wrapColumn.addLongIdentifier("wrap-column", true);
604    wrapColumn.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
605    parser.addArgument(wrapColumn);
606
607
608    doNotWrap = new BooleanArgument('T', "doNotWrap", 1,
609         INFO_LDIFSEARCH_ARG_DESC_DO_NOT_WRAP.get());
610    doNotWrap.addLongIdentifier("do-not-wrap", true);
611    doNotWrap.addLongIdentifier("dontWrap", true);
612    doNotWrap.addLongIdentifier("dont-wrap", true);
613    doNotWrap.addLongIdentifier("noWrap", true);
614    doNotWrap.addLongIdentifier("no-wrap", true);
615    doNotWrap.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_OUTPUT.get());
616    parser.addArgument(doNotWrap);
617
618
619    baseDN = new DNArgument('b', "baseDN", false, 1, null,
620         INFO_LDIFSEARCH_ARG_DESC_BASE_DN.get());
621    baseDN.addLongIdentifier("base-dn", true);
622    baseDN.addLongIdentifier("searchBaseDN", true);
623    baseDN.addLongIdentifier("search-base-dn", true);
624    baseDN.addLongIdentifier("searchBase", true);
625    baseDN.addLongIdentifier("search-base", true);
626    baseDN.addLongIdentifier("base", true);
627    baseDN.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_CRITERIA.get());
628    parser.addArgument(baseDN);
629
630
631    scope = new ScopeArgument('s', "scope", false, null,
632         INFO_LDIFSEARCH_ARG_DESC_SCOPE.get());
633    scope.addLongIdentifier("searchScope", true);
634    scope.addLongIdentifier("search-scope", true);
635    scope.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_CRITERIA.get());
636    parser.addArgument(scope);
637
638
639    filterFile = new FileArgument('f', "filterFile", false, 0, null,
640         INFO_LDIFSEARCH_ARG_DESC_FILTER_FILE.get(), true, true, true, false);
641    filterFile.addLongIdentifier("filter-file", true);
642    filterFile.addLongIdentifier("filtersFile", true);
643    filterFile.addLongIdentifier("filters-file", true);
644    filterFile.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_CRITERIA.get());
645    parser.addArgument(filterFile);
646
647
648    ldapURLFile = new FileArgument(null, "ldapURLFile", false, 0, null,
649         INFO_LDIFSEARCH_ARG_DESC_LDAP_URL_FILE.get(), true, true, true, false);
650    ldapURLFile.addLongIdentifier("ldap-url-file", true);
651    ldapURLFile.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_CRITERIA.get());
652    parser.addArgument(ldapURLFile);
653
654
655    sizeLimit = new IntegerArgument('z', "sizeLimit", false, 1, null,
656         INFO_LDIFSEARCH_ARG_DESC_SIZE_LIMIT.get(), 0, Integer.MAX_VALUE, 0);
657    sizeLimit.addLongIdentifier("size-limit", true);
658    sizeLimit.addLongIdentifier("searchSizeLimit", true);
659    sizeLimit.addLongIdentifier("search-size-limit", true);
660    sizeLimit.setArgumentGroupName(INFO_LDIFSEARCH_ARG_GROUP_CRITERIA.get());
661    sizeLimit.setHidden(true);
662    parser.addArgument(sizeLimit);
663
664
665    timeLimitSeconds = new IntegerArgument('t', "timeLimitSeconds", false, 1,
666         null, INFO_LDIFSEARCH_ARG_DESC_TIME_LIMIT_SECONDS.get(), 0,
667         Integer.MAX_VALUE, 0);
668    timeLimitSeconds.addLongIdentifier("time-limit-seconds", true);
669    timeLimitSeconds.addLongIdentifier("timeLimit", true);
670    timeLimitSeconds.setArgumentGroupName(
671         INFO_LDIFSEARCH_ARG_GROUP_CRITERIA.get());
672    timeLimitSeconds.setHidden(true);
673    parser.addArgument(timeLimitSeconds);
674
675
676    parser.addDependentArgumentSet(separateOutputFilePerSearch, outputFile);
677    parser.addDependentArgumentSet(compressOutput, outputFile);
678    parser.addDependentArgumentSet(encryptOutput, outputFile);
679    parser.addDependentArgumentSet(overwriteExistingOutputFile, outputFile);
680    parser.addDependentArgumentSet(outputEncryptionPassphraseFile,
681         encryptOutput);
682
683    parser.addExclusiveArgumentSet(wrapColumn, doNotWrap);
684    parser.addExclusiveArgumentSet(baseDN, ldapURLFile);
685    parser.addExclusiveArgumentSet(scope, ldapURLFile);
686    parser.addExclusiveArgumentSet(filterFile, ldapURLFile);
687    parser.addExclusiveArgumentSet(outputFormat, separateOutputFilePerSearch);
688  }
689
690
691
692  /**
693   * {@inheritDoc}
694   */
695  @Override()
696  public void doExtendedArgumentValidation()
697         throws ArgumentException
698  {
699    // If the output file exists and either compressOutput or encryptOutput is
700    // present, then the overwrite argument must also be present.
701    final File outFile = outputFile.getValue();
702    if ((outFile != null) && outFile.exists() &&
703         (compressOutput.isPresent() || encryptOutput.isPresent()) &&
704         (! overwriteExistingOutputFile.isPresent()))
705    {
706      throw new ArgumentException(
707           ERR_LDIFSEARCH_APPEND_WITH_COMPRESSION_OR_ENCRYPTION.get(
708                compressOutput.getIdentifierString(),
709                encryptOutput.getIdentifierString(),
710                overwriteExistingOutputFile.getIdentifierString()));
711    }
712
713
714    // Create the set of LDAP URLs to use when issuing the searches.
715    final List<String> trailingArgs = parser.getTrailingArguments();
716    final List<String> requestedAttributes = new ArrayList<>();
717    if (filterFile.isPresent())
718    {
719      // If there are trailing arguments, then make sure the first one is not a
720      // valid filter.
721      if (! trailingArgs.isEmpty())
722      {
723        try
724        {
725          Filter.create(trailingArgs.get(0));
726          throw new ArgumentException(
727               ERR_LDIFSEARCH_FILTER_FILE_WITH_TRAILING_FILTER.get());
728        }
729        catch (final LDAPException e)
730        {
731          // This was expected.
732        }
733      }
734
735      requestedAttributes.addAll(trailingArgs);
736      readFilterFile();
737    }
738    else if (ldapURLFile.isPresent())
739    {
740      // Make sure there aren't any trailing arguments.
741      if (! trailingArgs.isEmpty())
742      {
743        throw new ArgumentException(
744             ERR_LDIFSEARCH_LDAP_URL_FILE_WITH_TRAILING_ARGS.get());
745      }
746
747      readLDAPURLFile();
748
749
750      // If there are multiple LDAP URLs, and if they should not be sent to
751      // separate output files, then they must all have the same set of
752      // requested attributes.
753      if ((searchURLs.size() > 1) &&
754           (! separateOutputFilePerSearch.isPresent()))
755      {
756        final Iterator<LDAPURL> iterator = searchURLs.iterator();
757        final Set<String> requestedAttrs =
758             new HashSet<>(Arrays.asList(iterator.next().getAttributes()));
759        while (iterator.hasNext())
760        {
761          final Set<String> attrSet = new HashSet<>(Arrays.asList(
762               iterator.next().getAttributes()));
763          if (! requestedAttrs.equals(attrSet))
764          {
765            throw new ArgumentException(
766                 ERR_LDIFSEARCH_DIFFERENT_URL_ATTRS_IN_SAME_FILE.get(
767                      ldapURLFile.getIdentifierString(),
768                      separateOutputFilePerSearch.getIdentifierString()));
769          }
770        }
771      }
772    }
773    else
774    {
775      // Make sure there is at least one trailing argument, and that it's a
776      // valid filter.  If there are any others, then they must be the
777      // requested arguments.
778      if (trailingArgs.isEmpty())
779      {
780        throw new ArgumentException(ERR_LDIFSEARCH_NO_FILTER.get());
781      }
782
783
784      final Filter filter;
785      try
786      {
787        final List<String> trailingArgList = new ArrayList<>(trailingArgs);
788        final Iterator<String> trailingArgIterator = trailingArgList.iterator();
789        filter = Filter.create(trailingArgIterator.next());
790
791        while (trailingArgIterator.hasNext())
792        {
793          requestedAttributes.add(trailingArgIterator.next());
794        }
795      }
796      catch (final LDAPException e)
797      {
798        Debug.debugException(e);
799        throw new ArgumentException(
800             ERR_LDIFSEARCH_FIRST_TRAILING_ARG_NOT_FILTER.get(), e);
801      }
802
803
804      DN dn = baseDN.getValue();
805      if (dn == null)
806      {
807        dn = DN.NULL_DN;
808      }
809
810      SearchScope searchScope = scope.getValue();
811      if (searchScope == null)
812      {
813        searchScope = SearchScope.SUB;
814      }
815
816      try
817      {
818        searchURLs.add(new LDAPURL("ldap", null, null, dn,
819             requestedAttributes.toArray(StaticUtils.NO_STRINGS),
820             searchScope, filter));
821      }
822      catch (final LDAPException e)
823      {
824        Debug.debugException(e);
825        // This should never happen.
826        throw new ArgumentException(StaticUtils.getExceptionMessage(e), e);
827      }
828    }
829
830
831    // Create the result writer.
832    final String outputFormatStr =
833         StaticUtils.toLowerCase(outputFormat.getValue());
834    if (outputFormatStr.equals("json"))
835    {
836      resultWriter = new JSONLDAPResultWriter(getOut());
837    }
838    else if (outputFormatStr.equals("csv") ||
839             outputFormatStr.equals("multi-valued-csv") ||
840             outputFormatStr.equals("tab-delimited") ||
841             outputFormatStr.equals("multi-valued-tab-delimited"))
842    {
843      // These output formats cannot be used with the --ldapURLFile argument.
844      if (ldapURLFile.isPresent())
845      {
846        throw new ArgumentException(
847             ERR_LDIFSEARCH_OUTPUT_FORMAT_NOT_SUPPORTED_WITH_URLS.get(
848                  outputFormat.getValue(), ldapURLFile.getIdentifierString()));
849      }
850
851
852      // These output formats require a set of requested attributes.
853      if (requestedAttributes.isEmpty())
854      {
855        throw new ArgumentException(
856             ERR_LDIFSEARCH_OUTPUT_FORMAT_REQUIRES_REQUESTED_ATTRS.get(
857                  outputFormat.getValue()));
858      }
859
860      final OutputFormat format;
861      final boolean includeAllValues;
862      switch (outputFormatStr)
863      {
864        case "multi-valued-csv":
865          format = OutputFormat.CSV;
866          includeAllValues = true;
867          break;
868        case "tab-delimited":
869          format = OutputFormat.TAB_DELIMITED_TEXT;
870          includeAllValues = false;
871          break;
872        case "multi-valued-tab-delimited":
873          format = OutputFormat.TAB_DELIMITED_TEXT;
874          includeAllValues = true;
875          break;
876        case "csv":
877        default:
878          format = OutputFormat.CSV;
879          includeAllValues = false;
880          break;
881      }
882
883
884      resultWriter = new ColumnBasedLDAPResultWriter(getOut(),
885           format, requestedAttributes, WRAP_COLUMN, includeAllValues);
886    }
887    else if (outputFormatStr.equals("dns-only"))
888    {
889      resultWriter = new DNsOnlyLDAPResultWriter(getOut());
890    }
891    else if (outputFormatStr.equals("values-only"))
892    {
893      resultWriter = new ValuesOnlyLDAPResultWriter(getOut());
894    }
895    else
896    {
897      final int wc;
898      if (doNotWrap.isPresent())
899      {
900        wc = Integer.MAX_VALUE;
901      }
902      else if (wrapColumn.isPresent())
903      {
904        wc = wrapColumn.getValue();
905      }
906      else
907      {
908        wc = WRAP_COLUMN;
909      }
910
911      resultWriter = new LDIFLDAPResultWriter(getOut(), wc);
912    }
913  }
914
915
916
917  /**
918   * Uses the contents of any specified filter files, along with the configured
919   * base DN, scope, and requested attributes, to populate the set of search
920   * URLs.
921   *
922   * @throws  ArgumentException  If a problem is encountered while constructing
923   *                             the search URLs.
924   */
925  private void readFilterFile()
926          throws ArgumentException
927  {
928    DN dn = baseDN.getValue();
929    if (dn == null)
930    {
931      dn = DN.NULL_DN;
932    }
933
934    SearchScope searchScope = scope.getValue();
935    if (searchScope == null)
936    {
937      searchScope = SearchScope.SUB;
938    }
939
940    final String[] requestedAttributes =
941         parser.getTrailingArguments().toArray(StaticUtils.NO_STRINGS);
942
943    for (final File f : filterFile.getValues())
944    {
945      final InputStream inputStream;
946      try
947      {
948        inputStream = openInputStream(f);
949      }
950      catch (final LDAPException e)
951      {
952        Debug.debugException(e);
953        throw new ArgumentException(e.getMessage(), e);
954      }
955
956      try (BufferedReader reader =
957                new BufferedReader(new InputStreamReader(inputStream)))
958      {
959        while (true)
960        {
961          final String line = reader.readLine();
962          if (line == null)
963          {
964            break;
965          }
966
967          if (line.isEmpty() || line.startsWith("#"))
968          {
969            continue;
970          }
971
972          try
973          {
974            final Filter filter = Filter.create(line.trim());
975            searchURLs.add(new LDAPURL("ldap", null, null, dn,
976                 requestedAttributes, searchScope, filter));
977          }
978          catch (final LDAPException e)
979          {
980            Debug.debugException(e);
981            throw new ArgumentException(
982                 ERR_LDIFSEARCH_FILTER_FILE_INVALID_FILTER.get(line,
983                      f.getAbsolutePath(), e.getMessage()),
984                 e);
985          }
986        }
987      }
988      catch (final IOException e)
989      {
990        Debug.debugException(e);
991        throw new ArgumentException(
992             ERR_LDIFSEARCH_ERROR_READING_FILTER_FILE.get(f.getAbsolutePath(),
993                  StaticUtils.getExceptionMessage(e)),
994             e);
995      }
996      finally
997      {
998        try
999        {
1000          inputStream.close();
1001        }
1002        catch (final Exception e)
1003        {
1004          Debug.debugException(e);
1005        }
1006      }
1007
1008    }
1009
1010    if (searchURLs.isEmpty())
1011    {
1012      throw new ArgumentException(ERR_LDIFSEARCH_NO_FILTERS_FROM_FILE.get(
1013           filterFile.getValues().get(0).getAbsolutePath()));
1014    }
1015  }
1016
1017
1018
1019  /**
1020   * Uses the contents of any specified LDAP URL files to populate the set of
1021   * search URLs.
1022   *
1023   * @throws  ArgumentException  If a problem is encountered while constructing
1024   *                             the search URLs.
1025   */
1026  private void readLDAPURLFile()
1027          throws ArgumentException
1028  {
1029    for (final File f : ldapURLFile.getValues())
1030    {
1031      final InputStream inputStream;
1032      try
1033      {
1034        inputStream = openInputStream(f);
1035      }
1036      catch (final LDAPException e)
1037      {
1038        Debug.debugException(e);
1039        throw new ArgumentException(e.getMessage(), e);
1040      }
1041
1042      try (BufferedReader reader =
1043                new BufferedReader(new InputStreamReader(inputStream)))
1044      {
1045        while (true)
1046        {
1047          final String line = reader.readLine();
1048          if (line == null)
1049          {
1050            break;
1051          }
1052
1053          if (line.isEmpty() || line.startsWith("#"))
1054          {
1055            continue;
1056          }
1057
1058          try
1059          {
1060            searchURLs.add(new LDAPURL(line.trim()));
1061          }
1062          catch (final LDAPException e)
1063          {
1064            Debug.debugException(e);
1065            throw new ArgumentException(
1066                 ERR_LDIFSEARCH_LDAP_URL_FILE_INVALID_URL.get(line,
1067                      f.getAbsolutePath(), e.getMessage()),
1068                 e);
1069          }
1070        }
1071      }
1072      catch (final IOException e)
1073      {
1074        Debug.debugException(e);
1075        throw new ArgumentException(
1076             ERR_LDIFSEARCH_ERROR_READING_LDAP_URL_FILE.get(f.getAbsolutePath(),
1077                  StaticUtils.getExceptionMessage(e)),
1078             e);
1079      }
1080      finally
1081      {
1082        try
1083        {
1084          inputStream.close();
1085        }
1086        catch (final Exception e)
1087        {
1088          Debug.debugException(e);
1089        }
1090      }
1091    }
1092
1093    if (searchURLs.isEmpty())
1094    {
1095      throw new ArgumentException(ERR_LDIFSEARCH_NO_URLS_FROM_FILE.get(
1096           ldapURLFile.getValues().get(0).getAbsolutePath()));
1097    }
1098  }
1099
1100
1101
1102  /**
1103   * {@inheritDoc}
1104   */
1105  @Override()
1106  @NotNull()
1107  public ResultCode doToolProcessing()
1108  {
1109    // Get the schema to use when performing LDIF processing.
1110    final Schema schema;
1111    try
1112    {
1113      if (schemaPath.isPresent())
1114      {
1115        schema = getSchema(schemaPath.getValues());
1116      }
1117      else if (PING_SERVER_AVAILABLE)
1118      {
1119        schema = getSchema(Collections.singletonList(StaticUtils.constructPath(
1120             PING_SERVER_ROOT, "config", "schema")));
1121      }
1122      else
1123      {
1124        schema = Schema.getDefaultStandardSchema();
1125      }
1126    }
1127    catch (final Exception e)
1128    {
1129      Debug.debugException(e);
1130      logCompletionMessage(true,
1131           ERR_LDIFSEARCH_CANNOT_GET_SCHEMA.get(
1132                StaticUtils.getExceptionMessage(e)));
1133      return ResultCode.LOCAL_ERROR;
1134    }
1135
1136
1137    // Create search entry parers for all of the search URLs.
1138    final Map<LDAPURL,SearchEntryParer> urlMap = new LinkedHashMap<>();
1139    for (final LDAPURL url : searchURLs)
1140    {
1141      final SearchEntryParer parer = new SearchEntryParer(
1142           Arrays.asList(url.getAttributes()), schema);
1143      urlMap.put(url, parer);
1144    }
1145
1146
1147    // If we should check schema, then create the entry validator.
1148    final EntryValidator entryValidator;
1149    if (checkSchema.isPresent())
1150    {
1151      entryValidator = new EntryValidator(schema);
1152    }
1153    else
1154    {
1155      entryValidator = null;
1156    }
1157
1158
1159    // Create the output files, if appropriate.
1160    OutputStream outputStream = null;
1161    SearchEntryParer singleParer = null;
1162    final Map<LDAPURL,LDIFSearchSeparateSearchDetails> separateWriters =
1163         new LinkedHashMap<>();
1164    try
1165    {
1166      if (outputFile.isPresent())
1167      {
1168        final int numURLs = searchURLs.size();
1169        if (separateOutputFilePerSearch.isPresent() && (numURLs > 1))
1170        {
1171          int i=1;
1172          for (final LDAPURL url : searchURLs)
1173          {
1174            final File f = new
1175                 File(outputFile.getValue().getAbsolutePath() + '.' + i);
1176            final LDIFSearchSeparateSearchDetails details =
1177                 new LDIFSearchSeparateSearchDetails(url, f,
1178                      createLDIFWriter(f, url), schema);
1179            separateWriters.put(url, details);
1180            i++;
1181          }
1182        }
1183        else
1184        {
1185          try
1186          {
1187            outputStream = createOutputStream(outputFile.getValue());
1188            resultWriter.updateOutputStream(outputStream);
1189          }
1190          catch (final Exception e)
1191          {
1192            Debug.debugException(e);
1193            throw new LDAPException(ResultCode.LOCAL_ERROR,
1194                 ERR_LDIFSEARCH_CANNOT_WRITE_TO_FILE.get(
1195                      outputFile.getValue().getAbsolutePath(),
1196                      StaticUtils.getExceptionMessage(e)),
1197                 e);
1198          }
1199        }
1200      }
1201
1202
1203      // If we're not using separate writers, then write any appropriate header
1204      // to the top of the output.
1205      if (separateWriters.isEmpty())
1206      {
1207        resultWriter.writeHeader();
1208      }
1209
1210
1211      // Iterate through the LDIF files and process the entries they contain.
1212      boolean errorEncountered = false;
1213      final List<LDAPURL> matchingURLs = new ArrayList<>();
1214      final List<String> entryInvalidReasons = new ArrayList<>();
1215      for (final File f : ldifFile.getValues())
1216      {
1217        final LDIFReader ldifReader;
1218        try
1219        {
1220          ldifReader = new LDIFReader(openInputStream(f));
1221
1222          if (stripTrailingSpaces.isPresent())
1223          {
1224            ldifReader.setTrailingSpaceBehavior(TrailingSpaceBehavior.STRIP);
1225          }
1226          else
1227          {
1228            ldifReader.setTrailingSpaceBehavior(TrailingSpaceBehavior.REJECT);
1229          }
1230        }
1231        catch (final Exception e)
1232        {
1233          Debug.debugException(e);
1234          logCompletionMessage(true,
1235               ERR_LDIFSEARCH_CANNOT_OPEN_LDIF_FILE.get(f.getName(),
1236                    StaticUtils.getExceptionMessage(e)));
1237          return ResultCode.LOCAL_ERROR;
1238        }
1239
1240        try
1241        {
1242          while (true)
1243          {
1244            final Entry entry;
1245            try
1246            {
1247              entry = ldifReader.readEntry();
1248            }
1249            catch (final LDIFException e)
1250            {
1251              Debug.debugException(e);
1252              if (e.mayContinueReading())
1253              {
1254                commentToErr(ERR_LDIFSEARCH_RECOVERABLE_READ_ERROR.get(
1255                     f.getAbsolutePath(), StaticUtils.getExceptionMessage(e)));
1256                errorEncountered = true;
1257                continue;
1258              }
1259              else
1260              {
1261                logCompletionMessage(true,
1262                     ERR_LDIFSEARCH_UNRECOVERABLE_READ_ERROR.get(
1263                          f.getAbsolutePath(),
1264                          StaticUtils.getExceptionMessage(e)));
1265                return ResultCode.LOCAL_ERROR;
1266              }
1267            }
1268            catch (final Exception e)
1269            {
1270              logCompletionMessage(true,
1271                   ERR_LDIFSEARCH_UNRECOVERABLE_READ_ERROR.get(
1272                        f.getAbsolutePath(),
1273                        StaticUtils.getExceptionMessage(e)));
1274              return ResultCode.LOCAL_ERROR;
1275            }
1276
1277            if (entry == null)
1278            {
1279              break;
1280            }
1281
1282            if (entryValidator != null)
1283            {
1284              entryInvalidReasons.clear();
1285              if (! entryValidator.entryIsValid(entry, entryInvalidReasons))
1286              {
1287                commentToErr(ERR_LDIFSEARCH_ENTRY_VIOLATES_SCHEMA.get(
1288                     entry.getDN()));
1289                for (final String invalidReason : entryInvalidReasons)
1290                {
1291                  commentToErr("- " + invalidReason);
1292                }
1293
1294                err();
1295                errorEncountered = true;
1296                continue;
1297              }
1298            }
1299
1300            if (separateWriters.isEmpty())
1301            {
1302              matchingURLs.clear();
1303              for (final LDAPURL url : searchURLs)
1304              {
1305                if (urlMatchesEntry(url, entry))
1306                {
1307                  matchingURLs.add(url);
1308                }
1309              }
1310
1311              if (matchingURLs.isEmpty())
1312              {
1313                continue;
1314              }
1315
1316              try
1317              {
1318                if (searchURLs.size() > 1)
1319                {
1320                  resultWriter.writeComment(
1321                       INFO_LDIFSEARCH_ENTRY_MATCHES_URLS.get(entry.getDN()));
1322                  for (final LDAPURL url : matchingURLs)
1323                  {
1324                    resultWriter.writeComment(url.toString());
1325                  }
1326                }
1327
1328                if (singleParer == null)
1329                {
1330                  singleParer = new SearchEntryParer(
1331                       Arrays.asList(searchURLs.get(0).getAttributes()),
1332                       schema);
1333                }
1334
1335                resultWriter.writeSearchResultEntry(
1336                     new SearchResultEntry(singleParer.pareEntry(entry)));
1337
1338                if (! outputFile.isPresent())
1339                {
1340                  resultWriter.flush();
1341                }
1342              }
1343              catch (final Exception e)
1344              {
1345                Debug.debugException(e);
1346                if (outputFile.isPresent())
1347                {
1348                  logCompletionMessage(true,
1349                       ERR_LDIFSEARCH_WRITE_ERROR_WITH_FILE.get(entry.getDN(),
1350                            outputFile.getValue().getAbsolutePath(),
1351                            StaticUtils.getExceptionMessage(e)));
1352                }
1353                else
1354                {
1355                  logCompletionMessage(true,
1356                       ERR_LDIFSEARCH_WRITE_ERROR_NO_FILE.get(entry.getDN(),
1357                            StaticUtils.getExceptionMessage(e)));
1358                }
1359                return ResultCode.LOCAL_ERROR;
1360              }
1361            }
1362            else
1363            {
1364              for (final LDIFSearchSeparateSearchDetails details :
1365                   separateWriters.values())
1366              {
1367                final LDAPURL url = details.getLDAPURL();
1368                if (urlMatchesEntry(url, entry))
1369                {
1370                  try
1371                  {
1372                    final Entry paredEntry =
1373                         details.getSearchEntryParer().pareEntry(entry);
1374                    details.getLDIFWriter().writeEntry(paredEntry);
1375                  }
1376                  catch (final Exception ex)
1377                  {
1378                    Debug.debugException(ex);
1379                    logCompletionMessage(true,
1380                         ERR_LDIFSEARCH_WRITE_ERROR_WITH_FILE.get(entry.getDN(),
1381                              details.getOutputFile().getAbsolutePath(),
1382                              StaticUtils.getExceptionMessage(ex)));
1383                    return ResultCode.LOCAL_ERROR;
1384                  }
1385                }
1386              }
1387            }
1388          }
1389        }
1390        finally
1391        {
1392          try
1393          {
1394            ldifReader.close();
1395          }
1396          catch (final Exception e)
1397          {
1398            Debug.debugException(e);
1399          }
1400        }
1401      }
1402
1403      if (errorEncountered)
1404      {
1405        logCompletionMessage(true,
1406             WARN_LDIFSEARCH_COMPLETED_WITH_ERRORS.get());
1407        return ResultCode.PARAM_ERROR;
1408      }
1409      else
1410      {
1411        logCompletionMessage(false,
1412             INFO_LDIFSEARCH_COMPLETED_SUCCESSFULLY.get());
1413        return ResultCode.SUCCESS;
1414      }
1415    }
1416    catch (final LDAPException e)
1417    {
1418      Debug.debugException(e);
1419      logCompletionMessage(true, e.getMessage());
1420      return e.getResultCode();
1421    }
1422    finally
1423    {
1424      try
1425      {
1426        resultWriter.flush();
1427        if (outputStream != null)
1428        {
1429          outputStream.close();
1430        }
1431      }
1432      catch (final Exception e)
1433      {
1434        Debug.debugException(e);
1435      }
1436
1437      for (final LDIFSearchSeparateSearchDetails details :
1438           separateWriters.values())
1439      {
1440        try
1441        {
1442          details.getLDIFWriter().close();
1443        }
1444        catch (final Exception e)
1445        {
1446          Debug.debugException(e);
1447        }
1448      }
1449    }
1450  }
1451
1452
1453
1454  /**
1455   * Retrieves the schema contained in the specified paths.
1456   *
1457   * @param  paths  The paths to use to access the schema.
1458   *
1459   * @return  The schema read from the specified files.
1460   *
1461   * @throws  Exception  If a problem is encountered while loading the schema.
1462   */
1463  @NotNull()
1464  private static Schema getSchema(@NotNull final List<File> paths)
1465          throws Exception
1466  {
1467    final Set<File> schemaFiles = new LinkedHashSet<>();
1468    for (final File f : paths)
1469    {
1470      if (f.exists())
1471      {
1472        if (f.isFile())
1473        {
1474          schemaFiles.add(f);
1475        }
1476        else if (f.isDirectory())
1477        {
1478          final TreeMap<String,File> sortedFiles = new TreeMap<>();
1479          for (final File fileInDir : f.listFiles())
1480          {
1481            if (fileInDir.isFile())
1482            {
1483              sortedFiles.put(fileInDir.getName(), fileInDir);
1484            }
1485          }
1486
1487          schemaFiles.addAll(sortedFiles.values());
1488        }
1489      }
1490    }
1491
1492    return Schema.getSchema(new ArrayList<>(schemaFiles));
1493  }
1494
1495
1496
1497  /**
1498   * Opens the input stream to use to read from the specified file.
1499   *
1500   * @param  f  The file for which to open the input stream.  It may optionally
1501   *            be compressed and/or encrypted.
1502   *
1503   * @return  The input stream that was created.
1504   *
1505   * @throws  LDAPException  If a problem is encountered while opening the file.
1506   */
1507  @NotNull()
1508  private InputStream openInputStream(@NotNull final File f)
1509          throws LDAPException
1510  {
1511    if (ldifEncryptionPassphraseFile.isPresent() &&
1512       (! ldifEncryptionPassphraseFileRead))
1513    {
1514      readPassphraseFile(ldifEncryptionPassphraseFile.getValue());
1515      ldifEncryptionPassphraseFileRead = true;
1516    }
1517
1518
1519    boolean closeStream = true;
1520    InputStream inputStream = null;
1521    try
1522    {
1523      inputStream = new FileInputStream(f);
1524
1525      final ObjectPair<InputStream,char[]> p =
1526           ToolUtils.getPossiblyPassphraseEncryptedInputStream(
1527                inputStream, inputEncryptionPassphrases,
1528                (! ldifEncryptionPassphraseFile.isPresent()),
1529                INFO_LDIFSEARCH_ENTER_ENCRYPTION_PW.get(f.getName()),
1530                ERR_LDIFSEARCH_WRONG_ENCRYPTION_PW.get(), getOut(), getErr());
1531      inputStream = p.getFirst();
1532      addPassphrase(p.getSecond());
1533
1534      inputStream = ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream);
1535      closeStream = false;
1536      return inputStream;
1537    }
1538    catch (final Exception e)
1539    {
1540      Debug.debugException(e);
1541      throw new LDAPException(ResultCode.LOCAL_ERROR,
1542           ERR_LDIFSEARCH_ERROR_OPENING_INPUT_FILE.get(f.getAbsolutePath(),
1543                StaticUtils.getExceptionMessage(e)),
1544           e);
1545    }
1546    finally
1547    {
1548      if ((inputStream != null) && closeStream)
1549      {
1550        try
1551        {
1552          inputStream.close();
1553        }
1554        catch (final Exception e)
1555        {
1556          Debug.debugException(e);
1557        }
1558      }
1559    }
1560  }
1561
1562
1563
1564  /**
1565   * Reads the contents of the specified passphrase file and adds it to the list
1566   * of passphrases.
1567   *
1568   * @param  f  The passphrase file to read.
1569   *
1570   * @throws  LDAPException  If a problem is encountered while trying to read
1571   *                         the passphrase from the provided file.
1572   */
1573  private void readPassphraseFile(@NotNull final File f)
1574          throws LDAPException
1575  {
1576    try
1577    {
1578      addPassphrase(getPasswordFileReader().readPassword(f));
1579    }
1580    catch (final Exception e)
1581    {
1582      Debug.debugException(e);
1583      throw new LDAPException(ResultCode.LOCAL_ERROR,
1584           ERR_LDIFSEARCH_CANNOT_READ_PW_FILE.get(f.getAbsolutePath(),
1585                StaticUtils.getExceptionMessage(e)),
1586           e);
1587    }
1588  }
1589
1590
1591
1592  /**
1593   * Updates the list of encryption passphrases with the provided passphrase, if
1594   * it is not already present.
1595   *
1596   * @param  passphrase  The passphrase to be added.  It may optionally be
1597   *                     {@code null} (in which case no action will be taken).
1598   */
1599  private void addPassphrase(@Nullable final char[] passphrase)
1600  {
1601    if (passphrase == null)
1602    {
1603      return;
1604    }
1605
1606    for (final char[] existingPassphrase : inputEncryptionPassphrases)
1607    {
1608      if (Arrays.equals(existingPassphrase, passphrase))
1609      {
1610        return;
1611      }
1612    }
1613
1614    inputEncryptionPassphrases.add(passphrase);
1615  }
1616
1617
1618
1619  /**
1620   * Creates an output stream that may be used to write to the specified file.
1621   *
1622   * @param  f  The file to be written.
1623   *
1624   * @return  The output stream that was created.
1625   *
1626   * @throws  LDAPException  If a problem occurs while creating the output
1627   *                         stream.
1628   */
1629  @NotNull()
1630  private OutputStream createOutputStream(@NotNull final File f)
1631          throws LDAPException
1632  {
1633    OutputStream outputStream = null;
1634    boolean closeOutputStream = true;
1635    try
1636    {
1637      try
1638      {
1639
1640        outputStream = new FileOutputStream(f,
1641             (! overwriteExistingOutputFile.isPresent()));
1642      }
1643      catch (final Exception e)
1644      {
1645        Debug.debugException(e);
1646        throw new LDAPException(ResultCode.LOCAL_ERROR,
1647             ERR_LDIFSEARCH_CANNOT_OPEN_OUTPUT_FILE.get(f.getAbsolutePath(),
1648                  StaticUtils.getExceptionMessage(e)),
1649             e);
1650      }
1651
1652      if (encryptOutput.isPresent())
1653      {
1654        try
1655        {
1656          final char[] passphrase;
1657          if (outputEncryptionPassphraseFile.isPresent())
1658          {
1659            passphrase = getPasswordFileReader().readPassword(
1660                 outputEncryptionPassphraseFile.getValue());
1661          }
1662          else
1663          {
1664            passphrase = ToolUtils.promptForEncryptionPassphrase(false, true,
1665                 INFO_LDIFSEARCH_PROMPT_OUTPUT_FILE_ENC_PW.get(),
1666                 INFO_LDIFSEARCH_CONFIRM_OUTPUT_FILE_ENC_PW.get(), getOut(),
1667                 getErr()).toCharArray();
1668          }
1669
1670          outputStream = new PassphraseEncryptedOutputStream(passphrase,
1671               outputStream, null, true, true);
1672        }
1673        catch (final Exception e)
1674        {
1675          Debug.debugException(e);
1676          throw new LDAPException(ResultCode.LOCAL_ERROR,
1677               ERR_LDIFSEARCH_CANNOT_ENCRYPT_OUTPUT_FILE.get(
1678                    StaticUtils.getExceptionMessage(e)),
1679               e);
1680        }
1681      }
1682
1683      if (compressOutput.isPresent())
1684      {
1685        try
1686        {
1687          outputStream = new GZIPOutputStream(outputStream);
1688        }
1689        catch (final Exception e)
1690        {
1691          Debug.debugException(e);
1692          throw new LDAPException(ResultCode.LOCAL_ERROR,
1693               ERR_LDIFSEARCH_CANNOT_COMPRESS_OUTPUT_FILE.get(
1694                    f.getAbsolutePath(), StaticUtils.getExceptionMessage(e)),
1695               e);
1696        }
1697      }
1698
1699      closeOutputStream = false;
1700      return outputStream;
1701    }
1702    finally
1703    {
1704      if (closeOutputStream && (outputStream != null))
1705      {
1706        try
1707        {
1708          outputStream.close();
1709        }
1710        catch (final Exception e)
1711        {
1712          Debug.debugException(e);
1713        }
1714      }
1715    }
1716  }
1717
1718
1719
1720  /**
1721   * Creates an LDIF writer to write to the specified file.
1722   *
1723   * @param  f        The file to be written.
1724   * @param  ldapURL  The LDAP URL with which the file will be associated.  It
1725   *                  may be {@code null} if the file is shared across multiple
1726   *                  URLs.
1727   *
1728   * @return  The LDIF writer that was created.
1729   *
1730   * @throws  LDAPException  If a problem occurs while creating the LDIF writer.
1731   */
1732  @NotNull()
1733  private LDIFWriter createLDIFWriter(@NotNull final File f,
1734                                      @Nullable final LDAPURL ldapURL)
1735          throws LDAPException
1736  {
1737    boolean closeOutputStream = true;
1738    final OutputStream outputStream = createOutputStream(f);
1739    try
1740    {
1741      final LDIFWriter ldifWriter = new LDIFWriter(outputStream);
1742      if (doNotWrap.isPresent())
1743      {
1744        ldifWriter.setWrapColumn(0);
1745      }
1746      else if (wrapColumn.isPresent())
1747      {
1748        ldifWriter.setWrapColumn(wrapColumn.getValue());
1749      }
1750      else
1751      {
1752        ldifWriter.setWrapColumn(WRAP_COLUMN);
1753      }
1754
1755      if (ldapURL != null)
1756      {
1757        try
1758        {
1759          ldifWriter.writeComment(
1760               INFO_LDIFSEARCH_ENTRIES_MATCHING_URL.get(ldapURL.toString()),
1761               false, true);
1762        }
1763        catch (final Exception e)
1764        {
1765          Debug.debugException(e);
1766        }
1767      }
1768
1769      closeOutputStream = false;
1770      return ldifWriter;
1771    }
1772    finally
1773    {
1774      if (closeOutputStream)
1775      {
1776        try
1777        {
1778          outputStream.close();
1779        }
1780        catch (final Exception e)
1781        {
1782          Debug.debugException(e);
1783        }
1784      }
1785    }
1786  }
1787
1788
1789
1790  /**
1791   * Indicates whether the given entry matches the criteria in the provided LDAP
1792   * URL.
1793   *
1794   * @param  url    The URL for which to make the determination.
1795   * @param  entry  The entry for which to make the determination.
1796   *
1797   * @return  {@code true} if the entry matches the criteria in the LDAP URL, or
1798   *          {@code false} if not.
1799   */
1800  private boolean urlMatchesEntry(@NotNull final LDAPURL url,
1801                                  @NotNull final Entry entry)
1802  {
1803    try
1804    {
1805      return (entry.matchesBaseAndScope(url.getBaseDN(), url.getScope()) &&
1806           url.getFilter().matchesEntry(entry));
1807    }
1808    catch (final Exception e)
1809    {
1810      Debug.debugException(e);
1811      return false;
1812    }
1813  }
1814
1815
1816
1817  /**
1818   * Writes a line-wrapped, commented version of the provided message to
1819   * standard output.
1820   *
1821   * @param  message  The message to be written.
1822   */
1823  private void commentToOut(@NotNull final String message)
1824  {
1825    getOut().flush();
1826    for (final String line : StaticUtils.wrapLine(message, (WRAP_COLUMN - 2)))
1827    {
1828      out("# " + line);
1829    }
1830    getOut().flush();
1831  }
1832
1833
1834
1835  /**
1836   * Writes a line-wrapped, commented version of the provided message to
1837   * standard error.
1838   *
1839   * @param  message  The message to be written.
1840   */
1841  private void commentToErr(@NotNull final String message)
1842  {
1843    getErr().flush();
1844    for (final String line : StaticUtils.wrapLine(message, (WRAP_COLUMN - 2)))
1845    {
1846      err("# " + line);
1847    }
1848    getErr().flush();
1849  }
1850
1851
1852
1853  /**
1854   * Writes the provided message and sets it as the completion message.
1855   *
1856   * @param  isError  Indicates whether the message should be written to
1857   *                  standard error rather than standard output.
1858   * @param  message  The message to be written.
1859   */
1860  private void logCompletionMessage(final boolean isError,
1861                                    @NotNull final String message)
1862  {
1863    completionMessage.compareAndSet(null, message);
1864
1865    if (! outputFile.isPresent())
1866    {
1867      resultWriter.writeComment(message);
1868    }
1869  }
1870
1871
1872
1873  /**
1874   * {@inheritDoc}
1875   */
1876  @Override()
1877  @NotNull()
1878  public LinkedHashMap<String[],String> getExampleUsages()
1879  {
1880    final LinkedHashMap<String[],String> examples = new LinkedHashMap<>();
1881
1882    examples.put(
1883         new String[]
1884         {
1885           "--ldifFile", "data.ldif",
1886           "(uid=jdoe)"
1887         },
1888         INFO_LDIFSEARCH_EXAMPLE_1.get());
1889
1890    examples.put(
1891         new String[]
1892         {
1893           "--ldifFile", "data.ldif",
1894           "--outputFile", "people.ldif",
1895           "--baseDN", "dc=example,dc=com",
1896           "--scope", "sub",
1897           "(objectClass=person)",
1898           "givenName",
1899           "sn",
1900           "cn",
1901         },
1902         INFO_LDIFSEARCH_EXAMPLE_2.get());
1903
1904    return examples;
1905  }
1906}