001/*
002 * Copyright 2013-2022 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2013-2022 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2013-2022 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.examples;
037
038
039
040import java.io.OutputStream;
041import java.util.Collections;
042import java.util.LinkedHashMap;
043import java.util.List;
044import java.util.Map;
045import java.util.TreeMap;
046import java.util.concurrent.atomic.AtomicLong;
047
048import com.unboundid.asn1.ASN1OctetString;
049import com.unboundid.ldap.sdk.Attribute;
050import com.unboundid.ldap.sdk.DN;
051import com.unboundid.ldap.sdk.Filter;
052import com.unboundid.ldap.sdk.LDAPConnectionOptions;
053import com.unboundid.ldap.sdk.LDAPConnectionPool;
054import com.unboundid.ldap.sdk.LDAPException;
055import com.unboundid.ldap.sdk.LDAPSearchException;
056import com.unboundid.ldap.sdk.ResultCode;
057import com.unboundid.ldap.sdk.SearchRequest;
058import com.unboundid.ldap.sdk.SearchResult;
059import com.unboundid.ldap.sdk.SearchResultEntry;
060import com.unboundid.ldap.sdk.SearchResultReference;
061import com.unboundid.ldap.sdk.SearchResultListener;
062import com.unboundid.ldap.sdk.SearchScope;
063import com.unboundid.ldap.sdk.Version;
064import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
065import com.unboundid.util.Debug;
066import com.unboundid.util.LDAPCommandLineTool;
067import com.unboundid.util.NotNull;
068import com.unboundid.util.Nullable;
069import com.unboundid.util.StaticUtils;
070import com.unboundid.util.ThreadSafety;
071import com.unboundid.util.ThreadSafetyLevel;
072import com.unboundid.util.args.ArgumentException;
073import com.unboundid.util.args.ArgumentParser;
074import com.unboundid.util.args.DNArgument;
075import com.unboundid.util.args.IntegerArgument;
076import com.unboundid.util.args.StringArgument;
077
078
079
080/**
081 * This class provides a tool that may be used to identify references to entries
082 * that do not exist.  This tool can be useful for verifying existing data in
083 * directory servers that provide support for referential integrity.
084 * <BR><BR>
085 * All of the necessary information is provided using command line arguments.
086 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
087 * class, as well as the following additional arguments:
088 * <UL>
089 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
090 *       for the searches.  At least one base DN must be provided.</LI>
091 *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
092 *       that is expected to contain references to other entries.  This
093 *       attribute should be indexed for equality searches, and its values
094 *       should be DNs.  At least one attribute must be provided.</LI>
095 *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
096 *       to find entries with references to other entries should use the simple
097 *       paged results control to iterate across entries in fixed-size pages
098 *       rather than trying to use a single search to identify all entries that
099 *       reference other entries.</LI>
100 * </UL>
101 */
102@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
103public final class IdentifyReferencesToMissingEntries
104       extends LDAPCommandLineTool
105       implements SearchResultListener
106{
107  /**
108   * The serial version UID for this serializable class.
109   */
110  private static final long serialVersionUID = 1981894839719501258L;
111
112
113
114  // The number of entries examined so far.
115  @NotNull private final AtomicLong entriesExamined;
116
117  // The argument used to specify the base DNs to use for searches.
118  @Nullable private DNArgument baseDNArgument;
119
120  // The argument used to specify the search page size.
121  @Nullable private IntegerArgument pageSizeArgument;
122
123  // The connection to use for retrieving referenced entries.
124  @Nullable private LDAPConnectionPool getReferencedEntriesPool;
125
126  // A map with counts of missing references by attribute type.
127  @NotNull private final Map<String,AtomicLong> missingReferenceCounts;
128
129  // The names of the attributes for which to find missing references.
130  @Nullable private String[] attributes;
131
132  // The argument used to specify the attributes for which to find missing
133  // references.
134  @Nullable private StringArgument attributeArgument;
135
136
137
138  /**
139   * Parse the provided command line arguments and perform the appropriate
140   * processing.
141   *
142   * @param  args  The command line arguments provided to this program.
143   */
144  public static void main(@NotNull final String... args)
145  {
146    final ResultCode resultCode = main(args, System.out, System.err);
147    if (resultCode != ResultCode.SUCCESS)
148    {
149      System.exit(resultCode.intValue());
150    }
151  }
152
153
154
155  /**
156   * Parse the provided command line arguments and perform the appropriate
157   * processing.
158   *
159   * @param  args       The command line arguments provided to this program.
160   * @param  outStream  The output stream to which standard out should be
161   *                    written.  It may be {@code null} if output should be
162   *                    suppressed.
163   * @param  errStream  The output stream to which standard error should be
164   *                    written.  It may be {@code null} if error messages
165   *                    should be suppressed.
166   *
167   * @return A result code indicating whether the processing was successful.
168   */
169  @NotNull()
170  public static ResultCode main(@NotNull final String[] args,
171                                @Nullable final OutputStream outStream,
172                                @Nullable final OutputStream errStream)
173  {
174    final IdentifyReferencesToMissingEntries tool =
175         new IdentifyReferencesToMissingEntries(outStream, errStream);
176    return tool.runTool(args);
177  }
178
179
180
181  /**
182   * Creates a new instance of this tool.
183   *
184   * @param  outStream  The output stream to which standard out should be
185   *                    written.  It may be {@code null} if output should be
186   *                    suppressed.
187   * @param  errStream  The output stream to which standard error should be
188   *                    written.  It may be {@code null} if error messages
189   *                    should be suppressed.
190   */
191  public IdentifyReferencesToMissingEntries(
192              @Nullable final OutputStream outStream,
193              @Nullable final OutputStream errStream)
194  {
195    super(outStream, errStream);
196
197    baseDNArgument = null;
198    pageSizeArgument = null;
199    attributeArgument = null;
200    getReferencedEntriesPool = null;
201
202    entriesExamined = new AtomicLong(0L);
203    missingReferenceCounts = new TreeMap<>();
204  }
205
206
207
208  /**
209   * Retrieves the name of this tool.  It should be the name of the command used
210   * to invoke this tool.
211   *
212   * @return  The name for this tool.
213   */
214  @Override()
215  @NotNull()
216  public String getToolName()
217  {
218    return "identify-references-to-missing-entries";
219  }
220
221
222
223  /**
224   * Retrieves a human-readable description for this tool.
225   *
226   * @return  A human-readable description for this tool.
227   */
228  @Override()
229  @NotNull()
230  public String getToolDescription()
231  {
232    return "This tool may be used to identify entries containing one or more " +
233         "attributes which reference entries that do not exist.  This may " +
234         "require the ability to perform unindexed searches and/or the " +
235         "ability to use the simple paged results control.";
236  }
237
238
239
240  /**
241   * Retrieves a version string for this tool, if available.
242   *
243   * @return  A version string for this tool, or {@code null} if none is
244   *          available.
245   */
246  @Override()
247  @NotNull()
248  public String getToolVersion()
249  {
250    return Version.NUMERIC_VERSION_STRING;
251  }
252
253
254
255  /**
256   * Indicates whether this tool should provide support for an interactive mode,
257   * in which the tool offers a mode in which the arguments can be provided in
258   * a text-driven menu rather than requiring them to be given on the command
259   * line.  If interactive mode is supported, it may be invoked using the
260   * "--interactive" argument.  Alternately, if interactive mode is supported
261   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
262   * interactive mode may be invoked by simply launching the tool without any
263   * arguments.
264   *
265   * @return  {@code true} if this tool supports interactive mode, or
266   *          {@code false} if not.
267   */
268  @Override()
269  public boolean supportsInteractiveMode()
270  {
271    return true;
272  }
273
274
275
276  /**
277   * Indicates whether this tool defaults to launching in interactive mode if
278   * the tool is invoked without any command-line arguments.  This will only be
279   * used if {@link #supportsInteractiveMode()} returns {@code true}.
280   *
281   * @return  {@code true} if this tool defaults to using interactive mode if
282   *          launched without any command-line arguments, or {@code false} if
283   *          not.
284   */
285  @Override()
286  public boolean defaultsToInteractiveMode()
287  {
288    return true;
289  }
290
291
292
293  /**
294   * Indicates whether this tool should provide arguments for redirecting output
295   * to a file.  If this method returns {@code true}, then the tool will offer
296   * an "--outputFile" argument that will specify the path to a file to which
297   * all standard output and standard error content will be written, and it will
298   * also offer a "--teeToStandardOut" argument that can only be used if the
299   * "--outputFile" argument is present and will cause all output to be written
300   * to both the specified output file and to standard output.
301   *
302   * @return  {@code true} if this tool should provide arguments for redirecting
303   *          output to a file, or {@code false} if not.
304   */
305  @Override()
306  protected boolean supportsOutputFile()
307  {
308    return true;
309  }
310
311
312
313  /**
314   * Indicates whether this tool should default to interactively prompting for
315   * the bind password if a password is required but no argument was provided
316   * to indicate how to get the password.
317   *
318   * @return  {@code true} if this tool should default to interactively
319   *          prompting for the bind password, or {@code false} if not.
320   */
321  @Override()
322  protected boolean defaultToPromptForBindPassword()
323  {
324    return true;
325  }
326
327
328
329  /**
330   * Indicates whether this tool supports the use of a properties file for
331   * specifying default values for arguments that aren't specified on the
332   * command line.
333   *
334   * @return  {@code true} if this tool supports the use of a properties file
335   *          for specifying default values for arguments that aren't specified
336   *          on the command line, or {@code false} if not.
337   */
338  @Override()
339  public boolean supportsPropertiesFile()
340  {
341    return true;
342  }
343
344
345
346  /**
347   * Indicates whether the LDAP-specific arguments should include alternate
348   * versions of all long identifiers that consist of multiple words so that
349   * they are available in both camelCase and dash-separated versions.
350   *
351   * @return  {@code true} if this tool should provide multiple versions of
352   *          long identifiers for LDAP-specific arguments, or {@code false} if
353   *          not.
354   */
355  @Override()
356  protected boolean includeAlternateLongIdentifiers()
357  {
358    return true;
359  }
360
361
362
363  /**
364   * Indicates whether this tool should provide a command-line argument that
365   * allows for low-level SSL debugging.  If this returns {@code true}, then an
366   * "--enableSSLDebugging}" argument will be added that sets the
367   * "javax.net.debug" system property to "all" before attempting any
368   * communication.
369   *
370   * @return  {@code true} if this tool should offer an "--enableSSLDebugging"
371   *          argument, or {@code false} if not.
372   */
373  @Override()
374  protected boolean supportsSSLDebugging()
375  {
376    return true;
377  }
378
379
380
381  /**
382   * Adds the arguments needed by this command-line tool to the provided
383   * argument parser which are not related to connecting or authenticating to
384   * the directory server.
385   *
386   * @param  parser  The argument parser to which the arguments should be added.
387   *
388   * @throws  ArgumentException  If a problem occurs while adding the arguments.
389   */
390  @Override()
391  public void addNonLDAPArguments(@NotNull final ArgumentParser parser)
392         throws ArgumentException
393  {
394    String description = "The search base DN(s) to use to find entries with " +
395         "references to other entries.  At least one base DN must be " +
396         "specified.";
397    baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
398         description);
399    baseDNArgument.addLongIdentifier("base-dn", true);
400    parser.addArgument(baseDNArgument);
401
402    description = "The attribute(s) for which to find missing references.  " +
403         "At least one attribute must be specified, and each attribute " +
404         "must be indexed for equality searches and have values which are DNs.";
405    attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
406         description);
407    parser.addArgument(attributeArgument);
408
409    description = "The maximum number of entries to retrieve at a time when " +
410         "attempting to find entries with references to other entries.  This " +
411         "requires that the authenticated user have permission to use the " +
412         "simple paged results control, but it can avoid problems with the " +
413         "server sending entries too quickly for the client to handle.  By " +
414         "default, the simple paged results control will not be used.";
415    pageSizeArgument =
416         new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
417              description, 1, Integer.MAX_VALUE);
418    pageSizeArgument.addLongIdentifier("simple-page-size", true);
419    parser.addArgument(pageSizeArgument);
420  }
421
422
423
424  /**
425   * Retrieves the connection options that should be used for connections that
426   * are created with this command line tool.  Subclasses may override this
427   * method to use a custom set of connection options.
428   *
429   * @return  The connection options that should be used for connections that
430   *          are created with this command line tool.
431   */
432  @Override()
433  @NotNull()
434  public LDAPConnectionOptions getConnectionOptions()
435  {
436    final LDAPConnectionOptions options = new LDAPConnectionOptions();
437
438    options.setUseSynchronousMode(true);
439    options.setResponseTimeoutMillis(0L);
440
441    return options;
442  }
443
444
445
446  /**
447   * Performs the core set of processing for this tool.
448   *
449   * @return  A result code that indicates whether the processing completed
450   *          successfully.
451   */
452  @Override()
453  @NotNull()
454  public ResultCode doToolProcessing()
455  {
456    // Establish a connection to the target directory server to use for
457    // finding references to entries.
458    final LDAPConnectionPool findReferencesPool;
459    try
460    {
461      findReferencesPool = getConnectionPool(1, 1);
462      findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true);
463    }
464    catch (final LDAPException le)
465    {
466      Debug.debugException(le);
467      err("Unable to establish a connection to the directory server:  ",
468           StaticUtils.getExceptionMessage(le));
469      return le.getResultCode();
470    }
471
472    try
473    {
474      // Establish a second connection to use for retrieving referenced entries.
475      try
476      {
477        getReferencedEntriesPool = getConnectionPool(1,1);
478        getReferencedEntriesPool.
479             setRetryFailedOperationsDueToInvalidConnections(true);
480      }
481      catch (final LDAPException le)
482      {
483        Debug.debugException(le);
484        err("Unable to establish a connection to the directory server:  ",
485             StaticUtils.getExceptionMessage(le));
486        return le.getResultCode();
487      }
488
489
490      // Get the set of attributes for which to find missing references.
491      final List<String> attrList = attributeArgument.getValues();
492      attributes = new String[attrList.size()];
493      attrList.toArray(attributes);
494
495
496      // Construct a search filter that will be used to find all entries with
497      // references to other entries.
498      final Filter filter;
499      if (attributes.length == 1)
500      {
501        filter = Filter.createPresenceFilter(attributes[0]);
502        missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
503      }
504      else
505      {
506        final Filter[] orComps = new Filter[attributes.length];
507        for (int i=0; i < attributes.length; i++)
508        {
509          orComps[i] = Filter.createPresenceFilter(attributes[i]);
510          missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
511        }
512        filter = Filter.createORFilter(orComps);
513      }
514
515
516      // Iterate across all of the search base DNs and perform searches to find
517      // missing references.
518      for (final DN baseDN : baseDNArgument.getValues())
519      {
520        ASN1OctetString cookie = null;
521        do
522        {
523          final SearchRequest searchRequest = new SearchRequest(this,
524               baseDN.toString(), SearchScope.SUB, filter, attributes);
525          if (pageSizeArgument.isPresent())
526          {
527            searchRequest.addControl(new SimplePagedResultsControl(
528                 pageSizeArgument.getValue(), cookie, false));
529          }
530
531          SearchResult searchResult;
532          try
533          {
534            searchResult = findReferencesPool.search(searchRequest);
535          }
536          catch (final LDAPSearchException lse)
537          {
538            Debug.debugException(lse);
539            try
540            {
541              searchResult = findReferencesPool.search(searchRequest);
542            }
543            catch (final LDAPSearchException lse2)
544            {
545              Debug.debugException(lse2);
546              searchResult = lse2.getSearchResult();
547            }
548          }
549
550          if (searchResult.getResultCode() != ResultCode.SUCCESS)
551          {
552            err("An error occurred while attempting to search for missing " +
553                 "references to entries below " + baseDN + ":  " +
554                 searchResult.getDiagnosticMessage());
555            return searchResult.getResultCode();
556          }
557
558          final SimplePagedResultsControl pagedResultsResponse;
559          try
560          {
561            pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
562          }
563          catch (final LDAPException le)
564          {
565            Debug.debugException(le);
566            err("An error occurred while attempting to decode a simple " +
567                 "paged results response control in the response to a " +
568                 "search for entries below " + baseDN + ":  " +
569                 StaticUtils.getExceptionMessage(le));
570            return le.getResultCode();
571          }
572
573          if (pagedResultsResponse != null)
574          {
575            if (pagedResultsResponse.moreResultsToReturn())
576            {
577              cookie = pagedResultsResponse.getCookie();
578            }
579            else
580            {
581              cookie = null;
582            }
583          }
584        }
585        while (cookie != null);
586      }
587
588
589      // See if there were any missing references found.
590      boolean missingReferenceFound = false;
591      for (final Map.Entry<String,AtomicLong> e :
592           missingReferenceCounts.entrySet())
593      {
594        final long numMissing = e.getValue().get();
595        if (numMissing > 0L)
596        {
597          if (! missingReferenceFound)
598          {
599            err();
600            missingReferenceFound = true;
601          }
602
603          err("Found " + numMissing + ' ' + e.getKey() +
604               " references to entries that do not exist.");
605        }
606      }
607
608      if (missingReferenceFound)
609      {
610        return ResultCode.CONSTRAINT_VIOLATION;
611      }
612      else
613      {
614        out("No references were found to entries that do not exist.");
615        return ResultCode.SUCCESS;
616      }
617    }
618    finally
619    {
620      findReferencesPool.close();
621
622      if (getReferencedEntriesPool != null)
623      {
624        getReferencedEntriesPool.close();
625      }
626    }
627  }
628
629
630
631  /**
632   * Retrieves a map that correlates the number of missing references found by
633   * attribute type.
634   *
635   * @return  A map that correlates the number of missing references found by
636   *          attribute type.
637   */
638  @NotNull()
639  public Map<String,AtomicLong> getMissingReferenceCounts()
640  {
641    return Collections.unmodifiableMap(missingReferenceCounts);
642  }
643
644
645
646  /**
647   * Retrieves a set of information that may be used to generate example usage
648   * information.  Each element in the returned map should consist of a map
649   * between an example set of arguments and a string that describes the
650   * behavior of the tool when invoked with that set of arguments.
651   *
652   * @return  A set of information that may be used to generate example usage
653   *          information.  It may be {@code null} or empty if no example usage
654   *          information is available.
655   */
656  @Override()
657  @NotNull()
658  public LinkedHashMap<String[],String> getExampleUsages()
659  {
660    final LinkedHashMap<String[],String> exampleMap =
661         new LinkedHashMap<>(StaticUtils.computeMapCapacity(1));
662
663    final String[] args =
664    {
665      "--hostname", "server.example.com",
666      "--port", "389",
667      "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
668      "--bindPassword", "password",
669      "--baseDN", "dc=example,dc=com",
670      "--attribute", "member",
671      "--attribute", "uniqueMember",
672      "--simplePageSize", "100"
673    };
674    exampleMap.put(args,
675         "Identify all entries below dc=example,dc=com in which either the " +
676              "member or uniqueMember attribute references an entry that " +
677              "does not exist.");
678
679    return exampleMap;
680  }
681
682
683
684  /**
685   * Indicates that the provided search result entry has been returned by the
686   * server and may be processed by this search result listener.
687   *
688   * @param  searchEntry  The search result entry that has been returned by the
689   *                      server.
690   */
691  @Override()
692  public void searchEntryReturned(@NotNull final SearchResultEntry searchEntry)
693  {
694    try
695    {
696      // Find attributes which references to entries that do not exist.
697      for (final String attr : attributes)
698      {
699        final List<Attribute> attrList =
700             searchEntry.getAttributesWithOptions(attr, null);
701        for (final Attribute a : attrList)
702        {
703          for (final String value : a.getValues())
704          {
705            try
706            {
707              final SearchResultEntry e =
708                   getReferencedEntriesPool.getEntry(value, "1.1");
709              if (e == null)
710              {
711                err("Entry '", searchEntry.getDN(), "' includes attribute ",
712                     a.getName(), " that references entry '", value,
713                     "' which does not exist.");
714                missingReferenceCounts.get(attr).incrementAndGet();
715              }
716            }
717            catch (final LDAPException le)
718            {
719              Debug.debugException(le);
720              err("An error occurred while attempting to determine whether " +
721                   "entry '" + value + "' referenced in attribute " +
722                   a.getName() + " of entry '" + searchEntry.getDN() +
723                   "' exists:  " + StaticUtils.getExceptionMessage(le));
724              missingReferenceCounts.get(attr).incrementAndGet();
725            }
726          }
727        }
728      }
729    }
730    finally
731    {
732      final long count = entriesExamined.incrementAndGet();
733      if ((count % 1000L) == 0L)
734      {
735        out(count, " entries examined");
736      }
737    }
738  }
739
740
741
742  /**
743   * Indicates that the provided search result reference has been returned by
744   * the server and may be processed by this search result listener.
745   *
746   * @param  searchReference  The search result reference that has been returned
747   *                          by the server.
748   */
749  @Override()
750  public void searchReferenceReturned(
751                   @NotNull final SearchResultReference searchReference)
752  {
753    // No implementation is required.  This tool will not follow referrals.
754  }
755}