001/* 002 * Copyright 2013-2022 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2013-2022 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2013-2022 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.examples; 037 038 039 040import java.io.OutputStream; 041import java.util.Collections; 042import java.util.LinkedHashMap; 043import java.util.List; 044import java.util.Map; 045import java.util.TreeMap; 046import java.util.concurrent.atomic.AtomicLong; 047 048import com.unboundid.asn1.ASN1OctetString; 049import com.unboundid.ldap.sdk.Attribute; 050import com.unboundid.ldap.sdk.DN; 051import com.unboundid.ldap.sdk.Filter; 052import com.unboundid.ldap.sdk.LDAPConnectionOptions; 053import com.unboundid.ldap.sdk.LDAPConnectionPool; 054import com.unboundid.ldap.sdk.LDAPException; 055import com.unboundid.ldap.sdk.LDAPSearchException; 056import com.unboundid.ldap.sdk.ResultCode; 057import com.unboundid.ldap.sdk.SearchRequest; 058import com.unboundid.ldap.sdk.SearchResult; 059import com.unboundid.ldap.sdk.SearchResultEntry; 060import com.unboundid.ldap.sdk.SearchResultReference; 061import com.unboundid.ldap.sdk.SearchResultListener; 062import com.unboundid.ldap.sdk.SearchScope; 063import com.unboundid.ldap.sdk.Version; 064import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl; 065import com.unboundid.util.Debug; 066import com.unboundid.util.LDAPCommandLineTool; 067import com.unboundid.util.NotNull; 068import com.unboundid.util.Nullable; 069import com.unboundid.util.StaticUtils; 070import com.unboundid.util.ThreadSafety; 071import com.unboundid.util.ThreadSafetyLevel; 072import com.unboundid.util.args.ArgumentException; 073import com.unboundid.util.args.ArgumentParser; 074import com.unboundid.util.args.DNArgument; 075import com.unboundid.util.args.IntegerArgument; 076import com.unboundid.util.args.StringArgument; 077 078 079 080/** 081 * This class provides a tool that may be used to identify references to entries 082 * that do not exist. This tool can be useful for verifying existing data in 083 * directory servers that provide support for referential integrity. 084 * <BR><BR> 085 * All of the necessary information is provided using command line arguments. 086 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 087 * class, as well as the following additional arguments: 088 * <UL> 089 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use 090 * for the searches. At least one base DN must be provided.</LI> 091 * <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute 092 * that is expected to contain references to other entries. This 093 * attribute should be indexed for equality searches, and its values 094 * should be DNs. At least one attribute must be provided.</LI> 095 * <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search 096 * to find entries with references to other entries should use the simple 097 * paged results control to iterate across entries in fixed-size pages 098 * rather than trying to use a single search to identify all entries that 099 * reference other entries.</LI> 100 * </UL> 101 */ 102@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 103public final class IdentifyReferencesToMissingEntries 104 extends LDAPCommandLineTool 105 implements SearchResultListener 106{ 107 /** 108 * The serial version UID for this serializable class. 109 */ 110 private static final long serialVersionUID = 1981894839719501258L; 111 112 113 114 // The number of entries examined so far. 115 @NotNull private final AtomicLong entriesExamined; 116 117 // The argument used to specify the base DNs to use for searches. 118 @Nullable private DNArgument baseDNArgument; 119 120 // The argument used to specify the search page size. 121 @Nullable private IntegerArgument pageSizeArgument; 122 123 // The connection to use for retrieving referenced entries. 124 @Nullable private LDAPConnectionPool getReferencedEntriesPool; 125 126 // A map with counts of missing references by attribute type. 127 @NotNull private final Map<String,AtomicLong> missingReferenceCounts; 128 129 // The names of the attributes for which to find missing references. 130 @Nullable private String[] attributes; 131 132 // The argument used to specify the attributes for which to find missing 133 // references. 134 @Nullable private StringArgument attributeArgument; 135 136 137 138 /** 139 * Parse the provided command line arguments and perform the appropriate 140 * processing. 141 * 142 * @param args The command line arguments provided to this program. 143 */ 144 public static void main(@NotNull final String... args) 145 { 146 final ResultCode resultCode = main(args, System.out, System.err); 147 if (resultCode != ResultCode.SUCCESS) 148 { 149 System.exit(resultCode.intValue()); 150 } 151 } 152 153 154 155 /** 156 * Parse the provided command line arguments and perform the appropriate 157 * processing. 158 * 159 * @param args The command line arguments provided to this program. 160 * @param outStream The output stream to which standard out should be 161 * written. It may be {@code null} if output should be 162 * suppressed. 163 * @param errStream The output stream to which standard error should be 164 * written. It may be {@code null} if error messages 165 * should be suppressed. 166 * 167 * @return A result code indicating whether the processing was successful. 168 */ 169 @NotNull() 170 public static ResultCode main(@NotNull final String[] args, 171 @Nullable final OutputStream outStream, 172 @Nullable final OutputStream errStream) 173 { 174 final IdentifyReferencesToMissingEntries tool = 175 new IdentifyReferencesToMissingEntries(outStream, errStream); 176 return tool.runTool(args); 177 } 178 179 180 181 /** 182 * Creates a new instance of this tool. 183 * 184 * @param outStream The output stream to which standard out should be 185 * written. It may be {@code null} if output should be 186 * suppressed. 187 * @param errStream The output stream to which standard error should be 188 * written. It may be {@code null} if error messages 189 * should be suppressed. 190 */ 191 public IdentifyReferencesToMissingEntries( 192 @Nullable final OutputStream outStream, 193 @Nullable final OutputStream errStream) 194 { 195 super(outStream, errStream); 196 197 baseDNArgument = null; 198 pageSizeArgument = null; 199 attributeArgument = null; 200 getReferencedEntriesPool = null; 201 202 entriesExamined = new AtomicLong(0L); 203 missingReferenceCounts = new TreeMap<>(); 204 } 205 206 207 208 /** 209 * Retrieves the name of this tool. It should be the name of the command used 210 * to invoke this tool. 211 * 212 * @return The name for this tool. 213 */ 214 @Override() 215 @NotNull() 216 public String getToolName() 217 { 218 return "identify-references-to-missing-entries"; 219 } 220 221 222 223 /** 224 * Retrieves a human-readable description for this tool. 225 * 226 * @return A human-readable description for this tool. 227 */ 228 @Override() 229 @NotNull() 230 public String getToolDescription() 231 { 232 return "This tool may be used to identify entries containing one or more " + 233 "attributes which reference entries that do not exist. This may " + 234 "require the ability to perform unindexed searches and/or the " + 235 "ability to use the simple paged results control."; 236 } 237 238 239 240 /** 241 * Retrieves a version string for this tool, if available. 242 * 243 * @return A version string for this tool, or {@code null} if none is 244 * available. 245 */ 246 @Override() 247 @NotNull() 248 public String getToolVersion() 249 { 250 return Version.NUMERIC_VERSION_STRING; 251 } 252 253 254 255 /** 256 * Indicates whether this tool should provide support for an interactive mode, 257 * in which the tool offers a mode in which the arguments can be provided in 258 * a text-driven menu rather than requiring them to be given on the command 259 * line. If interactive mode is supported, it may be invoked using the 260 * "--interactive" argument. Alternately, if interactive mode is supported 261 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 262 * interactive mode may be invoked by simply launching the tool without any 263 * arguments. 264 * 265 * @return {@code true} if this tool supports interactive mode, or 266 * {@code false} if not. 267 */ 268 @Override() 269 public boolean supportsInteractiveMode() 270 { 271 return true; 272 } 273 274 275 276 /** 277 * Indicates whether this tool defaults to launching in interactive mode if 278 * the tool is invoked without any command-line arguments. This will only be 279 * used if {@link #supportsInteractiveMode()} returns {@code true}. 280 * 281 * @return {@code true} if this tool defaults to using interactive mode if 282 * launched without any command-line arguments, or {@code false} if 283 * not. 284 */ 285 @Override() 286 public boolean defaultsToInteractiveMode() 287 { 288 return true; 289 } 290 291 292 293 /** 294 * Indicates whether this tool should provide arguments for redirecting output 295 * to a file. If this method returns {@code true}, then the tool will offer 296 * an "--outputFile" argument that will specify the path to a file to which 297 * all standard output and standard error content will be written, and it will 298 * also offer a "--teeToStandardOut" argument that can only be used if the 299 * "--outputFile" argument is present and will cause all output to be written 300 * to both the specified output file and to standard output. 301 * 302 * @return {@code true} if this tool should provide arguments for redirecting 303 * output to a file, or {@code false} if not. 304 */ 305 @Override() 306 protected boolean supportsOutputFile() 307 { 308 return true; 309 } 310 311 312 313 /** 314 * Indicates whether this tool should default to interactively prompting for 315 * the bind password if a password is required but no argument was provided 316 * to indicate how to get the password. 317 * 318 * @return {@code true} if this tool should default to interactively 319 * prompting for the bind password, or {@code false} if not. 320 */ 321 @Override() 322 protected boolean defaultToPromptForBindPassword() 323 { 324 return true; 325 } 326 327 328 329 /** 330 * Indicates whether this tool supports the use of a properties file for 331 * specifying default values for arguments that aren't specified on the 332 * command line. 333 * 334 * @return {@code true} if this tool supports the use of a properties file 335 * for specifying default values for arguments that aren't specified 336 * on the command line, or {@code false} if not. 337 */ 338 @Override() 339 public boolean supportsPropertiesFile() 340 { 341 return true; 342 } 343 344 345 346 /** 347 * Indicates whether the LDAP-specific arguments should include alternate 348 * versions of all long identifiers that consist of multiple words so that 349 * they are available in both camelCase and dash-separated versions. 350 * 351 * @return {@code true} if this tool should provide multiple versions of 352 * long identifiers for LDAP-specific arguments, or {@code false} if 353 * not. 354 */ 355 @Override() 356 protected boolean includeAlternateLongIdentifiers() 357 { 358 return true; 359 } 360 361 362 363 /** 364 * Indicates whether this tool should provide a command-line argument that 365 * allows for low-level SSL debugging. If this returns {@code true}, then an 366 * "--enableSSLDebugging}" argument will be added that sets the 367 * "javax.net.debug" system property to "all" before attempting any 368 * communication. 369 * 370 * @return {@code true} if this tool should offer an "--enableSSLDebugging" 371 * argument, or {@code false} if not. 372 */ 373 @Override() 374 protected boolean supportsSSLDebugging() 375 { 376 return true; 377 } 378 379 380 381 /** 382 * Adds the arguments needed by this command-line tool to the provided 383 * argument parser which are not related to connecting or authenticating to 384 * the directory server. 385 * 386 * @param parser The argument parser to which the arguments should be added. 387 * 388 * @throws ArgumentException If a problem occurs while adding the arguments. 389 */ 390 @Override() 391 public void addNonLDAPArguments(@NotNull final ArgumentParser parser) 392 throws ArgumentException 393 { 394 String description = "The search base DN(s) to use to find entries with " + 395 "references to other entries. At least one base DN must be " + 396 "specified."; 397 baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}", 398 description); 399 baseDNArgument.addLongIdentifier("base-dn", true); 400 parser.addArgument(baseDNArgument); 401 402 description = "The attribute(s) for which to find missing references. " + 403 "At least one attribute must be specified, and each attribute " + 404 "must be indexed for equality searches and have values which are DNs."; 405 attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}", 406 description); 407 parser.addArgument(attributeArgument); 408 409 description = "The maximum number of entries to retrieve at a time when " + 410 "attempting to find entries with references to other entries. This " + 411 "requires that the authenticated user have permission to use the " + 412 "simple paged results control, but it can avoid problems with the " + 413 "server sending entries too quickly for the client to handle. By " + 414 "default, the simple paged results control will not be used."; 415 pageSizeArgument = 416 new IntegerArgument('z', "simplePageSize", false, 1, "{num}", 417 description, 1, Integer.MAX_VALUE); 418 pageSizeArgument.addLongIdentifier("simple-page-size", true); 419 parser.addArgument(pageSizeArgument); 420 } 421 422 423 424 /** 425 * Retrieves the connection options that should be used for connections that 426 * are created with this command line tool. Subclasses may override this 427 * method to use a custom set of connection options. 428 * 429 * @return The connection options that should be used for connections that 430 * are created with this command line tool. 431 */ 432 @Override() 433 @NotNull() 434 public LDAPConnectionOptions getConnectionOptions() 435 { 436 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 437 438 options.setUseSynchronousMode(true); 439 options.setResponseTimeoutMillis(0L); 440 441 return options; 442 } 443 444 445 446 /** 447 * Performs the core set of processing for this tool. 448 * 449 * @return A result code that indicates whether the processing completed 450 * successfully. 451 */ 452 @Override() 453 @NotNull() 454 public ResultCode doToolProcessing() 455 { 456 // Establish a connection to the target directory server to use for 457 // finding references to entries. 458 final LDAPConnectionPool findReferencesPool; 459 try 460 { 461 findReferencesPool = getConnectionPool(1, 1); 462 findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true); 463 } 464 catch (final LDAPException le) 465 { 466 Debug.debugException(le); 467 err("Unable to establish a connection to the directory server: ", 468 StaticUtils.getExceptionMessage(le)); 469 return le.getResultCode(); 470 } 471 472 try 473 { 474 // Establish a second connection to use for retrieving referenced entries. 475 try 476 { 477 getReferencedEntriesPool = getConnectionPool(1,1); 478 getReferencedEntriesPool. 479 setRetryFailedOperationsDueToInvalidConnections(true); 480 } 481 catch (final LDAPException le) 482 { 483 Debug.debugException(le); 484 err("Unable to establish a connection to the directory server: ", 485 StaticUtils.getExceptionMessage(le)); 486 return le.getResultCode(); 487 } 488 489 490 // Get the set of attributes for which to find missing references. 491 final List<String> attrList = attributeArgument.getValues(); 492 attributes = new String[attrList.size()]; 493 attrList.toArray(attributes); 494 495 496 // Construct a search filter that will be used to find all entries with 497 // references to other entries. 498 final Filter filter; 499 if (attributes.length == 1) 500 { 501 filter = Filter.createPresenceFilter(attributes[0]); 502 missingReferenceCounts.put(attributes[0], new AtomicLong(0L)); 503 } 504 else 505 { 506 final Filter[] orComps = new Filter[attributes.length]; 507 for (int i=0; i < attributes.length; i++) 508 { 509 orComps[i] = Filter.createPresenceFilter(attributes[i]); 510 missingReferenceCounts.put(attributes[i], new AtomicLong(0L)); 511 } 512 filter = Filter.createORFilter(orComps); 513 } 514 515 516 // Iterate across all of the search base DNs and perform searches to find 517 // missing references. 518 for (final DN baseDN : baseDNArgument.getValues()) 519 { 520 ASN1OctetString cookie = null; 521 do 522 { 523 final SearchRequest searchRequest = new SearchRequest(this, 524 baseDN.toString(), SearchScope.SUB, filter, attributes); 525 if (pageSizeArgument.isPresent()) 526 { 527 searchRequest.addControl(new SimplePagedResultsControl( 528 pageSizeArgument.getValue(), cookie, false)); 529 } 530 531 SearchResult searchResult; 532 try 533 { 534 searchResult = findReferencesPool.search(searchRequest); 535 } 536 catch (final LDAPSearchException lse) 537 { 538 Debug.debugException(lse); 539 try 540 { 541 searchResult = findReferencesPool.search(searchRequest); 542 } 543 catch (final LDAPSearchException lse2) 544 { 545 Debug.debugException(lse2); 546 searchResult = lse2.getSearchResult(); 547 } 548 } 549 550 if (searchResult.getResultCode() != ResultCode.SUCCESS) 551 { 552 err("An error occurred while attempting to search for missing " + 553 "references to entries below " + baseDN + ": " + 554 searchResult.getDiagnosticMessage()); 555 return searchResult.getResultCode(); 556 } 557 558 final SimplePagedResultsControl pagedResultsResponse; 559 try 560 { 561 pagedResultsResponse = SimplePagedResultsControl.get(searchResult); 562 } 563 catch (final LDAPException le) 564 { 565 Debug.debugException(le); 566 err("An error occurred while attempting to decode a simple " + 567 "paged results response control in the response to a " + 568 "search for entries below " + baseDN + ": " + 569 StaticUtils.getExceptionMessage(le)); 570 return le.getResultCode(); 571 } 572 573 if (pagedResultsResponse != null) 574 { 575 if (pagedResultsResponse.moreResultsToReturn()) 576 { 577 cookie = pagedResultsResponse.getCookie(); 578 } 579 else 580 { 581 cookie = null; 582 } 583 } 584 } 585 while (cookie != null); 586 } 587 588 589 // See if there were any missing references found. 590 boolean missingReferenceFound = false; 591 for (final Map.Entry<String,AtomicLong> e : 592 missingReferenceCounts.entrySet()) 593 { 594 final long numMissing = e.getValue().get(); 595 if (numMissing > 0L) 596 { 597 if (! missingReferenceFound) 598 { 599 err(); 600 missingReferenceFound = true; 601 } 602 603 err("Found " + numMissing + ' ' + e.getKey() + 604 " references to entries that do not exist."); 605 } 606 } 607 608 if (missingReferenceFound) 609 { 610 return ResultCode.CONSTRAINT_VIOLATION; 611 } 612 else 613 { 614 out("No references were found to entries that do not exist."); 615 return ResultCode.SUCCESS; 616 } 617 } 618 finally 619 { 620 findReferencesPool.close(); 621 622 if (getReferencedEntriesPool != null) 623 { 624 getReferencedEntriesPool.close(); 625 } 626 } 627 } 628 629 630 631 /** 632 * Retrieves a map that correlates the number of missing references found by 633 * attribute type. 634 * 635 * @return A map that correlates the number of missing references found by 636 * attribute type. 637 */ 638 @NotNull() 639 public Map<String,AtomicLong> getMissingReferenceCounts() 640 { 641 return Collections.unmodifiableMap(missingReferenceCounts); 642 } 643 644 645 646 /** 647 * Retrieves a set of information that may be used to generate example usage 648 * information. Each element in the returned map should consist of a map 649 * between an example set of arguments and a string that describes the 650 * behavior of the tool when invoked with that set of arguments. 651 * 652 * @return A set of information that may be used to generate example usage 653 * information. It may be {@code null} or empty if no example usage 654 * information is available. 655 */ 656 @Override() 657 @NotNull() 658 public LinkedHashMap<String[],String> getExampleUsages() 659 { 660 final LinkedHashMap<String[],String> exampleMap = 661 new LinkedHashMap<>(StaticUtils.computeMapCapacity(1)); 662 663 final String[] args = 664 { 665 "--hostname", "server.example.com", 666 "--port", "389", 667 "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com", 668 "--bindPassword", "password", 669 "--baseDN", "dc=example,dc=com", 670 "--attribute", "member", 671 "--attribute", "uniqueMember", 672 "--simplePageSize", "100" 673 }; 674 exampleMap.put(args, 675 "Identify all entries below dc=example,dc=com in which either the " + 676 "member or uniqueMember attribute references an entry that " + 677 "does not exist."); 678 679 return exampleMap; 680 } 681 682 683 684 /** 685 * Indicates that the provided search result entry has been returned by the 686 * server and may be processed by this search result listener. 687 * 688 * @param searchEntry The search result entry that has been returned by the 689 * server. 690 */ 691 @Override() 692 public void searchEntryReturned(@NotNull final SearchResultEntry searchEntry) 693 { 694 try 695 { 696 // Find attributes which references to entries that do not exist. 697 for (final String attr : attributes) 698 { 699 final List<Attribute> attrList = 700 searchEntry.getAttributesWithOptions(attr, null); 701 for (final Attribute a : attrList) 702 { 703 for (final String value : a.getValues()) 704 { 705 try 706 { 707 final SearchResultEntry e = 708 getReferencedEntriesPool.getEntry(value, "1.1"); 709 if (e == null) 710 { 711 err("Entry '", searchEntry.getDN(), "' includes attribute ", 712 a.getName(), " that references entry '", value, 713 "' which does not exist."); 714 missingReferenceCounts.get(attr).incrementAndGet(); 715 } 716 } 717 catch (final LDAPException le) 718 { 719 Debug.debugException(le); 720 err("An error occurred while attempting to determine whether " + 721 "entry '" + value + "' referenced in attribute " + 722 a.getName() + " of entry '" + searchEntry.getDN() + 723 "' exists: " + StaticUtils.getExceptionMessage(le)); 724 missingReferenceCounts.get(attr).incrementAndGet(); 725 } 726 } 727 } 728 } 729 } 730 finally 731 { 732 final long count = entriesExamined.incrementAndGet(); 733 if ((count % 1000L) == 0L) 734 { 735 out(count, " entries examined"); 736 } 737 } 738 } 739 740 741 742 /** 743 * Indicates that the provided search result reference has been returned by 744 * the server and may be processed by this search result listener. 745 * 746 * @param searchReference The search result reference that has been returned 747 * by the server. 748 */ 749 @Override() 750 public void searchReferenceReturned( 751 @NotNull final SearchResultReference searchReference) 752 { 753 // No implementation is required. This tool will not follow referrals. 754 } 755}