001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedReader; 007import java.io.ByteArrayOutputStream; 008import java.io.Closeable; 009import java.io.File; 010import java.io.IOException; 011import java.io.InputStream; 012import java.net.HttpURLConnection; 013import java.net.MalformedURLException; 014import java.net.URL; 015import java.nio.charset.StandardCharsets; 016import java.nio.file.Files; 017import java.nio.file.StandardCopyOption; 018import java.util.ArrayList; 019import java.util.Arrays; 020import java.util.Enumeration; 021import java.util.List; 022import java.util.Map; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.concurrent.TimeUnit; 025import java.util.zip.ZipEntry; 026import java.util.zip.ZipFile; 027 028import org.openstreetmap.josm.Main; 029import org.openstreetmap.josm.spi.preferences.Config; 030import org.openstreetmap.josm.tools.HttpClient; 031import org.openstreetmap.josm.tools.Logging; 032import org.openstreetmap.josm.tools.Pair; 033import org.openstreetmap.josm.tools.Utils; 034 035/** 036 * Downloads a file and caches it on disk in order to reduce network load. 037 * 038 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 039 * resources from the current *.jar file. (Local caching is only done for URLs.) 040 * <p> 041 * The mirrored file is only downloaded if it has been more than 7 days since 042 * last download. (Time can be configured.) 043 * <p> 044 * The file content is normally accessed with {@link #getInputStream()}, but 045 * you can also get the mirrored copy with {@link #getFile()}. 046 */ 047public class CachedFile implements Closeable { 048 049 /** 050 * Caching strategy. 051 */ 052 public enum CachingStrategy { 053 /** 054 * If cached file on disk is older than a certain time (7 days by default), 055 * consider the cache stale and try to download the file again. 056 */ 057 MaxAge, 058 /** 059 * Similar to MaxAge, considers the cache stale when a certain age is 060 * exceeded. In addition, a If-Modified-Since HTTP header is added. 061 * When the server replies "304 Not Modified", this is considered the same 062 * as a full download. 063 */ 064 IfModifiedSince 065 } 066 067 protected String name; 068 protected long maxAge; 069 protected String destDir; 070 protected String httpAccept; 071 protected CachingStrategy cachingStrategy; 072 073 private boolean fastFail; 074 private HttpClient activeConnection; 075 protected File cacheFile; 076 protected boolean initialized; 077 protected String parameter; 078 079 public static final long DEFAULT_MAXTIME = -1L; 080 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days 081 082 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>(); 083 084 /** 085 * Constructs a CachedFile object from a given filename, URL or internal resource. 086 * 087 * @param name can be:<ul> 088 * <li>relative or absolute file name</li> 089 * <li>{@code file:///SOME/FILE} the same as above</li> 090 * <li>{@code http://...} a URL. It will be cached on disk.</li> 091 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 092 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 093 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 094 */ 095 public CachedFile(String name) { 096 this.name = name; 097 } 098 099 /** 100 * Set the name of the resource. 101 * @param name can be:<ul> 102 * <li>relative or absolute file name</li> 103 * <li>{@code file:///SOME/FILE} the same as above</li> 104 * <li>{@code http://...} a URL. It will be cached on disk.</li> 105 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 106 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 107 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 108 * @return this object 109 */ 110 public CachedFile setName(String name) { 111 this.name = name; 112 return this; 113 } 114 115 /** 116 * Set maximum age of cache file. Only applies to URLs. 117 * When this time has passed after the last download of the file, the 118 * cache is considered stale and a new download will be attempted. 119 * @param maxAge the maximum cache age in seconds 120 * @return this object 121 */ 122 public CachedFile setMaxAge(long maxAge) { 123 this.maxAge = maxAge; 124 return this; 125 } 126 127 /** 128 * Set the destination directory for the cache file. Only applies to URLs. 129 * @param destDir the destination directory 130 * @return this object 131 */ 132 public CachedFile setDestDir(String destDir) { 133 this.destDir = destDir; 134 return this; 135 } 136 137 /** 138 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 139 * @param httpAccept the accepted MIME types 140 * @return this object 141 */ 142 public CachedFile setHttpAccept(String httpAccept) { 143 this.httpAccept = httpAccept; 144 return this; 145 } 146 147 /** 148 * Set the caching strategy. Only applies to URLs. 149 * @param cachingStrategy caching strategy 150 * @return this object 151 */ 152 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 153 this.cachingStrategy = cachingStrategy; 154 return this; 155 } 156 157 /** 158 * Sets the http headers. Only applies to URL pointing to http or https resources 159 * @param headers that should be sent together with request 160 * @return this object 161 */ 162 public CachedFile setHttpHeaders(Map<String, String> headers) { 163 this.httpHeaders.putAll(headers); 164 return this; 165 } 166 167 /** 168 * Sets whether opening HTTP connections should fail fast, i.e., whether a 169 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used. 170 * @param fastFail whether opening HTTP connections should fail fast 171 */ 172 public void setFastFail(boolean fastFail) { 173 this.fastFail = fastFail; 174 } 175 176 /** 177 * Sets additional URL parameter (used e.g. for maps) 178 * @param parameter the URL parameter 179 * @since 13536 180 */ 181 public void setParam(String parameter) { 182 this.parameter = parameter; 183 } 184 185 public String getName() { 186 if (parameter != null) 187 return name.replaceAll("%<(.*)>", ""); 188 return name; 189 } 190 191 /** 192 * Returns maximum age of cache file. Only applies to URLs. 193 * When this time has passed after the last download of the file, the 194 * cache is considered stale and a new download will be attempted. 195 * @return the maximum cache age in seconds 196 */ 197 public long getMaxAge() { 198 return maxAge; 199 } 200 201 public String getDestDir() { 202 return destDir; 203 } 204 205 public String getHttpAccept() { 206 return httpAccept; 207 } 208 209 public CachingStrategy getCachingStrategy() { 210 return cachingStrategy; 211 } 212 213 /** 214 * Get InputStream to the requested resource. 215 * @return the InputStream 216 * @throws IOException when the resource with the given name could not be retrieved 217 */ 218 public InputStream getInputStream() throws IOException { 219 File file = getFile(); 220 if (file == null) { 221 if (name != null && name.startsWith("resource://")) { 222 InputStream is = getClass().getResourceAsStream( 223 name.substring("resource:/".length())); 224 if (is == null) 225 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 226 return is; 227 } else { 228 throw new IOException("No file found for: "+name); 229 } 230 } 231 return Files.newInputStream(file.toPath()); 232 } 233 234 /** 235 * Get the full content of the requested resource as a byte array. 236 * @return the full content of the requested resource as byte array 237 * @throws IOException in case of an I/O error 238 */ 239 public byte[] getByteContent() throws IOException { 240 try (InputStream is = getInputStream()) { 241 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 242 int nRead; 243 byte[] data = new byte[8192]; 244 while ((nRead = is.read(data, 0, data.length)) != -1) { 245 buffer.write(data, 0, nRead); 246 } 247 buffer.flush(); 248 return buffer.toByteArray(); 249 } 250 } 251 252 /** 253 * Returns {@link #getInputStream()} wrapped in a buffered reader. 254 * <p> 255 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 256 * 257 * @return buffered reader 258 * @throws IOException if any I/O error occurs 259 * @since 9411 260 */ 261 public BufferedReader getContentReader() throws IOException { 262 return new BufferedReader(UTFInputStreamReader.create(getInputStream())); 263 } 264 265 /** 266 * Get local file for the requested resource. 267 * @return The local cache file for URLs. If the resource is a local file, 268 * returns just that file. 269 * @throws IOException when the resource with the given name could not be retrieved 270 */ 271 public synchronized File getFile() throws IOException { 272 if (initialized) 273 return cacheFile; 274 initialized = true; 275 URL url; 276 try { 277 url = new URL(name); 278 if ("file".equals(url.getProtocol())) { 279 cacheFile = new File(name.substring("file:/".length() - 1)); 280 if (!cacheFile.exists()) { 281 cacheFile = new File(name.substring("file://".length() - 1)); 282 } 283 } else { 284 cacheFile = checkLocal(url); 285 } 286 } catch (MalformedURLException e) { 287 if (name == null || name.startsWith("resource://")) { 288 return null; 289 } else if (name.startsWith("josmdir://")) { 290 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length())); 291 } else if (name.startsWith("josmplugindir://")) { 292 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length())); 293 } else { 294 cacheFile = new File(name); 295 } 296 } 297 if (cacheFile == null) 298 throw new IOException("Unable to get cache file for "+getName()); 299 return cacheFile; 300 } 301 302 /** 303 * Looks for a certain entry inside a zip file and returns the entry path. 304 * 305 * Replies a file in the top level directory of the ZIP file which has an 306 * extension <code>extension</code>. If more than one files have this 307 * extension, the last file whose name includes <code>namepart</code> 308 * is opened. 309 * 310 * @param extension the extension of the file we're looking for 311 * @param namepart the name part 312 * @return The zip entry path of the matching file. <code>null</code> if this cached file 313 * doesn't represent a zip file or if there was no matching 314 * file in the ZIP file. 315 */ 316 public String findZipEntryPath(String extension, String namepart) { 317 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 318 if (ze == null) return null; 319 return ze.a; 320 } 321 322 /** 323 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 324 * @param extension the extension of the file we're looking for 325 * @param namepart the name part 326 * @return InputStream to the matching file. <code>null</code> if this cached file 327 * doesn't represent a zip file or if there was no matching 328 * file in the ZIP file. 329 * @since 6148 330 */ 331 public InputStream findZipEntryInputStream(String extension, String namepart) { 332 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 333 if (ze == null) return null; 334 return ze.b; 335 } 336 337 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 338 File file = null; 339 try { 340 file = getFile(); 341 } catch (IOException ex) { 342 Logging.log(Logging.LEVEL_WARN, ex); 343 } 344 if (file == null) 345 return null; 346 Pair<String, InputStream> res = null; 347 try { 348 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 349 ZipEntry resentry = null; 350 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 351 while (entries.hasMoreElements()) { 352 ZipEntry entry = entries.nextElement(); 353 // choose any file with correct extension. When more than one file, prefer the one which matches namepart 354 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) { 355 resentry = entry; 356 } 357 } 358 if (resentry != null) { 359 InputStream is = zipFile.getInputStream(resentry); 360 res = Pair.create(resentry.getName(), is); 361 } else { 362 Utils.close(zipFile); 363 } 364 } catch (IOException e) { 365 if (file.getName().endsWith(".zip")) { 366 Logging.log(Logging.LEVEL_WARN, 367 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 368 file.getName(), e.toString(), extension, namepart), e); 369 } 370 } 371 return res; 372 } 373 374 /** 375 * Clear the cache for the given resource. 376 * This forces a fresh download. 377 * @param name the URL 378 */ 379 public static void cleanup(String name) { 380 cleanup(name, null); 381 } 382 383 /** 384 * Clear the cache for the given resource. 385 * This forces a fresh download. 386 * @param name the URL 387 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 388 */ 389 public static void cleanup(String name, String destDir) { 390 URL url; 391 try { 392 url = new URL(name); 393 if (!"file".equals(url.getProtocol())) { 394 String prefKey = getPrefKey(url, destDir, null); 395 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey)); 396 if (localPath.size() == 2) { 397 File lfile = new File(localPath.get(1)); 398 if (lfile.exists()) { 399 Utils.deleteFile(lfile); 400 } 401 } 402 Config.getPref().putList(prefKey, null); 403 } 404 } catch (MalformedURLException e) { 405 Logging.warn(e); 406 } 407 } 408 409 /** 410 * Get preference key to store the location and age of the cached file. 411 * 2 resources that point to the same url, but that are to be stored in different 412 * directories will not share a cache file. 413 * @param url URL 414 * @param destDir destination directory 415 * @param parameter additional URL parameter (used e.g. for maps) 416 * @return Preference key 417 */ 418 private static String getPrefKey(URL url, String destDir, String parameter) { 419 StringBuilder prefKey = new StringBuilder("mirror."); 420 if (destDir != null) { 421 prefKey.append(destDir).append('.'); 422 } 423 if (parameter != null) { 424 prefKey.append(url.toString().replaceAll("%<(.*)>", "")); 425 } else { 426 prefKey.append(url.toString()); 427 } 428 return prefKey.toString().replaceAll("=", "_"); 429 } 430 431 private File checkLocal(URL url) throws IOException { 432 String prefKey = getPrefKey(url, destDir, parameter); 433 String urlStr = url.toExternalForm(); 434 if (parameter != null) 435 urlStr = urlStr.replaceAll("%<(.*)>", ""); 436 long age = 0L; 437 long maxAgeMillis = maxAge; 438 Long ifModifiedSince = null; 439 File localFile = null; 440 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey)); 441 boolean offline = false; 442 try { 443 checkOfflineAccess(urlStr); 444 } catch (OfflineAccessException e) { 445 Logging.trace(e); 446 offline = true; 447 } 448 if (localPathEntry.size() == 2) { 449 localFile = new File(localPathEntry.get(1)); 450 if (!localFile.exists()) { 451 localFile = null; 452 } else { 453 if (maxAge == DEFAULT_MAXTIME 454 || maxAge <= 0 // arbitrary value <= 0 is deprecated 455 ) { 456 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7))); 457 } 458 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 459 if (offline || age < maxAgeMillis) { 460 return localFile; 461 } 462 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 463 ifModifiedSince = Long.valueOf(localPathEntry.get(0)); 464 } 465 } 466 } 467 if (destDir == null) { 468 destDir = Config.getDirs().getCacheDirectory(true).getPath(); 469 } 470 471 File destDirFile = new File(destDir); 472 if (!destDirFile.exists()) { 473 Utils.mkDirs(destDirFile); 474 } 475 476 // No local file + offline => nothing to do 477 if (offline) { 478 return null; 479 } 480 481 if (parameter != null) { 482 String u = url.toExternalForm(); 483 String uc; 484 if (parameter.isEmpty()) { 485 uc = u.replaceAll("%<(.*)>", ""); 486 } else { 487 uc = u.replaceAll("%<(.*)>", "$1"+parameter); 488 } 489 if (!uc.equals(u)) 490 url = new URL(uc); 491 } 492 493 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 494 String localPath = "mirror_" + a; 495 destDirFile = new File(destDir, localPath + ".tmp"); 496 try { 497 activeConnection = HttpClient.create(url) 498 .setAccept(httpAccept) 499 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince) 500 .setHeaders(httpHeaders); 501 if (fastFail) { 502 activeConnection.setReadTimeout(1000); 503 } 504 final HttpClient.Response con = activeConnection.connect(); 505 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 506 Logging.debug("304 Not Modified ({0})", urlStr); 507 if (localFile == null) 508 throw new AssertionError(); 509 Config.getPref().putList(prefKey, 510 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 511 return localFile; 512 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 513 throw new IOException(tr("The requested URL {0} was not found", urlStr)); 514 } 515 try (InputStream is = con.getContent()) { 516 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 517 } 518 activeConnection = null; 519 localFile = new File(destDir, localPath); 520 if (Main.platform.rename(destDirFile, localFile)) { 521 Config.getPref().putList(prefKey, 522 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 523 } else { 524 Logging.warn(tr("Failed to rename file {0} to {1}.", 525 destDirFile.getPath(), localFile.getPath())); 526 } 527 } catch (IOException e) { 528 if (age >= maxAgeMillis && age < maxAgeMillis*2) { 529 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 530 return localFile; 531 } else { 532 throw e; 533 } 534 } 535 536 return localFile; 537 } 538 539 private static void checkOfflineAccess(String urlString) { 540 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 541 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl()); 542 } 543 544 /** 545 * Attempts to disconnect an URL connection. 546 * @see HttpClient#disconnect() 547 * @since 9411 548 */ 549 @Override 550 public void close() { 551 if (activeConnection != null) { 552 activeConnection.disconnect(); 553 } 554 } 555 556 /** 557 * Clears the cached file 558 * @throws IOException if any I/O error occurs 559 * @since 10993 560 */ 561 public void clear() throws IOException { 562 URL url; 563 try { 564 url = new URL(name); 565 if ("file".equals(url.getProtocol())) { 566 return; // this is local file - do not delete it 567 } 568 } catch (MalformedURLException e) { 569 return; // if it's not a URL, then it still might be a local file - better not to delete 570 } 571 File f = getFile(); 572 if (f != null && f.exists()) { 573 Utils.deleteFile(f); 574 } 575 } 576}