001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.ByteArrayInputStream;
007import java.io.IOException;
008import java.net.URL;
009import java.nio.charset.StandardCharsets;
010import java.util.HashSet;
011import java.util.List;
012import java.util.Map;
013import java.util.Map.Entry;
014import java.util.Optional;
015import java.util.Set;
016import java.util.concurrent.ConcurrentHashMap;
017import java.util.concurrent.ConcurrentMap;
018import java.util.concurrent.ThreadPoolExecutor;
019import java.util.concurrent.TimeUnit;
020import java.util.regex.Matcher;
021import java.util.regex.Pattern;
022
023import org.apache.commons.jcs.access.behavior.ICacheAccess;
024import org.openstreetmap.gui.jmapviewer.Tile;
025import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
026import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
027import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
028import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
029import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
030import org.openstreetmap.josm.data.cache.CacheEntry;
031import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
032import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
033import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
034import org.openstreetmap.josm.data.preferences.LongProperty;
035import org.openstreetmap.josm.tools.HttpClient;
036import org.openstreetmap.josm.tools.Logging;
037
038/**
039 * Class bridging TMS requests to JCS cache requests
040 *
041 * @author Wiktor Niesiobędzki
042 * @since 8168
043 */
044public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
045    private static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
046    private static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
047    private static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException>(.+)</ServiceException>.+");
048    protected final Tile tile;
049    private volatile URL url;
050
051    // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
052    // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
053    private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
054
055    /**
056     * Constructor for creating a job, to get a specific tile from cache
057     * @param listener Tile loader listener
058     * @param tile to be fetched from cache
059     * @param cache object
060     * @param connectTimeout when connecting to remote resource
061     * @param readTimeout when connecting to remote resource
062     * @param headers HTTP headers to be sent together with request
063     * @param downloadExecutor that will be executing the jobs
064     */
065    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
066            ICacheAccess<String, BufferedImageCacheEntry> cache,
067            int connectTimeout, int readTimeout, Map<String, String> headers,
068            ThreadPoolExecutor downloadExecutor) {
069        super(cache, connectTimeout, readTimeout, headers, downloadExecutor);
070        this.tile = tile;
071        if (listener != null) {
072            String deduplicationKey = getCacheKey();
073            synchronized (inProgress) {
074                inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener);
075            }
076        }
077    }
078
079    @Override
080    public String getCacheKey() {
081        if (tile != null) {
082            TileSource tileSource = tile.getTileSource();
083            return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':'
084                    + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
085        }
086        return null;
087    }
088
089    /*
090     *  this doesn't needs to be synchronized, as it's not that costly to keep only one execution
091     *  in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
092     *  data from cache, that's why URL creation is postponed until it's needed
093     *
094     *  We need to have static url value for TileLoaderJob, as for some TileSources we might get different
095     *  URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
096     *
097     */
098    @Override
099    public URL getUrl() throws IOException {
100        if (url == null) {
101            synchronized (this) {
102                if (url == null) {
103                    String sUrl = tile.getUrl();
104                    if (!"".equals(sUrl)) {
105                        url = new URL(sUrl);
106                    }
107                }
108            }
109        }
110        return url;
111    }
112
113    @Override
114    public boolean isObjectLoadable() {
115        if (cacheData != null) {
116            byte[] content = cacheData.getContent();
117            try {
118                return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
119            } catch (IOException e) {
120                Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
121                        new Object[] {tile.getKey(), e.getMessage()}
122                        );
123            }
124        }
125        return false;
126    }
127
128    @Override
129    protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
130        attributes.setMetadata(tile.getTileSource().getMetadata(headers));
131        if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
132            attributes.setNoTileAtZoom(true);
133            return false; // do no try to load data from no-tile at zoom, cache empty object instead
134        }
135        return super.isResponseLoadable(headers, statusCode, content);
136    }
137
138    @Override
139    protected boolean cacheAsEmpty() {
140        return isNoTileAtZoom() || super.cacheAsEmpty();
141    }
142
143    @Override
144    public void submit(boolean force) {
145        tile.initLoading();
146        try {
147            super.submit(this, force);
148        } catch (IOException | IllegalArgumentException e) {
149            // if we fail to submit the job, mark tile as loaded and set error message
150            Logging.log(Logging.LEVEL_WARN, e);
151            tile.finishLoading();
152            tile.setError(e.getMessage());
153        }
154    }
155
156    @Override
157    public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
158        this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
159        Set<TileLoaderListener> listeners;
160        synchronized (inProgress) {
161            listeners = inProgress.remove(getCacheKey());
162        }
163        boolean status = result.equals(LoadResult.SUCCESS);
164
165        try {
166            tile.finishLoading(); // whatever happened set that loading has finished
167            // set tile metadata
168            if (this.attributes != null) {
169                for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
170                    tile.putValue(e.getKey(), e.getValue());
171                }
172            }
173
174            switch(result) {
175            case SUCCESS:
176                handleNoTileAtZoom();
177                if (attributes != null) {
178                    int httpStatusCode = attributes.getResponseCode();
179                    if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
180                        if (attributes.getErrorMessage() == null) {
181                            tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
182                        } else {
183                            tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
184                        }
185                        status = false;
186                    }
187                }
188                status &= tryLoadTileImage(object); //try to keep returned image as background
189                break;
190            case FAILURE:
191                tile.setError("Problem loading tile");
192                tryLoadTileImage(object);
193                break;
194            case CANCELED:
195                tile.loadingCanceled();
196                // do nothing
197            }
198
199            // always check, if there is some listener interested in fact, that tile has finished loading
200            if (listeners != null) { // listeners might be null, if some other thread notified already about success
201                for (TileLoaderListener l: listeners) {
202                    l.tileLoadingFinished(tile, status);
203                }
204            }
205        } catch (IOException e) {
206            Logging.warn("JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
207            tile.setError(e);
208            tile.setLoaded(false);
209            if (listeners != null) { // listeners might be null, if some other thread notified already about success
210                for (TileLoaderListener l: listeners) {
211                    l.tileLoadingFinished(tile, false);
212                }
213            }
214        }
215    }
216
217    /**
218     * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
219     *
220     * @return base URL of TMS or server url as defined in super class
221     */
222    @Override
223    protected String getServerKey() {
224        TileSource ts = tile.getSource();
225        if (ts instanceof AbstractTMSTileSource) {
226            return ((AbstractTMSTileSource) ts).getBaseUrl();
227        }
228        return super.getServerKey();
229    }
230
231    @Override
232    protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
233        return new BufferedImageCacheEntry(content);
234    }
235
236    @Override
237    public void submit() {
238        submit(false);
239    }
240
241    @Override
242    protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
243        CacheEntryAttributes ret = super.parseHeaders(urlConn);
244        // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
245        // at least for some short period of time, but not too long
246        if (ret.getExpirationTime() < now + MINIMUM_EXPIRES.get()) {
247            ret.setExpirationTime(now + MINIMUM_EXPIRES.get());
248        }
249        if (ret.getExpirationTime() > now + MAXIMUM_EXPIRES.get()) {
250            ret.setExpirationTime(now + MAXIMUM_EXPIRES.get());
251        }
252        return ret;
253    }
254
255    private boolean handleNoTileAtZoom() {
256        if (isNoTileAtZoom()) {
257            Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
258            tile.setError("No tile at this zoom level");
259            tile.putValue("tile-info", "no-tile");
260            return true;
261        }
262        return false;
263    }
264
265    private boolean isNoTileAtZoom() {
266        if (attributes == null) {
267            Logging.warn("Cache attributes are null");
268        }
269        return attributes != null && attributes.isNoTileAtZoom();
270    }
271
272    private boolean tryLoadTileImage(CacheEntry object) throws IOException {
273        if (object != null) {
274            byte[] content = object.getContent();
275            if (content.length > 0) {
276                try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
277                    tile.loadImage(in);
278                    if (tile.getImage() == null) {
279                        String s = new String(content, StandardCharsets.UTF_8);
280                        Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
281                        if (m.matches()) {
282                            tile.setError(m.group(1));
283                            Logging.error(m.group(1));
284                            Logging.debug(s);
285                        } else {
286                            tile.setError(tr("Could not load image from tile server"));
287                        }
288                        return false;
289                    }
290                } catch (UnsatisfiedLinkError e) {
291                    throw new IOException(e);
292                }
293            }
294        }
295        return true;
296    }
297}