001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.io.InputStream;
008import java.net.Authenticator.RequestorType;
009import java.net.HttpURLConnection;
010import java.net.MalformedURLException;
011import java.net.URL;
012import java.util.List;
013
014import javax.xml.parsers.ParserConfigurationException;
015
016import org.openstreetmap.josm.Main;
017import org.openstreetmap.josm.data.gpx.GpxData;
018import org.openstreetmap.josm.data.notes.Note;
019import org.openstreetmap.josm.data.osm.DataSet;
020import org.openstreetmap.josm.gui.progress.ProgressMonitor;
021import org.openstreetmap.josm.io.auth.CredentialsAgentException;
022import org.openstreetmap.josm.io.auth.CredentialsManager;
023import org.openstreetmap.josm.tools.HttpClient;
024import org.openstreetmap.josm.tools.Logging;
025import org.openstreetmap.josm.tools.Utils;
026import org.openstreetmap.josm.tools.XmlParsingException;
027import org.w3c.dom.Document;
028import org.w3c.dom.Node;
029import org.xml.sax.SAXException;
030
031/**
032 * This DataReader reads directly from the REST API of the osm server.
033 *
034 * It supports plain text transfer as well as gzip or deflate encoded transfers;
035 * if compressed transfers are unwanted, set property osm-server.use-compression
036 * to false.
037 *
038 * @author imi
039 */
040public abstract class OsmServerReader extends OsmConnection {
041    private final OsmApi api = OsmApi.getOsmApi();
042    private boolean doAuthenticate;
043    protected boolean gpxParsedProperly;
044
045    /**
046     * Constructs a new {@code OsmServerReader}.
047     */
048    public OsmServerReader() {
049        try {
050            doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null;
051        } catch (CredentialsAgentException e) {
052            Logging.warn(e);
053        }
054    }
055
056    /**
057     * Open a connection to the given url and return a reader on the input stream
058     * from that connection. In case of user cancel, return <code>null</code>.
059     * Relative URL's are directed to API base URL.
060     * @param urlStr The url to connect to.
061     * @param progressMonitor progress monitoring and abort handler
062     * @return A reader reading the input stream (servers answer) or <code>null</code>.
063     * @throws OsmTransferException if data transfer errors occur
064     */
065    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
066        return getInputStream(urlStr, progressMonitor, null);
067    }
068
069    /**
070     * Open a connection to the given url and return a reader on the input stream
071     * from that connection. In case of user cancel, return <code>null</code>.
072     * Relative URL's are directed to API base URL.
073     * @param urlStr The url to connect to.
074     * @param progressMonitor progress monitoring and abort handler
075     * @param reason The reason to show on console. Can be {@code null} if no reason is given
076     * @return A reader reading the input stream (servers answer) or <code>null</code>.
077     * @throws OsmTransferException if data transfer errors occur
078     */
079    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
080        try {
081            api.initialize(progressMonitor);
082            String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr);
083            return getInputStreamRaw(url, progressMonitor, reason);
084        } finally {
085            progressMonitor.invalidate();
086        }
087    }
088
089    /**
090     * Return the base URL for relative URL requests
091     * @return base url of API
092     */
093    protected String getBaseUrl() {
094        return api.getBaseUrl();
095    }
096
097    /**
098     * Open a connection to the given url and return a reader on the input stream
099     * from that connection. In case of user cancel, return <code>null</code>.
100     * @param urlStr The exact url to connect to.
101     * @param progressMonitor progress monitoring and abort handler
102     * @return An reader reading the input stream (servers answer) or <code>null</code>.
103     * @throws OsmTransferException if data transfer errors occur
104     */
105    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
106        return getInputStreamRaw(urlStr, progressMonitor, null);
107    }
108
109    /**
110     * Open a connection to the given url and return a reader on the input stream
111     * from that connection. In case of user cancel, return <code>null</code>.
112     * @param urlStr The exact url to connect to.
113     * @param progressMonitor progress monitoring and abort handler
114     * @param reason The reason to show on console. Can be {@code null} if no reason is given
115     * @return An reader reading the input stream (servers answer) or <code>null</code>.
116     * @throws OsmTransferException if data transfer errors occur
117     */
118    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
119        return getInputStreamRaw(urlStr, progressMonitor, reason, false);
120    }
121
122    /**
123     * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream
124     * from that connection. In case of user cancel, return <code>null</code>.
125     * @param urlStr The exact url to connect to.
126     * @param progressMonitor progress monitoring and abort handler
127     * @param reason The reason to show on console. Can be {@code null} if no reason is given
128     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
129     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
130     * @return An reader reading the input stream (servers answer) or <code>null</code>.
131     * @throws OsmTransferException if data transfer errors occur
132     */
133    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
134            boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
135        return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null);
136    }
137
138    /**
139     * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream
140     * from that connection. In case of user cancel, return <code>null</code>.
141     * @param urlStr The exact url to connect to.
142     * @param progressMonitor progress monitoring and abort handler
143     * @param reason The reason to show on console. Can be {@code null} if no reason is given
144     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
145     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
146     * @param httpMethod HTTP method ("GET", "POST" or "PUT")
147     * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method.
148     * @return An reader reading the input stream (servers answer) or <code>null</code>.
149     * @throws OsmTransferException if data transfer errors occur
150     * @since 12596
151     */
152    @SuppressWarnings("resource")
153    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
154            boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException {
155        try {
156            OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Main.getJOSMWebsite());
157            OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl());
158
159            URL url = null;
160            try {
161                url = new URL(urlStr.replace(" ", "%20"));
162            } catch (MalformedURLException e) {
163                throw new OsmTransferException(e);
164            }
165
166            String protocol = url.getProtocol();
167            if ("file".equals(protocol) || "jar".equals(protocol)) {
168                try {
169                    return Utils.openStream(url);
170                } catch (IOException e) {
171                    throw new OsmTransferException(e);
172                }
173            }
174
175            final HttpClient client = HttpClient.create(url, httpMethod)
176                    .setFinishOnCloseOutput(false)
177                    .setReasonForRequest(reason)
178                    .setOutputMessage(tr("Downloading data..."))
179                    .setRequestBody(requestBody);
180            activeConnection = client;
181            adaptRequest(client);
182            if (doAuthenticate) {
183                addAuth(client);
184            }
185            if (cancel)
186                throw new OsmTransferCanceledException("Operation canceled");
187
188            final HttpClient.Response response;
189            try {
190                response = client.connect(progressMonitor);
191            } catch (IOException e) {
192                Logging.error(e);
193                OsmTransferException ote = new OsmTransferException(
194                        tr("Could not connect to the OSM server. Please check your internet connection."), e);
195                ote.setUrl(url.toString());
196                throw ote;
197            }
198            try {
199                if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
200                    CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER);
201                    throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null);
202                }
203
204                if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH)
205                    throw new OsmTransferCanceledException("Proxy Authentication Required");
206
207                if (response.getResponseCode() != HttpURLConnection.HTTP_OK) {
208                    String errorHeader = response.getHeaderField("Error");
209                    String errorBody = fetchResponseText(response);
210                    throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null,
211                            response.getContentType());
212                }
213
214                response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition);
215                return response.getContent();
216            } catch (OsmTransferException e) {
217                throw e;
218            } catch (IOException e) {
219                throw new OsmTransferException(e);
220            }
221        } finally {
222            progressMonitor.invalidate();
223        }
224    }
225
226    private static String fetchResponseText(final HttpClient.Response response) {
227        try {
228            return response.fetchContent();
229        } catch (IOException e) {
230            Logging.error(e);
231            return tr("Reading error text failed.");
232        }
233    }
234
235    /**
236     * Allows subclasses to modify the request.
237     * @param request the prepared request
238     * @since 9308
239     */
240    protected void adaptRequest(HttpClient request) {
241    }
242
243    /**
244     * Download OSM files from somewhere
245     * @param progressMonitor The progress monitor
246     * @return The corresponding dataset
247     * @throws OsmTransferException if any error occurs
248     */
249    public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException;
250
251    /**
252     * Download compressed OSM files from somewhere
253     * @param progressMonitor The progress monitor
254     * @param compression compression to use
255     * @return The corresponding dataset
256     * @throws OsmTransferException if any error occurs
257     * @since 13352
258     */
259    public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
260        return null;
261    }
262
263    /**
264     * Download OSM Change uncompressed files from somewhere
265     * @param progressMonitor The progress monitor
266     * @return The corresponding dataset
267     * @throws OsmTransferException if any error occurs
268     */
269    public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException {
270        return null;
271    }
272
273    /**
274     * Download OSM Change compressed files from somewhere
275     * @param progressMonitor The progress monitor
276     * @param compression compression to use
277     * @return The corresponding dataset
278     * @throws OsmTransferException if any error occurs
279     * @since 13352
280     */
281    public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
282        return null;
283    }
284
285    /**
286     * Download BZip2-compressed OSM Change files from somewhere
287     * @param progressMonitor The progress monitor
288     * @return The corresponding dataset
289     * @throws OsmTransferException if any error occurs
290     * @deprecated use {@link #parseOsmChange(ProgressMonitor, Compression)} instead
291     */
292    @Deprecated
293    public DataSet parseOsmChangeBzip2(ProgressMonitor progressMonitor) throws OsmTransferException {
294        return parseOsmChange(progressMonitor, Compression.BZIP2);
295    }
296
297    /**
298     * Download GZip-compressed OSM Change files from somewhere
299     * @param progressMonitor The progress monitor
300     * @return The corresponding dataset
301     * @throws OsmTransferException if any error occurs
302     * @deprecated use {@link #parseOsmChange(ProgressMonitor, Compression)} instead
303     */
304    @Deprecated
305    public DataSet parseOsmChangeGzip(ProgressMonitor progressMonitor) throws OsmTransferException {
306        return parseOsmChange(progressMonitor, Compression.GZIP);
307    }
308
309    /**
310     * Retrieve raw gps waypoints from the server API.
311     * @param progressMonitor The progress monitor
312     * @return The corresponding GPX tracks
313     * @throws OsmTransferException if any error occurs
314     */
315    public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException {
316        return null;
317    }
318
319    /**
320     * Retrieve compressed GPX files from somewhere.
321     * @param progressMonitor The progress monitor
322     * @param compression compression to use
323     * @return The corresponding GPX tracks
324     * @throws OsmTransferException if any error occurs
325     * @since 13352
326     */
327    public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
328        return null;
329    }
330
331    /**
332     * Retrieve BZip2-compressed GPX files from somewhere.
333     * @param progressMonitor The progress monitor
334     * @return The corresponding GPX tracks
335     * @throws OsmTransferException if any error occurs
336     * @deprecated use {@link #parseRawGps(ProgressMonitor, Compression)} instead
337     * @since 6244
338     */
339    @Deprecated
340    public GpxData parseRawGpsBzip2(ProgressMonitor progressMonitor) throws OsmTransferException {
341        return parseRawGps(progressMonitor, Compression.BZIP2);
342    }
343
344    /**
345     * Download BZip2-compressed OSM files from somewhere
346     * @param progressMonitor The progress monitor
347     * @return The corresponding dataset
348     * @throws OsmTransferException if any error occurs
349     * @deprecated use {@link #parseOsm(ProgressMonitor, Compression)} instead
350     */
351    @Deprecated
352    public DataSet parseOsmBzip2(ProgressMonitor progressMonitor) throws OsmTransferException {
353        return parseOsm(progressMonitor, Compression.BZIP2);
354    }
355
356    /**
357     * Download GZip-compressed OSM files from somewhere
358     * @param progressMonitor The progress monitor
359     * @return The corresponding dataset
360     * @throws OsmTransferException if any error occurs
361     * @deprecated use {@link #parseOsm(ProgressMonitor, Compression)} instead
362     */
363    @Deprecated
364    public DataSet parseOsmGzip(ProgressMonitor progressMonitor) throws OsmTransferException {
365        return parseOsm(progressMonitor, Compression.GZIP);
366    }
367
368    /**
369     * Download Zip-compressed OSM files from somewhere
370     * @param progressMonitor The progress monitor
371     * @return The corresponding dataset
372     * @throws OsmTransferException if any error occurs
373     * @deprecated use {@link #parseOsm(ProgressMonitor, Compression)} instead
374     * @since 6882
375     */
376    @Deprecated
377    public DataSet parseOsmZip(final ProgressMonitor progressMonitor) throws OsmTransferException {
378        return parseOsm(progressMonitor, Compression.ZIP);
379    }
380
381    /**
382     * Returns true if this reader is adding authentication credentials to the read
383     * request sent to the server.
384     *
385     * @return true if this reader is adding authentication credentials to the read
386     * request sent to the server
387     */
388    public boolean isDoAuthenticate() {
389        return doAuthenticate;
390    }
391
392    /**
393     * Sets whether this reader adds authentication credentials to the read
394     * request sent to the server.
395     *
396     * @param doAuthenticate  true if  this reader adds authentication credentials to the read
397     * request sent to the server
398     */
399    public void setDoAuthenticate(boolean doAuthenticate) {
400        this.doAuthenticate = doAuthenticate;
401    }
402
403    /**
404     * Determines if the GPX data has been parsed properly.
405     * @return true if the GPX data has been parsed properly, false otherwise
406     * @see GpxReader#parse
407     */
408    public final boolean isGpxParsedProperly() {
409        return gpxParsedProperly;
410    }
411
412    /**
413     * Downloads notes from the API, given API limit parameters
414     *
415     * @param noteLimit How many notes to download.
416     * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes.
417     * @param progressMonitor Progress monitor for user feedback
418     * @return List of notes returned by the API
419     * @throws OsmTransferException if any errors happen
420     */
421    public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
422        return null;
423    }
424
425    /**
426     * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added
427     *
428     * @param progressMonitor progress monitor
429     * @return A list of notes parsed from the URL
430     * @throws OsmTransferException if any error occurs during dialog with OSM API
431     */
432    public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException {
433        return null;
434    }
435
436    /**
437     * Download notes from a URL that contains a compressed notes dump file
438     * @param progressMonitor progress monitor
439     * @param compression compression to use
440     * @return A list of notes parsed from the URL
441     * @throws OsmTransferException if any error occurs during dialog with OSM API
442     * @since 13352
443     */
444    public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
445        return null;
446    }
447
448    /**
449     * Download notes from a URL that contains a bzip2 compressed notes dump file
450     * @param progressMonitor progress monitor
451     * @return A list of notes parsed from the URL
452     * @throws OsmTransferException if any error occurs during dialog with OSM API
453     * @deprecated Use {@link #parseRawNotes(ProgressMonitor, Compression)} instead
454     */
455    @Deprecated
456    public List<Note> parseRawNotesBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
457        return parseRawNotes(progressMonitor, Compression.BZIP2);
458    }
459
460    /**
461     * Returns an attribute from the given DOM node.
462     * @param node DOM node
463     * @param name attribute name
464     * @return attribute value for the given attribute
465     * @since 12510
466     */
467    protected static String getAttribute(Node node, String name) {
468        return node.getAttributes().getNamedItem(name).getNodeValue();
469    }
470
471    /**
472     * DOM document parser.
473     * @param <R> resulting type
474     * @since 12510
475     */
476    @FunctionalInterface
477    protected interface DomParser<R> {
478        /**
479         * Parses a given DOM document.
480         * @param doc DOM document
481         * @return parsed data
482         * @throws XmlParsingException if an XML parsing error occurs
483         */
484        R parse(Document doc) throws XmlParsingException;
485    }
486
487    /**
488     * Fetches generic data from the DOM document resulting an API call.
489     * @param api the OSM API call
490     * @param subtask the subtask translated message
491     * @param parser the parser converting the DOM document (OSM API result)
492     * @param <T> data type
493     * @param monitor The progress monitor
494     * @param reason The reason to show on console. Can be {@code null} if no reason is given
495     * @return The converted data
496     * @throws OsmTransferException if something goes wrong
497     * @since 12510
498     */
499    public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason)
500            throws OsmTransferException {
501        try {
502            monitor.beginTask("");
503            monitor.indeterminateSubTask(subtask);
504            try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) {
505                return parser.parse(Utils.parseSafeDOM(in));
506            }
507        } catch (OsmTransferException e) {
508            throw e;
509        } catch (IOException | ParserConfigurationException | SAXException e) {
510            throw new OsmTransferException(e);
511        } finally {
512            monitor.finishTask();
513        }
514    }
515}