1 package net.pterodactylus.arachne.core;
3 import java.io.IOException;
5 import java.net.URLConnection;
6 import java.util.logging.Level;
7 import java.util.logging.Logger;
9 import de.ina.util.validation.Validation;
12 * Fetches URLs, parses the received content (if it is HTML) and adds all
13 * resulting links to the queue in the core.
15 * @author David ‘Bombe’ Roden <bombe@pterodactylus.net>
17 class URLFetcher implements Runnable {
20 private static final Logger logger = Logger.getLogger(URLFetcher.class.getName());
23 private final Core core;
25 /** The URL to fetch. */
26 private final URL url;
29 * Creates a new fetcher for the given URL.
36 public URLFetcher(Core core, URL url) {
37 Validation.begin().isNotNull("core", core).isNotNull("url", url).check();
45 * @see java.lang.Runnable#run()
48 logger.log(Level.INFO, "Starting URL Fetcher for “" + url + "”.");
50 URLConnection urlConnection = url.openConnection();
51 long contentLength = urlConnection.getContentLength();
52 String contentType = urlConnection.getContentType();
53 logger.log(Level.INFO, "Type is “" + contentType + "”, length is " + contentLength + ".");
54 } catch (IOException ioe1) {
55 logger.log(Level.WARNING, "Could not fetch “" + url + "”.", ioe1);