From: David ‘Bombe’ Roden Date: Mon, 9 Mar 2009 17:01:25 +0000 (+0100) Subject: Add URL fetcher. X-Git-Url: https://git.pterodactylus.net/?a=commitdiff_plain;h=8102ef3404b97d753d8166bd0f995745907d43a5;p=arachne.git Add URL fetcher. --- diff --git a/src/net/pterodactylus/arachne/core/URLFetcher.java b/src/net/pterodactylus/arachne/core/URLFetcher.java new file mode 100644 index 0000000..8f8465d --- /dev/null +++ b/src/net/pterodactylus/arachne/core/URLFetcher.java @@ -0,0 +1,59 @@ +package net.pterodactylus.arachne.core; + +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.logging.Level; +import java.util.logging.Logger; + +import de.ina.util.validation.Validation; + +/** + * Fetches URLs, parses the received content (if it is HTML) and adds all + * resulting links to the queue in the core. + * + * @author David ‘Bombe’ Roden + */ +class URLFetcher implements Runnable { + + /** The logger. */ + private static final Logger logger = Logger.getLogger(URLFetcher.class.getName()); + + /** The core. */ + private final Core core; + + /** The URL to fetch. */ + private final URL url; + + /** + * Creates a new fetcher for the given URL. + * + * @param url + * The URL to fetch + * @param core + * TODO + */ + public URLFetcher(Core core, URL url) { + Validation.begin().isNotNull("core", core).isNotNull("url", url).check(); + this.core = core; + this.url = url; + } + + /** + * {@inheritdoc} + * + * @see java.lang.Runnable#run() + */ + public void run() { + logger.log(Level.INFO, "Starting URL Fetcher for “" + url + "”."); + try { + URLConnection urlConnection = url.openConnection(); + long contentLength = urlConnection.getContentLength(); + String contentType = urlConnection.getContentType(); + logger.log(Level.INFO, "Type is “" + contentType + "”, length is " + contentLength + "."); + } catch (IOException ioe1) { + logger.log(Level.WARNING, "Could not fetch “" + url + "”.", ioe1); + } + } + +}