From: David ‘Bombe’ Roden <bombe@pterodactylus.net>
Date: Mon, 9 Mar 2009 17:01:25 +0000 (+0100)
Subject: Add URL fetcher.
X-Git-Url: https://git.pterodactylus.net/?a=commitdiff_plain;h=8102ef3404b97d753d8166bd0f995745907d43a5;p=arachne.git

Add URL fetcher.
---

diff --git a/src/net/pterodactylus/arachne/core/URLFetcher.java b/src/net/pterodactylus/arachne/core/URLFetcher.java
new file mode 100644
index 0000000..8f8465d
--- /dev/null
+++ b/src/net/pterodactylus/arachne/core/URLFetcher.java
@@ -0,0 +1,59 @@
+package net.pterodactylus.arachne.core;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import de.ina.util.validation.Validation;
+
+/**
+ * Fetches URLs, parses the received content (if it is HTML) and adds all
+ * resulting links to the queue in the core.
+ *
+ * @author David ‘Bombe’ Roden <bombe@pterodactylus.net>
+ */
+class URLFetcher implements Runnable {
+
+	/** The logger. */
+	private static final Logger logger = Logger.getLogger(URLFetcher.class.getName());
+
+	/** The core. */
+	private final Core core;
+
+	/** The URL to fetch. */
+	private final URL url;
+
+	/**
+	 * Creates a new fetcher for the given URL.
+	 *
+	 * @param url
+	 *            The URL to fetch
+	 * @param core
+	 *            TODO
+	 */
+	public URLFetcher(Core core, URL url) {
+		Validation.begin().isNotNull("core", core).isNotNull("url", url).check();
+		this.core = core;
+		this.url = url;
+	}
+
+	/**
+	 * {@inheritdoc}
+	 *
+	 * @see java.lang.Runnable#run()
+	 */
+	public void run() {
+		logger.log(Level.INFO, "Starting URL Fetcher for “" + url + "”.");
+		try {
+			URLConnection urlConnection = url.openConnection();
+			long contentLength = urlConnection.getContentLength();
+			String contentType = urlConnection.getContentType();
+			logger.log(Level.INFO, "Type is “" + contentType + "”, length is " + contentLength + ".");
+		} catch (IOException ioe1) {
+			logger.log(Level.WARNING, "Could not fetch “" + url + "”.", ioe1);
+		}
+	}
+
+}