From 8102ef3404b97d753d8166bd0f995745907d43a5 Mon Sep 17 00:00:00 2001
From: =?utf8?q?David=20=E2=80=98Bombe=E2=80=99=20Roden?=
 <bombe@pterodactylus.net>
Date: Mon, 9 Mar 2009 18:01:25 +0100
Subject: [PATCH] Add URL fetcher.

---
 src/net/pterodactylus/arachne/core/URLFetcher.java | 59 ++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 src/net/pterodactylus/arachne/core/URLFetcher.java

diff --git a/src/net/pterodactylus/arachne/core/URLFetcher.java b/src/net/pterodactylus/arachne/core/URLFetcher.java
new file mode 100644
index 0000000..8f8465d
--- /dev/null
+++ b/src/net/pterodactylus/arachne/core/URLFetcher.java
@@ -0,0 +1,59 @@
+package net.pterodactylus.arachne.core;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import de.ina.util.validation.Validation;
+
+/**
+ * Fetches URLs, parses the received content (if it is HTML) and adds all
+ * resulting links to the queue in the core.
+ *
+ * @author David ‘Bombe’ Roden <bombe@pterodactylus.net>
+ */
+class URLFetcher implements Runnable {
+
+	/** The logger. */
+	private static final Logger logger = Logger.getLogger(URLFetcher.class.getName());
+
+	/** The core. */
+	private final Core core;
+
+	/** The URL to fetch. */
+	private final URL url;
+
+	/**
+	 * Creates a new fetcher for the given URL.
+	 *
+	 * @param url
+	 *            The URL to fetch
+	 * @param core
+	 *            TODO
+	 */
+	public URLFetcher(Core core, URL url) {
+		Validation.begin().isNotNull("core", core).isNotNull("url", url).check();
+		this.core = core;
+		this.url = url;
+	}
+
+	/**
+	 * {@inheritdoc}
+	 *
+	 * @see java.lang.Runnable#run()
+	 */
+	public void run() {
+		logger.log(Level.INFO, "Starting URL Fetcher for “" + url + "”.");
+		try {
+			URLConnection urlConnection = url.openConnection();
+			long contentLength = urlConnection.getContentLength();
+			String contentType = urlConnection.getContentType();
+			logger.log(Level.INFO, "Type is “" + contentType + "”, length is " + contentLength + ".");
+		} catch (IOException ioe1) {
+			logger.log(Level.WARNING, "Could not fetch “" + url + "”.", ioe1);
+		}
+	}
+
+}
-- 
2.7.4