From: David ‘Bombe’ Roden Date: Thu, 10 Jan 2013 21:24:55 +0000 (+0100) Subject: Merge branch 'master' into rewrite X-Git-Tag: 0.1~38 X-Git-Url: https://git.pterodactylus.net/?a=commitdiff_plain;h=6ec36ef950c23c135bf0e112d932c5b7068189b8;hp=eee108815d981f253877848e1fa60952219e5bff;p=rhynodge.git Merge branch 'master' into rewrite Conflicts: src/main/java/net/pterodactylus/rhynodge/filters/KickAssTorrentsFilter.java --- diff --git a/src/main/java/net/pterodactylus/rhynodge/filters/KickAssTorrentsFilter.java b/src/main/java/net/pterodactylus/rhynodge/filters/KickAssTorrentsFilter.java index 11865e7..c368bf3 100644 --- a/src/main/java/net/pterodactylus/rhynodge/filters/KickAssTorrentsFilter.java +++ b/src/main/java/net/pterodactylus/rhynodge/filters/KickAssTorrentsFilter.java @@ -17,18 +17,10 @@ package net.pterodactylus.rhynodge.filters; -import static com.google.common.base.Preconditions.checkState; - -import java.net.URI; -import java.net.URISyntaxException; - import net.pterodactylus.rhynodge.Filter; -import net.pterodactylus.rhynodge.State; import net.pterodactylus.rhynodge.queries.HttpQuery; -import net.pterodactylus.rhynodge.states.FailedState; import net.pterodactylus.rhynodge.states.HtmlState; import net.pterodactylus.rhynodge.states.TorrentState; -import net.pterodactylus.rhynodge.states.TorrentState.TorrentFile; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -41,127 +33,73 @@ import org.jsoup.select.Elements; * * @author David ‘Bombe’ Roden */ -public class KickAssTorrentsFilter implements Filter { +public class KickAssTorrentsFilter extends TorrentSiteFilter { + + // + // TORRENTSITEFILTER METHODS + // /** * {@inheritDoc} */ @Override - public State filter(State state) { - if (!state.success()) { - return FailedState.from(state); - } - checkState(state instanceof HtmlState, "state is not an HtmlState but a %s", state.getClass().getName()); - - /* get result table. */ - Document document = ((HtmlState) state).document(); - Elements mainTable = document.select("table.data"); - if (mainTable.isEmpty()) { - /* no main table? */ - return new FailedState(); - } - - /* iterate over all rows. */ - TorrentState torrentState = new TorrentState(); - Elements dataRows = mainTable.select("tr:gt(0)"); - for (Element dataRow : dataRows) { - String name = extractName(dataRow); - String size = extractSize(dataRow); - String magnetUri = extractMagnetUri(dataRow); - String downloadUri; - int fileCount = extractFileCount(dataRow); - int seedCount = extractSeedCount(dataRow); - int leechCount = extractLeechCount(dataRow); - try { - downloadUri = new URI(((HtmlState) state).uri()).resolve(extractDownloadUri(dataRow)).toString(); - TorrentFile torrentFile = new TorrentFile(name, size, magnetUri, downloadUri, fileCount, seedCount, leechCount); - torrentState.addTorrentFile(torrentFile); - } catch (URISyntaxException use1) { - /* ignore; if uri was wrong, we wouldn’t be here. */ - } - } - - return torrentState; + protected Elements getDataRows(Document document) { + return document.select("table.data").select("tr:gt(0)"); } - // - // STATIC METHODS - // - /** - * Extracts the name from the given row. - * - * @param dataRow - * The row to extract the name from - * @return The extracted name + * {@inheritDoc} */ - private static String extractName(Element dataRow) { + @Override + protected String extractName(Element dataRow) { return dataRow.select("div.torrentname a.normalgrey").text(); } /** - * Extracts the size from the given row. - * - * @param dataRow - * The row to extract the size from - * @return The extracted size + * {@inheritDoc} */ - private static String extractSize(Element dataRow) { + @Override + protected String extractSize(Element dataRow) { return dataRow.select("td:eq(1)").text(); } /** - * Extracts the magnet URI from the given row. - * - * @param dataRow - * The row to extract the magnet URI from - * @return The extracted magnet URI + * {@inheritDoc} */ - private static String extractMagnetUri(Element dataRow) { + @Override + protected String extractMagnetUri(Element dataRow) { return dataRow.select("a.imagnet").attr("href"); } /** - * Extracts the download URI from the given row. - * - * @param dataRow - * The row to extract the download URI from - * @return The extracted download URI + * {@inheritDoc} */ - private static String extractDownloadUri(Element dataRow) { + @Override + protected String extractDownloadUri(Element dataRow) { return dataRow.select("a.idownload:not(.partner1Button)").attr("href"); } /** - * Extracts the file count from the given row. - * - * @param dataRow - * The row to extract the file count from - * @return The extracted file count + * {@inheritDoc} */ - private static int extractFileCount(Element dataRow) { + @Override + protected int extractFileCount(Element dataRow) { return Integer.valueOf(dataRow.select("td:eq(2)").text()); } /** - * Extracts the seed count from the given row. - * - * @param dataRow - * The row to extract the seed count from - * @return The extracted seed count + * {@inheritDoc} */ - private static int extractSeedCount(Element dataRow) { + @Override + protected int extractSeedCount(Element dataRow) { return Integer.valueOf(dataRow.select("td:eq(4)").text()); } /** - * Extracts the leech count from the given row. - * - * @param dataRow - * The row to extract the leech count from - * @return The extracted leech count + * {@inheritDoc} */ - private static int extractLeechCount(Element dataRow) { + @Override + protected int extractLeechCount(Element dataRow) { return Integer.valueOf(dataRow.select("td:eq(5)").text()); } diff --git a/src/main/java/net/pterodactylus/rhynodge/filters/PirateBayFilter.java b/src/main/java/net/pterodactylus/rhynodge/filters/PirateBayFilter.java new file mode 100644 index 0000000..45c33c1 --- /dev/null +++ b/src/main/java/net/pterodactylus/rhynodge/filters/PirateBayFilter.java @@ -0,0 +1,98 @@ +/* + * Rhynodge - PirateBayFilter.java - Copyright © 2013 David Roden + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.pterodactylus.rhynodge.filters; + +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +/** + * {@link TorrentSiteFilter} implementation that can parse + * {@code thepiratebay.se} result pages. + * + * @author David ‘Bombe’ Roden + */ +public class PirateBayFilter extends TorrentSiteFilter { + + /** + * {@inheritDoc} + */ + @Override + protected Elements getDataRows(Document document) { + return document.select("table#searchResult tbody tr:has(.vertTh)"); + } + + /** + * {@inheritDoc} + */ + @Override + protected String extractName(Element dataRow) { + return dataRow.select(".detName a").text(); + } + + /** + * {@inheritDoc} + */ + @Override + protected String extractSize(Element dataRow) { + return dataRow.select(".detDesc").text().split(Pattern.quote(","))[1].trim(); + } + + /** + * {@inheritDoc} + */ + @Override + protected String extractMagnetUri(Element dataRow) { + return dataRow.select("a[href^=magnet:]").attr("href"); + } + + /** + * {@inheritDoc} + */ + @Override + protected String extractDownloadUri(Element dataRow) { + return dataRow.select("a[href^=//torrents.]").attr("href"); + } + + /** + * {@inheritDoc} + */ + @Override + protected int extractFileCount(Element dataRow) { + return 0; + } + + /** + * {@inheritDoc} + */ + @Override + protected int extractSeedCount(Element dataRow) { + return Integer.valueOf(dataRow.select("td:eq(2)").text()); + } + + /** + * {@inheritDoc} + */ + @Override + protected int extractLeechCount(Element dataRow) { + return Integer.valueOf(dataRow.select("td:eq(3)").text()); + } + +} diff --git a/src/main/java/net/pterodactylus/rhynodge/filters/TorrentSiteFilter.java b/src/main/java/net/pterodactylus/rhynodge/filters/TorrentSiteFilter.java new file mode 100644 index 0000000..b3b5691 --- /dev/null +++ b/src/main/java/net/pterodactylus/rhynodge/filters/TorrentSiteFilter.java @@ -0,0 +1,167 @@ +/* + * Rhynodge - KickAssTorrentsFilter.java - Copyright © 2013 David Roden + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.pterodactylus.rhynodge.filters; + +import static com.google.common.base.Preconditions.checkState; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; + +import net.pterodactylus.rhynodge.Filter; +import net.pterodactylus.rhynodge.State; +import net.pterodactylus.rhynodge.queries.HttpQuery; +import net.pterodactylus.rhynodge.states.FailedState; +import net.pterodactylus.rhynodge.states.HtmlState; +import net.pterodactylus.rhynodge.states.TorrentState; +import net.pterodactylus.rhynodge.states.TorrentState.TorrentFile; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +/** + * {@link Filter} implementation that parses a {@link TorrentState} from an + * {@link HtmlState} which was generated by a {@link HttpQuery} to + * {@code kickasstorrents.ph}. + * + * @author David ‘Bombe’ Roden + */ +public abstract class TorrentSiteFilter implements Filter { + + /** + * {@inheritDoc} + */ + @Override + public State filter(State state) { + if (!state.success()) { + return FailedState.from(state); + } + checkState(state instanceof HtmlState, "state is not an HtmlState but a %s", state.getClass().getName()); + + /* get result table. */ + Document document = ((HtmlState) state).document(); + + /* iterate over all rows. */ + Elements dataRows = getDataRows(document); + TorrentState torrentState = new TorrentState(); + for (Element dataRow : dataRows) { + String name = extractName(dataRow); + String size = extractSize(dataRow); + String magnetUri = extractMagnetUri(dataRow); + String downloadUri = extractDownloadUri(dataRow); + int fileCount = extractFileCount(dataRow); + int seedCount = extractSeedCount(dataRow); + int leechCount = extractLeechCount(dataRow); + try { + if ((downloadUri != null) && (downloadUri.length() > 0)) { + downloadUri = new URI(((HtmlState) state).uri()).resolve(URLEncoder.encode(downloadUri, "UTF-8").replace("%2F", "/")).toString(); + } else { + downloadUri = null; + } + TorrentFile torrentFile = new TorrentFile(name, size, magnetUri, downloadUri, fileCount, seedCount, leechCount); + torrentState.addTorrentFile(torrentFile); + } catch (URISyntaxException use1) { + /* ignore; if uri was wrong, we wouldn’t be here. */ + } catch (UnsupportedEncodingException uee1) { + /* ignore, all JVMs can do UTF-8. */ + } + } + + return torrentState; + } + + // + // ABSTRACT METHODS + // + + /** + * Returns the data rows from the given document. + * + * @param document + * The document to get the data rows from + * @return The data rows + */ + protected abstract Elements getDataRows(Document document); + + /** + * Extracts the name from the given row. + * + * @param dataRow + * The row to extract the name from + * @return The extracted name + */ + protected abstract String extractName(Element dataRow); + + /** + * Extracts the size from the given row. + * + * @param dataRow + * The row to extract the size from + * @return The extracted size + */ + protected abstract String extractSize(Element dataRow); + + /** + * Extracts the magnet URI from the given row. + * + * @param dataRow + * The row to extract the magnet URI from + * @return The extracted magnet URI + */ + protected abstract String extractMagnetUri(Element dataRow); + + /** + * Extracts the download URI from the given row. + * + * @param dataRow + * The row to extract the download URI from + * @return The extracted download URI + */ + protected abstract String extractDownloadUri(Element dataRow); + + /** + * Extracts the file count from the given row. + * + * @param dataRow + * The row to extract the file count from + * @return The extracted file count, or {@code 0} if the file count can not + * be extracted + */ + protected abstract int extractFileCount(Element dataRow); + + /** + * Extracts the seed count from the given row. + * + * @param dataRow + * The row to extract the seed count from + * @return The extracted seed count + */ + protected abstract int extractSeedCount(Element dataRow); + + /** + * Extracts the leech count from the given row. + * + * @param dataRow + * The row to extract the leech count from + * @return The extracted leech count + */ + protected abstract int extractLeechCount(Element dataRow); + +} diff --git a/src/main/java/net/pterodactylus/rhynodge/states/TorrentState.java b/src/main/java/net/pterodactylus/rhynodge/states/TorrentState.java index ce5b06f..c4c3e0b 100644 --- a/src/main/java/net/pterodactylus/rhynodge/states/TorrentState.java +++ b/src/main/java/net/pterodactylus/rhynodge/states/TorrentState.java @@ -179,7 +179,8 @@ public class TorrentState extends AbstractState implements Iterable /** * Returns the magnet URI of the file. * - * @return The magnet URI of the file + * @return The magnet URI of the file, or {@code null} if there is no + * magnet URI for this torrent file */ public String magnetUri() { return magnetUri; @@ -188,7 +189,8 @@ public class TorrentState extends AbstractState implements Iterable /** * Returns the download URI of the file. * - * @return The download URI of the file + * @return The download URI of the file, or {@code null} if there is no + * download URI for this torrent file */ public String downloadUri() { return downloadUri; diff --git a/src/main/java/net/pterodactylus/rhynodge/triggers/NewEpisodeTrigger.java b/src/main/java/net/pterodactylus/rhynodge/triggers/NewEpisodeTrigger.java index f23869d..a8e321c 100644 --- a/src/main/java/net/pterodactylus/rhynodge/triggers/NewEpisodeTrigger.java +++ b/src/main/java/net/pterodactylus/rhynodge/triggers/NewEpisodeTrigger.java @@ -152,8 +152,12 @@ public class NewEpisodeTrigger implements Trigger { stringBuilder.append("- ").append(episode.identifier()).append("\n"); for (TorrentFile torrentFile : episode) { stringBuilder.append(" - ").append(torrentFile.name()).append(", ").append(torrentFile.size()).append("\n"); - stringBuilder.append(" Magnet: ").append(torrentFile.magnetUri()).append("\n"); - stringBuilder.append(" Download: ").append(torrentFile.downloadUri()).append("\n"); + if ((torrentFile.magnetUri() != null) && (torrentFile.magnetUri().length() > 0)) { + stringBuilder.append(" Magnet: ").append(torrentFile.magnetUri()).append("\n"); + } + if ((torrentFile.downloadUri() != null) && (torrentFile.downloadUri().length() > 0)) { + stringBuilder.append(" Download: ").append(torrentFile.downloadUri()).append("\n"); + } } } } @@ -163,8 +167,12 @@ public class NewEpisodeTrigger implements Trigger { stringBuilder.append("- ").append(episode.identifier()).append("\n"); for (TorrentFile torrentFile : episode) { stringBuilder.append(" - ").append(torrentFile.name()).append(", ").append(torrentFile.size()).append("\n"); - stringBuilder.append(" Magnet: ").append(torrentFile.magnetUri()).append("\n"); - stringBuilder.append(" Download: ").append(torrentFile.downloadUri()).append("\n"); + if ((torrentFile.magnetUri() != null) && (torrentFile.magnetUri().length() > 0)) { + stringBuilder.append(" Magnet: ").append(torrentFile.magnetUri()).append("\n"); + } + if ((torrentFile.downloadUri() != null) && (torrentFile.downloadUri().length() > 0)) { + stringBuilder.append(" Download: ").append(torrentFile.downloadUri()).append("\n"); + } } } } @@ -199,8 +207,14 @@ public class NewEpisodeTrigger implements Trigger { htmlBuilder.append("").append(torrentFile.fileCount()).append(" file(s), "); htmlBuilder.append("").append(torrentFile.seedCount()).append(" seed(s), "); htmlBuilder.append("").append(torrentFile.leechCount()).append(" leecher(s)\n"); - htmlBuilder.append("
Magnet "); - htmlBuilder.append("Download
\n"); + htmlBuilder.append("
"); + if ((torrentFile.magnetUri() != null) && (torrentFile.magnetUri().length() > 0)) { + htmlBuilder.append("Magnet "); + } + if ((torrentFile.downloadUri() != null) && (torrentFile.downloadUri().length() > 0)) { + htmlBuilder.append("Download"); + } + htmlBuilder.append("
\n"); } htmlBuilder.append("\n"); } @@ -219,8 +233,14 @@ public class NewEpisodeTrigger implements Trigger { htmlBuilder.append("").append(torrentFile.fileCount()).append(" file(s), "); htmlBuilder.append("").append(torrentFile.seedCount()).append(" seed(s), "); htmlBuilder.append("").append(torrentFile.leechCount()).append(" leecher(s)\n"); - htmlBuilder.append("
Magnet "); - htmlBuilder.append("Download
\n"); + htmlBuilder.append("
"); + if ((torrentFile.magnetUri() != null) && (torrentFile.magnetUri().length() > 0)) { + htmlBuilder.append("Magnet "); + } + if ((torrentFile.downloadUri() != null) && (torrentFile.downloadUri().length() > 0)) { + htmlBuilder.append("Download"); + } + htmlBuilder.append("
\n"); } htmlBuilder.append("\n"); } diff --git a/src/main/java/net/pterodactylus/rhynodge/triggers/NewTorrentTrigger.java b/src/main/java/net/pterodactylus/rhynodge/triggers/NewTorrentTrigger.java index a21a8d6..2803bb5 100644 --- a/src/main/java/net/pterodactylus/rhynodge/triggers/NewTorrentTrigger.java +++ b/src/main/java/net/pterodactylus/rhynodge/triggers/NewTorrentTrigger.java @@ -96,8 +96,12 @@ public class NewTorrentTrigger implements Trigger { plainText.append(torrentFile.name()).append('\n'); plainText.append('\t').append(torrentFile.size()).append(" in ").append(torrentFile.fileCount()).append(" file(s)\n"); plainText.append('\t').append(torrentFile.seedCount()).append(" seed(s), ").append(torrentFile.leechCount()).append(" leecher(s)\n"); - plainText.append('\t').append(torrentFile.magnetUri()).append('\n'); - plainText.append('\t').append(torrentFile.downloadUri()).append('\n'); + if ((torrentFile.magnetUri() != null) && (torrentFile.magnetUri().length() > 0)) { + plainText.append('\t').append(torrentFile.magnetUri()).append('\n'); + } + if ((torrentFile.downloadUri() != null) && (torrentFile.downloadUri().length() > 0)) { + plainText.append('\t').append(torrentFile.downloadUri()).append('\n'); + } plainText.append('\n'); } return plainText.toString(); @@ -119,8 +123,12 @@ public class NewTorrentTrigger implements Trigger { htmlText.append("
  • ").append(StringEscapeUtils.escapeHtml4(torrentFile.name())).append("
    • "); htmlText.append("
    Size: ").append(StringEscapeUtils.escapeHtml4(torrentFile.size())).append(" in ").append(torrentFile.fileCount()).append(" file(s)
    "); htmlText.append("
    ").append(torrentFile.seedCount()).append(" seed(s), ").append(torrentFile.leechCount()).append(" leecher(s)
    "); - htmlText.append(String.format("", StringEscapeUtils.escapeHtml4(torrentFile.magnetUri()))); - htmlText.append(String.format("", StringEscapeUtils.escapeHtml4(torrentFile.downloadUri()))); + if ((torrentFile.magnetUri() != null) && (torrentFile.magnetUri().length() > 0)) { + htmlText.append(String.format("", StringEscapeUtils.escapeHtml4(torrentFile.magnetUri()))); + } + if ((torrentFile.downloadUri() != null) && (torrentFile.downloadUri().length() > 0)) { + htmlText.append(String.format("", StringEscapeUtils.escapeHtml4(torrentFile.downloadUri()))); + } } htmlText.append("\n"); htmlText.append("\n"); diff --git a/src/main/resources/chains/thepiratebay-example.json b/src/main/resources/chains/thepiratebay-example.json new file mode 100644 index 0000000..ad79213 --- /dev/null +++ b/src/main/resources/chains/thepiratebay-example.json @@ -0,0 +1,50 @@ +{ + "enabled": false, + "name": "Example Reaction", + + "query": + { + "class": "HttpQuery", + "parameters": [ + { + "name": "url", + "value": "http://thepiratebay.se/search/Example%20Words/0/3/0" + } + ] + }, + + "filters": [ + { + "class": "HtmlFilter" + }, + { + "class": "PirateBayFilter" + } + ], + + "trigger": + { + "class": "NewTorrentTrigger" + }, + + "action": + { + "class": "EmailAction", + "parameters": [ + { + "name": "smtpHostname", + "value": "smtp" + }, + { + "name": "sender", + "value": "rhynodge@rhynodge.net" + }, + { + "name": "recipient", + "value": "recipient@recipient.de" + } + ] + }, + + "updateInterval": 3600 +}