src/main/java/net/pterodactylus/sone/core/SoneDownloaderImpl.java

   1 /*
   2  * Sone - SoneDownloader.java - Copyright © 2010–2013 David Roden
   3  *
   4  * This program is free software: you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.pterodactylus.sone.core;
  19
  20 import java.io.InputStream;
  21 import java.net.MalformedURLException;
  22 import java.util.ArrayList;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Set;
  26 import java.util.logging.Level;
  27 import java.util.logging.Logger;
  28
  29 import net.pterodactylus.sone.core.FreenetInterface.Fetched;
  30 import net.pterodactylus.sone.data.Album;
  31 import net.pterodactylus.sone.data.Client;
  32 import net.pterodactylus.sone.data.Image;
  33 import net.pterodactylus.sone.data.Post;
  34 import net.pterodactylus.sone.data.PostReply;
  35 import net.pterodactylus.sone.data.Profile;
  36 import net.pterodactylus.sone.data.Sone;
  37 import net.pterodactylus.sone.data.Sone.SoneStatus;
  38 import net.pterodactylus.sone.data.SoneImpl;
  39 import net.pterodactylus.sone.database.PostBuilder;
  40 import net.pterodactylus.sone.database.PostReplyBuilder;
  41 import net.pterodactylus.util.io.Closer;
  42 import net.pterodactylus.util.logging.Logging;
  43 import net.pterodactylus.util.number.Numbers;
  44 import net.pterodactylus.util.service.AbstractService;
  45 import net.pterodactylus.util.xml.SimpleXML;
  46 import net.pterodactylus.util.xml.XML;
  47
  48 import freenet.client.FetchResult;
  49 import freenet.keys.FreenetURI;
  50 import freenet.support.api.Bucket;
  51
  52 import com.google.common.annotations.VisibleForTesting;
  53 import org.w3c.dom.Document;
  54
  55 /**
  56  * The Sone downloader is responsible for download Sones as they are updated.
  57  *
  58  * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
  59  */
  60 public class SoneDownloaderImpl extends AbstractService implements SoneDownloader {
  61
  62         /** The logger. */
  63         private static final Logger logger = Logging.getLogger(SoneDownloaderImpl.class);
  64
  65         /** The maximum protocol version. */
  66         private static final int MAX_PROTOCOL_VERSION = 0;
  67
  68         /** The core. */
  69         private final Core core;
  70
  71         /** The Freenet interface. */
  72         private final FreenetInterface freenetInterface;
  73
  74         /** The sones to update. */
  75         private final Set<Sone> sones = new HashSet<Sone>();
  76
  77         /**
  78          * Creates a new Sone downloader.
  79          *
  80          * @param core
  81          *              The core
  82          * @param freenetInterface
  83          *              The Freenet interface
  84          */
  85         public SoneDownloaderImpl(Core core, FreenetInterface freenetInterface) {
  86                 super("Sone Downloader", false);
  87                 this.core = core;
  88                 this.freenetInterface = freenetInterface;
  89         }
  90
  91         //
  92         // ACTIONS
  93         //
  94
  95         /**
  96          * Adds the given Sone to the set of Sones that will be watched for updates.
  97          *
  98          * @param sone
  99          *              The Sone to add
 100          */
 101         @Override
 102         public void addSone(Sone sone) {
 103                 if (!sones.add(sone)) {
 104                         freenetInterface.unregisterUsk(sone);
 105                 }
 106                 freenetInterface.registerUsk(sone, this);
 107         }
 108
 109         /**
 110          * Fetches the updated Sone. This method is a callback method for
 111          * {@link FreenetInterface#registerUsk(Sone, SoneDownloader)}.
 112          *
 113          * @param sone
 114          *              The Sone to fetch
 115          */
 116         @Override
 117         public void fetchSone(Sone sone) {
 118                 fetchSone(sone, sone.getRequestUri().sskForUSK());
 119         }
 120
 121         /**
 122          * Fetches the updated Sone. This method can be used to fetch a Sone from a
 123          * specific URI.
 124          *
 125          * @param sone
 126          *              The Sone to fetch
 127          * @param soneUri
 128          *              The URI to fetch the Sone from
 129          */
 130         @Override
 131         public void fetchSone(Sone sone, FreenetURI soneUri) {
 132                 fetchSone(sone, soneUri, false);
 133         }
 134
 135         /**
 136          * Fetches the Sone from the given URI.
 137          *
 138          * @param sone
 139          *              The Sone to fetch
 140          * @param soneUri
 141          *              The URI of the Sone to fetch
 142          * @param fetchOnly
 143          *              {@code true} to only fetch and parse the Sone, {@code false}
 144          *              to {@link Core#updateSone(Sone) update} it in the core
 145          * @return The downloaded Sone, or {@code null} if the Sone could not be
 146          *         downloaded
 147          */
 148         @Override
 149         public Sone fetchSone(Sone sone, FreenetURI soneUri, boolean fetchOnly) {
 150                 logger.log(Level.FINE, String.format("Starting fetch for Sone “%s” from %s…", sone, soneUri));
 151                 FreenetURI requestUri = soneUri.setMetaString(new String[] { "sone.xml" });
 152                 sone.setStatus(SoneStatus.downloading);
 153                 try {
 154                         Fetched fetchResults = freenetInterface.fetchUri(requestUri);
 155                         if (fetchResults == null) {
 156                                 /* TODO - mark Sone as bad. */
 157                                 return null;
 158                         }
 159                         logger.log(Level.FINEST, String.format("Got %d bytes back.", fetchResults.getFetchResult().size()));
 160                         Sone parsedSone = parseSone(sone, fetchResults.getFetchResult(), fetchResults.getFreenetUri());
 161                         if (parsedSone != null) {
 162                                 if (!fetchOnly) {
 163                                         parsedSone.setStatus((parsedSone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
 164                                         core.updateSone(parsedSone);
 165                                         addSone(parsedSone);
 166                                 }
 167                         }
 168                         return parsedSone;
 169                 } finally {
 170                         sone.setStatus((sone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
 171                 }
 172         }
 173
 174         /**
 175          * Parses a Sone from a fetch result.
 176          *
 177          * @param originalSone
 178          *              The sone to parse, or {@code null} if the Sone is yet unknown
 179          * @param fetchResult
 180          *              The fetch result
 181          * @param requestUri
 182          *              The requested URI
 183          * @return The parsed Sone, or {@code null} if the Sone could not be parsed
 184          */
 185         private Sone parseSone(Sone originalSone, FetchResult fetchResult, FreenetURI requestUri) {
 186                 logger.log(Level.FINEST, String.format("Parsing FetchResult (%d bytes, %s) for %s…", fetchResult.size(), fetchResult.getMimeType(), originalSone));
 187                 Bucket soneBucket = fetchResult.asBucket();
 188                 InputStream soneInputStream = null;
 189                 try {
 190                         soneInputStream = soneBucket.getInputStream();
 191                         Sone parsedSone = parseSone(originalSone, soneInputStream);
 192                         if (parsedSone != null) {
 193                                 parsedSone.setLatestEdition(requestUri.getEdition());
 194                                 if (requestUri.getKeyType().equals("USK")) {
 195                                         parsedSone.setRequestUri(requestUri.setMetaString(new String[0]));
 196                                 } else {
 197                                         parsedSone.setRequestUri(requestUri.setKeyType("USK").setDocName("Sone").setMetaString(new String[0]));
 198                                 }
 199                         }
 200                         return parsedSone;
 201                 } catch (Exception e1) {
 202                         logger.log(Level.WARNING, String.format("Could not parse Sone from %s!", requestUri), e1);
 203                 } finally {
 204                         Closer.close(soneInputStream);
 205                         soneBucket.free();
 206                 }
 207                 return null;
 208         }
 209
 210         /**
 211          * Parses a Sone from the given input stream and creates a new Sone from the
 212          * parsed data.
 213          *
 214          * @param originalSone
 215          *              The Sone to update
 216          * @param soneInputStream
 217          *              The input stream to parse the Sone from
 218          * @return The parsed Sone
 219          * @throws SoneException
 220          *              if a parse error occurs, or the protocol is invalid
 221          */
 222         @VisibleForTesting
 223         protected Sone parseSone(Sone originalSone, InputStream soneInputStream) throws SoneException {
 224                 /* TODO - impose a size limit? */
 225
 226                 Document document;
 227                 /* XML parsing is not thread-safe. */
 228                 synchronized (this) {
 229                         document = XML.transformToDocument(soneInputStream);
 230                 }
 231                 if (document == null) {
 232                         /* TODO - mark Sone as bad. */
 233                         logger.log(Level.WARNING, String.format("Could not parse XML for Sone %s!", originalSone));
 234                         return null;
 235                 }
 236
 237                 Sone sone = new SoneImpl(originalSone.getId(), originalSone.isLocal()).setIdentity(originalSone.getIdentity());
 238
 239                 SimpleXML soneXml;
 240                 try {
 241                         soneXml = SimpleXML.fromDocument(document);
 242                 } catch (NullPointerException npe1) {
 243                         /* for some reason, invalid XML can cause NPEs. */
 244                         logger.log(Level.WARNING, String.format("XML for Sone %s can not be parsed!", sone), npe1);
 245                         return null;
 246                 }
 247
 248                 Integer protocolVersion = null;
 249                 String soneProtocolVersion = soneXml.getValue("protocol-version", null);
 250                 if (soneProtocolVersion != null) {
 251                         protocolVersion = Numbers.safeParseInteger(soneProtocolVersion);
 252                 }
 253                 if (protocolVersion == null) {
 254                         logger.log(Level.INFO, "No protocol version found, assuming 0.");
 255                         protocolVersion = 0;
 256                 }
 257
 258                 if (protocolVersion < 0) {
 259                         logger.log(Level.WARNING, String.format("Invalid protocol version: %d! Not parsing Sone.", protocolVersion));
 260                         return null;
 261                 }
 262
 263                 /* check for valid versions. */
 264                 if (protocolVersion > MAX_PROTOCOL_VERSION) {
 265                         logger.log(Level.WARNING, String.format("Unknown protocol version: %d! Not parsing Sone.", protocolVersion));
 266                         return null;
 267                 }
 268
 269                 String soneTime = soneXml.getValue("time", null);
 270                 if (soneTime == null) {
 271                         /* TODO - mark Sone as bad. */
 272                         logger.log(Level.WARNING, String.format("Downloaded time for Sone %s was null!", sone));
 273                         return null;
 274                 }
 275                 try {
 276                         sone.setTime(Long.parseLong(soneTime));
 277                 } catch (NumberFormatException nfe1) {
 278                         /* TODO - mark Sone as bad. */
 279                         logger.log(Level.WARNING, String.format("Downloaded Sone %s with invalid time: %s", sone, soneTime));
 280                         return null;
 281                 }
 282
 283                 SimpleXML clientXml = soneXml.getNode("client");
 284                 if (clientXml != null) {
 285                         String clientName = clientXml.getValue("name", null);
 286                         String clientVersion = clientXml.getValue("version", null);
 287                         if ((clientName == null) || (clientVersion == null)) {
 288                                 logger.log(Level.WARNING, String.format("Download Sone %s with client XML but missing name or version!", sone));
 289                                 return null;
 290                         }
 291                         sone.setClient(new Client(clientName, clientVersion));
 292                 }
 293
 294                 String soneRequestUri = soneXml.getValue("request-uri", null);
 295                 if (soneRequestUri != null) {
 296                         try {
 297                                 sone.setRequestUri(new FreenetURI(soneRequestUri));
 298                         } catch (MalformedURLException mue1) {
 299                                 /* TODO - mark Sone as bad. */
 300                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has invalid request URI: %s", sone, soneRequestUri), mue1);
 301                                 return null;
 302                         }
 303                 }
 304
 305                 if (originalSone.getInsertUri() != null) {
 306                         sone.setInsertUri(originalSone.getInsertUri());
 307                 }
 308
 309                 SimpleXML profileXml = soneXml.getNode("profile");
 310                 if (profileXml == null) {
 311                         /* TODO - mark Sone as bad. */
 312                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no profile!", sone));
 313                         return null;
 314                 }
 315
 316                 /* parse profile. */
 317                 String profileFirstName = profileXml.getValue("first-name", null);
 318                 String profileMiddleName = profileXml.getValue("middle-name", null);
 319                 String profileLastName = profileXml.getValue("last-name", null);
 320                 Integer profileBirthDay = Numbers.safeParseInteger(profileXml.getValue("birth-day", null));
 321                 Integer profileBirthMonth = Numbers.safeParseInteger(profileXml.getValue("birth-month", null));
 322                 Integer profileBirthYear = Numbers.safeParseInteger(profileXml.getValue("birth-year", null));
 323                 Profile profile = new Profile(sone).setFirstName(profileFirstName).setMiddleName(profileMiddleName).setLastName(profileLastName);
 324                 profile.setBirthDay(profileBirthDay).setBirthMonth(profileBirthMonth).setBirthYear(profileBirthYear);
 325                 /* avatar is processed after images are loaded. */
 326                 String avatarId = profileXml.getValue("avatar", null);
 327
 328                 /* parse profile fields. */
 329                 SimpleXML profileFieldsXml = profileXml.getNode("fields");
 330                 if (profileFieldsXml != null) {
 331                         for (SimpleXML fieldXml : profileFieldsXml.getNodes("field")) {
 332                                 String fieldName = fieldXml.getValue("field-name", null);
 333                                 String fieldValue = fieldXml.getValue("field-value", "");
 334                                 if (fieldName == null) {
 335                                         logger.log(Level.WARNING, String.format("Downloaded profile field for Sone %s with missing data! Name: %s, Value: %s", sone, fieldName, fieldValue));
 336                                         return null;
 337                                 }
 338                                 try {
 339                                         profile.addField(fieldName.trim()).setValue(fieldValue);
 340                                 } catch (IllegalArgumentException iae1) {
 341                                         logger.log(Level.WARNING, String.format("Duplicate field: %s", fieldName), iae1);
 342                                         return null;
 343                                 }
 344                         }
 345                 }
 346
 347                 /* parse posts. */
 348                 SimpleXML postsXml = soneXml.getNode("posts");
 349                 Set<Post> posts = new HashSet<Post>();
 350                 if (postsXml == null) {
 351                         /* TODO - mark Sone as bad. */
 352                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no posts!", sone));
 353                 } else {
 354                         for (SimpleXML postXml : postsXml.getNodes("post")) {
 355                                 String postId = postXml.getValue("id", null);
 356                                 String postRecipientId = postXml.getValue("recipient", null);
 357                                 String postTime = postXml.getValue("time", null);
 358                                 String postText = postXml.getValue("text", null);
 359                                 if ((postId == null) || (postTime == null) || (postText == null)) {
 360                                         /* TODO - mark Sone as bad. */
 361                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with missing data! ID: %s, Time: %s, Text: %s", sone, postId, postTime, postText));
 362                                         return null;
 363                                 }
 364                                 try {
 365                                         PostBuilder postBuilder = core.postBuilder();
 366                                         /* TODO - parse time correctly. */
 367                                         postBuilder.withId(postId).from(sone.getId()).withTime(Long.parseLong(postTime)).withText(postText);
 368                                         if ((postRecipientId != null) && (postRecipientId.length() == 43)) {
 369                                                 postBuilder.to(postRecipientId);
 370                                         }
 371                                         posts.add(postBuilder.build());
 372                                 } catch (NumberFormatException nfe1) {
 373                                         /* TODO - mark Sone as bad. */
 374                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with invalid time: %s", sone, postTime));
 375                                         return null;
 376                                 }
 377                         }
 378                 }
 379
 380                 /* parse replies. */
 381                 SimpleXML repliesXml = soneXml.getNode("replies");
 382                 Set<PostReply> replies = new HashSet<PostReply>();
 383                 if (repliesXml == null) {
 384                         /* TODO - mark Sone as bad. */
 385                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no replies!", sone));
 386                 } else {
 387                         for (SimpleXML replyXml : repliesXml.getNodes("reply")) {
 388                                 String replyId = replyXml.getValue("id", null);
 389                                 String replyPostId = replyXml.getValue("post-id", null);
 390                                 String replyTime = replyXml.getValue("time", null);
 391                                 String replyText = replyXml.getValue("text", null);
 392                                 if ((replyId == null) || (replyPostId == null) || (replyTime == null) || (replyText == null)) {
 393                                         /* TODO - mark Sone as bad. */
 394                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with missing data! ID: %s, Post: %s, Time: %s, Text: %s", sone, replyId, replyPostId, replyTime, replyText));
 395                                         return null;
 396                                 }
 397                                 try {
 398                                         PostReplyBuilder postReplyBuilder = core.postReplyBuilder();
 399                                         /* TODO - parse time correctly. */
 400                                         postReplyBuilder.withId(replyId).from(sone.getId()).to(replyPostId).withTime(Long.parseLong(replyTime)).withText(replyText);
 401                                         replies.add(postReplyBuilder.build());
 402                                 } catch (NumberFormatException nfe1) {
 403                                         /* TODO - mark Sone as bad. */
 404                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with invalid time: %s", sone, replyTime));
 405                                         return null;
 406                                 }
 407                         }
 408                 }
 409
 410                 /* parse liked post IDs. */
 411                 SimpleXML likePostIdsXml = soneXml.getNode("post-likes");
 412                 Set<String> likedPostIds = new HashSet<String>();
 413                 if (likePostIdsXml == null) {
 414                         /* TODO - mark Sone as bad. */
 415                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no post likes!", sone));
 416                 } else {
 417                         for (SimpleXML likedPostIdXml : likePostIdsXml.getNodes("post-like")) {
 418                                 String postId = likedPostIdXml.getValue();
 419                                 likedPostIds.add(postId);
 420                         }
 421                 }
 422
 423                 /* parse liked reply IDs. */
 424                 SimpleXML likeReplyIdsXml = soneXml.getNode("reply-likes");
 425                 Set<String> likedReplyIds = new HashSet<String>();
 426                 if (likeReplyIdsXml == null) {
 427                         /* TODO - mark Sone as bad. */
 428                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no reply likes!", sone));
 429                 } else {
 430                         for (SimpleXML likedReplyIdXml : likeReplyIdsXml.getNodes("reply-like")) {
 431                                 String replyId = likedReplyIdXml.getValue();
 432                                 likedReplyIds.add(replyId);
 433                         }
 434                 }
 435
 436                 /* parse albums. */
 437                 SimpleXML albumsXml = soneXml.getNode("albums");
 438                 List<Album> topLevelAlbums = new ArrayList<Album>();
 439                 if (albumsXml != null) {
 440                         for (SimpleXML albumXml : albumsXml.getNodes("album")) {
 441                                 String id = albumXml.getValue("id", null);
 442                                 String parentId = albumXml.getValue("parent", null);
 443                                 String title = albumXml.getValue("title", null);
 444                                 String description = albumXml.getValue("description", "");
 445                                 String albumImageId = albumXml.getValue("album-image", null);
 446                                 if ((id == null) || (title == null)) {
 447                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid album!", sone));
 448                                         return null;
 449                                 }
 450                                 Album parent = null;
 451                                 if (parentId != null) {
 452                                         parent = core.getAlbum(parentId, false);
 453                                         if (parent == null) {
 454                                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has album with invalid parent!", sone));
 455                                                 return null;
 456                                         }
 457                                 }
 458                                 Album album = core.getOrCreateAlbum(id).setSone(sone).modify().setTitle(title).setDescription(description).update();
 459                                 if (parent != null) {
 460                                         parent.addAlbum(album);
 461                                 } else {
 462                                         topLevelAlbums.add(album);
 463                                 }
 464                                 SimpleXML imagesXml = albumXml.getNode("images");
 465                                 if (imagesXml != null) {
 466                                         for (SimpleXML imageXml : imagesXml.getNodes("image")) {
 467                                                 String imageId = imageXml.getValue("id", null);
 468                                                 String imageCreationTimeString = imageXml.getValue("creation-time", null);
 469                                                 String imageKey = imageXml.getValue("key", null);
 470                                                 String imageTitle = imageXml.getValue("title", null);
 471                                                 String imageDescription = imageXml.getValue("description", "");
 472                                                 String imageWidthString = imageXml.getValue("width", null);
 473                                                 String imageHeightString = imageXml.getValue("height", null);
 474                                                 if ((imageId == null) || (imageCreationTimeString == null) || (imageKey == null) || (imageTitle == null) || (imageWidthString == null) || (imageHeightString == null)) {
 475                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid images!", sone));
 476                                                         return null;
 477                                                 }
 478                                                 long creationTime = Numbers.safeParseLong(imageCreationTimeString, 0L);
 479                                                 int imageWidth = Numbers.safeParseInteger(imageWidthString, 0);
 480                                                 int imageHeight = Numbers.safeParseInteger(imageHeightString, 0);
 481                                                 if ((imageWidth < 1) || (imageHeight < 1)) {
 482                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains image %s with invalid dimensions (%s, %s)!", sone, imageId, imageWidthString, imageHeightString));
 483                                                         return null;
 484                                                 }
 485                                                 Image image = core.getImage(imageId).modify().setSone(sone).setKey(imageKey).setCreationTime(creationTime).update();
 486                                                 image = image.modify().setTitle(imageTitle).setDescription(imageDescription).update();
 487                                                 image = image.modify().setWidth(imageWidth).setHeight(imageHeight).update();
 488                                                 album.addImage(image);
 489                                         }
 490                                 }
 491                                 album.modify().setAlbumImage(albumImageId).update();
 492                         }
 493                 }
 494
 495                 /* process avatar. */
 496                 if (avatarId != null) {
 497                         profile.setAvatar(core.getImage(avatarId, false));
 498                 }
 499
 500                 /* okay, apparently everything was parsed correctly. Now import. */
 501                 /* atomic setter operation on the Sone. */
 502                 synchronized (sone) {
 503                         sone.setProfile(profile);
 504                         sone.setPosts(posts);
 505                         sone.setReplies(replies);
 506                         sone.setLikePostIds(likedPostIds);
 507                         sone.setLikeReplyIds(likedReplyIds);
 508                         for (Album album : topLevelAlbums) {
 509                                 sone.getRootAlbum().addAlbum(album);
 510                         }
 511                 }
 512
 513                 return sone;
 514         }
 515
 516         @Override
 517         public Runnable fetchSoneWithUriAction(final Sone sone) {
 518                 return new Runnable() {
 519                         @Override
 520                         public void run() {
 521                                 fetchSone(sone, sone.getRequestUri());
 522                         }
 523                 };
 524         }
 525
 526         @Override
 527         public Runnable fetchSoneAction(final Sone sone) {
 528                 return new Runnable() {
 529                         @Override
 530                         public void run() {
 531                                 fetchSone(sone);
 532                         }
 533                 };
 534         }
 535
 536         /** {@inheritDoc} */
 537         @Override
 538         protected void serviceStop() {
 539                 for (Sone sone : sones) {
 540                         freenetInterface.unregisterUsk(sone);
 541                 }
 542         }
 543
 544 }