Implement hashCode() and equals().
[Sone.git] / src / main / java / net / pterodactylus / sone / core / SoneDownloader.java
1 /*
2  * Sone - SoneDownloader.java - Copyright © 2010–2013 David Roden
3  *
4  * This program is free software: you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16  */
17
18 package net.pterodactylus.sone.core;
19
20 import java.io.InputStream;
21 import java.net.MalformedURLException;
22 import java.util.ArrayList;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Set;
26 import java.util.logging.Level;
27 import java.util.logging.Logger;
28
29 import net.pterodactylus.sone.core.FreenetInterface.Fetched;
30 import net.pterodactylus.sone.data.Album;
31 import net.pterodactylus.sone.data.Client;
32 import net.pterodactylus.sone.data.Image;
33 import net.pterodactylus.sone.data.Post;
34 import net.pterodactylus.sone.data.PostReply;
35 import net.pterodactylus.sone.data.Profile;
36 import net.pterodactylus.sone.data.Sone;
37 import net.pterodactylus.sone.data.Sone.SoneStatus;
38 import net.pterodactylus.sone.data.SoneImpl;
39 import net.pterodactylus.sone.database.PostBuilder;
40 import net.pterodactylus.sone.database.PostReplyBuilder;
41 import net.pterodactylus.util.io.Closer;
42 import net.pterodactylus.util.logging.Logging;
43 import net.pterodactylus.util.number.Numbers;
44 import net.pterodactylus.util.service.AbstractService;
45 import net.pterodactylus.util.xml.SimpleXML;
46 import net.pterodactylus.util.xml.XML;
47
48 import org.w3c.dom.Document;
49
50 import freenet.client.FetchResult;
51 import freenet.keys.FreenetURI;
52 import freenet.support.api.Bucket;
53
54 /**
55  * The Sone downloader is responsible for download Sones as they are updated.
56  *
57  * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
58  */
59 public class SoneDownloader extends AbstractService {
60
61         /** The logger. */
62         private static final Logger logger = Logging.getLogger(SoneDownloader.class);
63
64         /** The maximum protocol version. */
65         private static final int MAX_PROTOCOL_VERSION = 0;
66
67         /** The core. */
68         private final Core core;
69
70         /** The Freenet interface. */
71         private final FreenetInterface freenetInterface;
72
73         /** The sones to update. */
74         private final Set<Sone> sones = new HashSet<Sone>();
75
76         /**
77          * Creates a new Sone downloader.
78          *
79          * @param core
80          *            The core
81          * @param freenetInterface
82          *            The Freenet interface
83          */
84         public SoneDownloader(Core core, FreenetInterface freenetInterface) {
85                 super("Sone Downloader", false);
86                 this.core = core;
87                 this.freenetInterface = freenetInterface;
88         }
89
90         //
91         // ACTIONS
92         //
93
94         /**
95          * Adds the given Sone to the set of Sones that will be watched for updates.
96          *
97          * @param sone
98          *            The Sone to add
99          */
100         public void addSone(Sone sone) {
101                 if (!sones.add(sone)) {
102                         freenetInterface.unregisterUsk(sone);
103                 }
104                 freenetInterface.registerUsk(sone, this);
105         }
106
107         /**
108          * Removes the given Sone from the downloader.
109          *
110          * @param sone
111          *            The Sone to stop watching
112          */
113         public void removeSone(Sone sone) {
114                 if (sones.remove(sone)) {
115                         freenetInterface.unregisterUsk(sone);
116                 }
117         }
118
119         /**
120          * Fetches the updated Sone. This method is a callback method for
121          * {@link FreenetInterface#registerUsk(Sone, SoneDownloader)}.
122          *
123          * @param sone
124          *            The Sone to fetch
125          */
126         public void fetchSone(Sone sone) {
127                 fetchSone(sone, sone.getRequestUri().sskForUSK());
128         }
129
130         /**
131          * Fetches the updated Sone. This method can be used to fetch a Sone from a
132          * specific URI.
133          *
134          * @param sone
135          *            The Sone to fetch
136          * @param soneUri
137          *            The URI to fetch the Sone from
138          */
139         public void fetchSone(Sone sone, FreenetURI soneUri) {
140                 fetchSone(sone, soneUri, false);
141         }
142
143         /**
144          * Fetches the Sone from the given URI.
145          *
146          * @param sone
147          *            The Sone to fetch
148          * @param soneUri
149          *            The URI of the Sone to fetch
150          * @param fetchOnly
151          *            {@code true} to only fetch and parse the Sone, {@code false}
152          *            to {@link Core#updateSone(Sone) update} it in the core
153          * @return The downloaded Sone, or {@code null} if the Sone could not be
154          *         downloaded
155          */
156         public Sone fetchSone(Sone sone, FreenetURI soneUri, boolean fetchOnly) {
157                 logger.log(Level.FINE, String.format("Starting fetch for Sone “%s” from %s…", sone, soneUri));
158                 FreenetURI requestUri = soneUri.setMetaString(new String[] { "sone.xml" });
159                 sone.setStatus(SoneStatus.downloading);
160                 try {
161                         Fetched fetchResults = freenetInterface.fetchUri(requestUri);
162                         if (fetchResults == null) {
163                                 /* TODO - mark Sone as bad. */
164                                 return null;
165                         }
166                         logger.log(Level.FINEST, String.format("Got %d bytes back.", fetchResults.getFetchResult().size()));
167                         Sone parsedSone = parseSone(sone, fetchResults.getFetchResult(), fetchResults.getFreenetUri());
168                         if (parsedSone != null) {
169                                 if (!fetchOnly) {
170                                         parsedSone.setStatus((parsedSone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
171                                         core.updateSone(parsedSone);
172                                         addSone(parsedSone);
173                                 }
174                         }
175                         return parsedSone;
176                 } finally {
177                         sone.setStatus((sone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
178                 }
179         }
180
181         /**
182          * Parses a Sone from a fetch result.
183          *
184          * @param originalSone
185          *            The sone to parse, or {@code null} if the Sone is yet unknown
186          * @param fetchResult
187          *            The fetch result
188          * @param requestUri
189          *            The requested URI
190          * @return The parsed Sone, or {@code null} if the Sone could not be parsed
191          */
192         public Sone parseSone(Sone originalSone, FetchResult fetchResult, FreenetURI requestUri) {
193                 logger.log(Level.FINEST, String.format("Parsing FetchResult (%d bytes, %s) for %s…", fetchResult.size(), fetchResult.getMimeType(), originalSone));
194                 Bucket soneBucket = fetchResult.asBucket();
195                 InputStream soneInputStream = null;
196                 try {
197                         soneInputStream = soneBucket.getInputStream();
198                         Sone parsedSone = parseSone(originalSone, soneInputStream);
199                         if (parsedSone != null) {
200                                 parsedSone.setLatestEdition(requestUri.getEdition());
201                                 if (requestUri.getKeyType().equals("USK")) {
202                                         parsedSone.setRequestUri(requestUri.setMetaString(new String[0]));
203                                 } else {
204                                         parsedSone.setRequestUri(requestUri.setKeyType("USK").setDocName("Sone").setMetaString(new String[0]));
205                                 }
206                         }
207                         return parsedSone;
208                 } catch (Exception e1) {
209                         logger.log(Level.WARNING, String.format("Could not parse Sone from %s!", requestUri), e1);
210                 } finally {
211                         Closer.close(soneInputStream);
212                         soneBucket.free();
213                 }
214                 return null;
215         }
216
217         /**
218          * Parses a Sone from the given input stream and creates a new Sone from the
219          * parsed data.
220          *
221          * @param originalSone
222          *            The Sone to update
223          * @param soneInputStream
224          *            The input stream to parse the Sone from
225          * @return The parsed Sone
226          * @throws SoneException
227          *             if a parse error occurs, or the protocol is invalid
228          */
229         public Sone parseSone(Sone originalSone, InputStream soneInputStream) throws SoneException {
230                 /* TODO - impose a size limit? */
231
232                 Document document;
233                 /* XML parsing is not thread-safe. */
234                 synchronized (this) {
235                         document = XML.transformToDocument(soneInputStream);
236                 }
237                 if (document == null) {
238                         /* TODO - mark Sone as bad. */
239                         logger.log(Level.WARNING, String.format("Could not parse XML for Sone %s!", originalSone));
240                         return null;
241                 }
242
243                 Sone sone = new SoneImpl(originalSone.getId(), originalSone.isLocal()).setIdentity(originalSone.getIdentity());
244
245                 SimpleXML soneXml;
246                 try {
247                         soneXml = SimpleXML.fromDocument(document);
248                 } catch (NullPointerException npe1) {
249                         /* for some reason, invalid XML can cause NPEs. */
250                         logger.log(Level.WARNING, String.format("XML for Sone %s can not be parsed!", sone), npe1);
251                         return null;
252                 }
253
254                 Integer protocolVersion = null;
255                 String soneProtocolVersion = soneXml.getValue("protocol-version", null);
256                 if (soneProtocolVersion != null) {
257                         protocolVersion = Numbers.safeParseInteger(soneProtocolVersion);
258                 }
259                 if (protocolVersion == null) {
260                         logger.log(Level.INFO, "No protocol version found, assuming 0.");
261                         protocolVersion = 0;
262                 }
263
264                 if (protocolVersion < 0) {
265                         logger.log(Level.WARNING, String.format("Invalid protocol version: %d! Not parsing Sone.", protocolVersion));
266                         return null;
267                 }
268
269                 /* check for valid versions. */
270                 if (protocolVersion > MAX_PROTOCOL_VERSION) {
271                         logger.log(Level.WARNING, String.format("Unknown protocol version: %d! Not parsing Sone.", protocolVersion));
272                         return null;
273                 }
274
275                 String soneTime = soneXml.getValue("time", null);
276                 if (soneTime == null) {
277                         /* TODO - mark Sone as bad. */
278                         logger.log(Level.WARNING, String.format("Downloaded time for Sone %s was null!", sone));
279                         return null;
280                 }
281                 try {
282                         sone.setTime(Long.parseLong(soneTime));
283                 } catch (NumberFormatException nfe1) {
284                         /* TODO - mark Sone as bad. */
285                         logger.log(Level.WARNING, String.format("Downloaded Sone %s with invalid time: %s", sone, soneTime));
286                         return null;
287                 }
288
289                 SimpleXML clientXml = soneXml.getNode("client");
290                 if (clientXml != null) {
291                         String clientName = clientXml.getValue("name", null);
292                         String clientVersion = clientXml.getValue("version", null);
293                         if ((clientName == null) || (clientVersion == null)) {
294                                 logger.log(Level.WARNING, String.format("Download Sone %s with client XML but missing name or version!", sone));
295                                 return null;
296                         }
297                         sone.setClient(new Client(clientName, clientVersion));
298                 }
299
300                 String soneRequestUri = soneXml.getValue("request-uri", null);
301                 if (soneRequestUri != null) {
302                         try {
303                                 sone.setRequestUri(new FreenetURI(soneRequestUri));
304                         } catch (MalformedURLException mue1) {
305                                 /* TODO - mark Sone as bad. */
306                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has invalid request URI: %s", sone, soneRequestUri), mue1);
307                                 return null;
308                         }
309                 }
310
311                 if (originalSone.getInsertUri() != null) {
312                         sone.setInsertUri(originalSone.getInsertUri());
313                 }
314
315                 SimpleXML profileXml = soneXml.getNode("profile");
316                 if (profileXml == null) {
317                         /* TODO - mark Sone as bad. */
318                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no profile!", sone));
319                         return null;
320                 }
321
322                 /* parse profile. */
323                 String profileFirstName = profileXml.getValue("first-name", null);
324                 String profileMiddleName = profileXml.getValue("middle-name", null);
325                 String profileLastName = profileXml.getValue("last-name", null);
326                 Integer profileBirthDay = Numbers.safeParseInteger(profileXml.getValue("birth-day", null));
327                 Integer profileBirthMonth = Numbers.safeParseInteger(profileXml.getValue("birth-month", null));
328                 Integer profileBirthYear = Numbers.safeParseInteger(profileXml.getValue("birth-year", null));
329                 Profile profile = new Profile(sone).setFirstName(profileFirstName).setMiddleName(profileMiddleName).setLastName(profileLastName);
330                 profile.setBirthDay(profileBirthDay).setBirthMonth(profileBirthMonth).setBirthYear(profileBirthYear);
331                 /* avatar is processed after images are loaded. */
332                 String avatarId = profileXml.getValue("avatar", null);
333
334                 /* parse profile fields. */
335                 SimpleXML profileFieldsXml = profileXml.getNode("fields");
336                 if (profileFieldsXml != null) {
337                         for (SimpleXML fieldXml : profileFieldsXml.getNodes("field")) {
338                                 String fieldName = fieldXml.getValue("field-name", null);
339                                 String fieldValue = fieldXml.getValue("field-value", "");
340                                 if (fieldName == null) {
341                                         logger.log(Level.WARNING, String.format("Downloaded profile field for Sone %s with missing data! Name: %s, Value: %s", sone, fieldName, fieldValue));
342                                         return null;
343                                 }
344                                 try {
345                                         profile.addField(fieldName).setValue(fieldValue);
346                                 } catch (IllegalArgumentException iae1) {
347                                         logger.log(Level.WARNING, String.format("Duplicate field: %s", fieldName), iae1);
348                                         return null;
349                                 }
350                         }
351                 }
352
353                 /* parse posts. */
354                 SimpleXML postsXml = soneXml.getNode("posts");
355                 Set<Post> posts = new HashSet<Post>();
356                 if (postsXml == null) {
357                         /* TODO - mark Sone as bad. */
358                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no posts!", sone));
359                 } else {
360                         for (SimpleXML postXml : postsXml.getNodes("post")) {
361                                 String postId = postXml.getValue("id", null);
362                                 String postRecipientId = postXml.getValue("recipient", null);
363                                 String postTime = postXml.getValue("time", null);
364                                 String postText = postXml.getValue("text", null);
365                                 if ((postId == null) || (postTime == null) || (postText == null)) {
366                                         /* TODO - mark Sone as bad. */
367                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with missing data! ID: %s, Time: %s, Text: %s", sone, postId, postTime, postText));
368                                         return null;
369                                 }
370                                 try {
371                                         PostBuilder postBuilder = core.postBuilder();
372                                         /* TODO - parse time correctly. */
373                                         postBuilder.withId(postId).from(sone.getId()).withTime(Long.parseLong(postTime)).withText(postText);
374                                         if ((postRecipientId != null) && (postRecipientId.length() == 43)) {
375                                                 postBuilder.to(postRecipientId);
376                                         }
377                                         posts.add(postBuilder.build());
378                                 } catch (NumberFormatException nfe1) {
379                                         /* TODO - mark Sone as bad. */
380                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with invalid time: %s", sone, postTime));
381                                         return null;
382                                 }
383                         }
384                 }
385
386                 /* parse replies. */
387                 SimpleXML repliesXml = soneXml.getNode("replies");
388                 Set<PostReply> replies = new HashSet<PostReply>();
389                 if (repliesXml == null) {
390                         /* TODO - mark Sone as bad. */
391                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no replies!", sone));
392                 } else {
393                         for (SimpleXML replyXml : repliesXml.getNodes("reply")) {
394                                 String replyId = replyXml.getValue("id", null);
395                                 String replyPostId = replyXml.getValue("post-id", null);
396                                 String replyTime = replyXml.getValue("time", null);
397                                 String replyText = replyXml.getValue("text", null);
398                                 if ((replyId == null) || (replyPostId == null) || (replyTime == null) || (replyText == null)) {
399                                         /* TODO - mark Sone as bad. */
400                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with missing data! ID: %s, Post: %s, Time: %s, Text: %s", sone, replyId, replyPostId, replyTime, replyText));
401                                         return null;
402                                 }
403                                 try {
404                                         PostReplyBuilder postReplyBuilder = core.postReplyBuilder();
405                                         /* TODO - parse time correctly. */
406                                         postReplyBuilder.withId(replyId).from(sone.getId()).to(replyPostId).withTime(Long.parseLong(replyTime)).withText(replyText);
407                                         replies.add(postReplyBuilder.build());
408                                 } catch (NumberFormatException nfe1) {
409                                         /* TODO - mark Sone as bad. */
410                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with invalid time: %s", sone, replyTime));
411                                         return null;
412                                 }
413                         }
414                 }
415
416                 /* parse liked post IDs. */
417                 SimpleXML likePostIdsXml = soneXml.getNode("post-likes");
418                 Set<String> likedPostIds = new HashSet<String>();
419                 if (likePostIdsXml == null) {
420                         /* TODO - mark Sone as bad. */
421                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no post likes!", sone));
422                 } else {
423                         for (SimpleXML likedPostIdXml : likePostIdsXml.getNodes("post-like")) {
424                                 String postId = likedPostIdXml.getValue();
425                                 likedPostIds.add(postId);
426                         }
427                 }
428
429                 /* parse liked reply IDs. */
430                 SimpleXML likeReplyIdsXml = soneXml.getNode("reply-likes");
431                 Set<String> likedReplyIds = new HashSet<String>();
432                 if (likeReplyIdsXml == null) {
433                         /* TODO - mark Sone as bad. */
434                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no reply likes!", sone));
435                 } else {
436                         for (SimpleXML likedReplyIdXml : likeReplyIdsXml.getNodes("reply-like")) {
437                                 String replyId = likedReplyIdXml.getValue();
438                                 likedReplyIds.add(replyId);
439                         }
440                 }
441
442                 /* parse albums. */
443                 SimpleXML albumsXml = soneXml.getNode("albums");
444                 List<Album> topLevelAlbums = new ArrayList<Album>();
445                 if (albumsXml != null) {
446                         for (SimpleXML albumXml : albumsXml.getNodes("album")) {
447                                 String id = albumXml.getValue("id", null);
448                                 String parentId = albumXml.getValue("parent", null);
449                                 String title = albumXml.getValue("title", null);
450                                 String description = albumXml.getValue("description", "");
451                                 String albumImageId = albumXml.getValue("album-image", null);
452                                 if ((id == null) || (title == null) || (description == null)) {
453                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid album!", sone));
454                                         return null;
455                                 }
456                                 Album parent = null;
457                                 if (parentId != null) {
458                                         parent = core.getAlbum(parentId, false);
459                                         if (parent == null) {
460                                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has album with invalid parent!", sone));
461                                                 return null;
462                                         }
463                                 }
464                                 Album album = core.getAlbum(id).setSone(sone).modify().setTitle(title).setDescription(description).update();
465                                 if (parent != null) {
466                                         parent.addAlbum(album);
467                                 } else {
468                                         topLevelAlbums.add(album);
469                                 }
470                                 SimpleXML imagesXml = albumXml.getNode("images");
471                                 if (imagesXml != null) {
472                                         for (SimpleXML imageXml : imagesXml.getNodes("image")) {
473                                                 String imageId = imageXml.getValue("id", null);
474                                                 String imageCreationTimeString = imageXml.getValue("creation-time", null);
475                                                 String imageKey = imageXml.getValue("key", null);
476                                                 String imageTitle = imageXml.getValue("title", null);
477                                                 String imageDescription = imageXml.getValue("description", "");
478                                                 String imageWidthString = imageXml.getValue("width", null);
479                                                 String imageHeightString = imageXml.getValue("height", null);
480                                                 if ((imageId == null) || (imageCreationTimeString == null) || (imageKey == null) || (imageTitle == null) || (imageWidthString == null) || (imageHeightString == null)) {
481                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid images!", sone));
482                                                         return null;
483                                                 }
484                                                 long creationTime = Numbers.safeParseLong(imageCreationTimeString, 0L);
485                                                 int imageWidth = Numbers.safeParseInteger(imageWidthString, 0);
486                                                 int imageHeight = Numbers.safeParseInteger(imageHeightString, 0);
487                                                 if ((imageWidth < 1) || (imageHeight < 1)) {
488                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains image %s with invalid dimensions (%s, %s)!", sone, imageId, imageWidthString, imageHeightString));
489                                                         return null;
490                                                 }
491                                                 Image image = core.getImage(imageId).modify().setSone(sone).setKey(imageKey).setCreationTime(creationTime).update();
492                                                 image = image.modify().setTitle(imageTitle).setDescription(imageDescription).update();
493                                                 image = image.modify().setWidth(imageWidth).setHeight(imageHeight).update();
494                                                 album.addImage(image);
495                                         }
496                                 }
497                                 album.modify().setAlbumImage(albumImageId).update();
498                         }
499                 }
500
501                 /* process avatar. */
502                 if (avatarId != null) {
503                         profile.setAvatar(core.getImage(avatarId, false));
504                 }
505
506                 /* okay, apparently everything was parsed correctly. Now import. */
507                 /* atomic setter operation on the Sone. */
508                 synchronized (sone) {
509                         sone.setProfile(profile);
510                         sone.setPosts(posts);
511                         sone.setReplies(replies);
512                         sone.setLikePostIds(likedPostIds);
513                         sone.setLikeReplyIds(likedReplyIds);
514                         for (Album album : topLevelAlbums) {
515                                 sone.getRootAlbum().addAlbum(album);
516                         }
517                 }
518
519                 return sone;
520         }
521
522         //
523         // SERVICE METHODS
524         //
525
526         /**
527          * {@inheritDoc}
528          */
529         @Override
530         protected void serviceStop() {
531                 for (Sone sone : sones) {
532                         freenetInterface.unregisterUsk(sone);
533                 }
534         }
535
536 }