Set version to 0.8.3.
[Sone.git] / src / main / java / net / pterodactylus / sone / core / SoneDownloader.java
1 /*
2  * Sone - SoneDownloader.java - Copyright © 2010–2012 David Roden
3  *
4  * This program is free software: you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16  */
17
18 package net.pterodactylus.sone.core;
19
20 import java.io.InputStream;
21 import java.net.MalformedURLException;
22 import java.util.ArrayList;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Set;
26 import java.util.logging.Level;
27 import java.util.logging.Logger;
28
29 import net.pterodactylus.sone.data.Album;
30 import net.pterodactylus.sone.data.Client;
31 import net.pterodactylus.sone.data.Image;
32 import net.pterodactylus.sone.data.Post;
33 import net.pterodactylus.sone.data.PostReply;
34 import net.pterodactylus.sone.data.Profile;
35 import net.pterodactylus.sone.data.Sone;
36 import net.pterodactylus.sone.data.Sone.SoneStatus;
37 import net.pterodactylus.util.collection.Pair;
38 import net.pterodactylus.util.io.Closer;
39 import net.pterodactylus.util.logging.Logging;
40 import net.pterodactylus.util.number.Numbers;
41 import net.pterodactylus.util.service.AbstractService;
42 import net.pterodactylus.util.xml.SimpleXML;
43 import net.pterodactylus.util.xml.XML;
44
45 import org.w3c.dom.Document;
46
47 import freenet.client.FetchResult;
48 import freenet.keys.FreenetURI;
49 import freenet.support.api.Bucket;
50
51 /**
52  * The Sone downloader is responsible for download Sones as they are updated.
53  *
54  * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
55  */
56 public class SoneDownloader extends AbstractService {
57
58         /** The logger. */
59         private static final Logger logger = Logging.getLogger(SoneDownloader.class);
60
61         /** The maximum protocol version. */
62         private static final int MAX_PROTOCOL_VERSION = 0;
63
64         /** The core. */
65         private final Core core;
66
67         /** The Freenet interface. */
68         private final FreenetInterface freenetInterface;
69
70         /** The sones to update. */
71         private final Set<Sone> sones = new HashSet<Sone>();
72
73         /**
74          * Creates a new Sone downloader.
75          *
76          * @param core
77          *            The core
78          * @param freenetInterface
79          *            The Freenet interface
80          */
81         public SoneDownloader(Core core, FreenetInterface freenetInterface) {
82                 super("Sone Downloader", false);
83                 this.core = core;
84                 this.freenetInterface = freenetInterface;
85         }
86
87         //
88         // ACTIONS
89         //
90
91         /**
92          * Adds the given Sone to the set of Sones that will be watched for updates.
93          *
94          * @param sone
95          *            The Sone to add
96          */
97         public void addSone(Sone sone) {
98                 if (!sones.add(sone)) {
99                         freenetInterface.unregisterUsk(sone);
100                 }
101                 freenetInterface.registerUsk(sone, this);
102         }
103
104         /**
105          * Removes the given Sone from the downloader.
106          *
107          * @param sone
108          *            The Sone to stop watching
109          */
110         public void removeSone(Sone sone) {
111                 if (sones.remove(sone)) {
112                         freenetInterface.unregisterUsk(sone);
113                 }
114         }
115
116         /**
117          * Fetches the updated Sone. This method is a callback method for
118          * {@link FreenetInterface#registerUsk(Sone, SoneDownloader)}.
119          *
120          * @param sone
121          *            The Sone to fetch
122          */
123         public void fetchSone(Sone sone) {
124                 fetchSone(sone, sone.getRequestUri().sskForUSK());
125         }
126
127         /**
128          * Fetches the updated Sone. This method can be used to fetch a Sone from a
129          * specific URI.
130          *
131          * @param sone
132          *            The Sone to fetch
133          * @param soneUri
134          *            The URI to fetch the Sone from
135          */
136         public void fetchSone(Sone sone, FreenetURI soneUri) {
137                 fetchSone(sone, soneUri, false);
138         }
139
140         /**
141          * Fetches the Sone from the given URI.
142          *
143          * @param sone
144          *            The Sone to fetch
145          * @param soneUri
146          *            The URI of the Sone to fetch
147          * @param fetchOnly
148          *            {@code true} to only fetch and parse the Sone, {@code false}
149          *            to {@link Core#updateSone(Sone) update} it in the core
150          * @return The downloaded Sone, or {@code null} if the Sone could not be
151          *         downloaded
152          */
153         public Sone fetchSone(Sone sone, FreenetURI soneUri, boolean fetchOnly) {
154                 logger.log(Level.FINE, String.format("Starting fetch for Sone “%s” from %s…", sone, soneUri));
155                 FreenetURI requestUri = soneUri.setMetaString(new String[] { "sone.xml" });
156                 sone.setStatus(SoneStatus.downloading);
157                 try {
158                         Pair<FreenetURI, FetchResult> fetchResults = freenetInterface.fetchUri(requestUri);
159                         if (fetchResults == null) {
160                                 /* TODO - mark Sone as bad. */
161                                 return null;
162                         }
163                         logger.log(Level.FINEST, String.format("Got %d bytes back.", fetchResults.getRight().size()));
164                         Sone parsedSone = parseSone(sone, fetchResults.getRight(), fetchResults.getLeft());
165                         if (parsedSone != null) {
166                                 if (!fetchOnly) {
167                                         core.updateSone(parsedSone);
168                                         addSone(parsedSone);
169                                 }
170                         }
171                         return parsedSone;
172                 } finally {
173                         sone.setStatus((sone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
174                 }
175         }
176
177         /**
178          * Parses a Sone from a fetch result.
179          *
180          * @param originalSone
181          *            The sone to parse, or {@code null} if the Sone is yet unknown
182          * @param fetchResult
183          *            The fetch result
184          * @param requestUri
185          *            The requested URI
186          * @return The parsed Sone, or {@code null} if the Sone could not be parsed
187          */
188         public Sone parseSone(Sone originalSone, FetchResult fetchResult, FreenetURI requestUri) {
189                 logger.log(Level.FINEST, String.format("Parsing FetchResult (%d bytes, %s) for %s…", fetchResult.size(), fetchResult.getMimeType(), originalSone));
190                 Bucket soneBucket = fetchResult.asBucket();
191                 InputStream soneInputStream = null;
192                 try {
193                         soneInputStream = soneBucket.getInputStream();
194                         Sone parsedSone = parseSone(originalSone, soneInputStream);
195                         if (parsedSone != null) {
196                                 parsedSone.setLatestEdition(requestUri.getEdition());
197                                 if (requestUri.getKeyType().equals("USK")) {
198                                         parsedSone.setRequestUri(requestUri.setMetaString(new String[0]));
199                                 } else {
200                                         parsedSone.setRequestUri(requestUri.setKeyType("USK").setDocName("Sone").setMetaString(new String[0]));
201                                 }
202                         }
203                         return parsedSone;
204                 } catch (Exception e1) {
205                         logger.log(Level.WARNING, String.format("Could not parse Sone from %s!", requestUri), e1);
206                 } finally {
207                         Closer.close(soneInputStream);
208                         soneBucket.free();
209                 }
210                 return null;
211         }
212
213         /**
214          * Parses a Sone from the given input stream and creates a new Sone from the
215          * parsed data.
216          *
217          * @param originalSone
218          *            The Sone to update
219          * @param soneInputStream
220          *            The input stream to parse the Sone from
221          * @return The parsed Sone
222          * @throws SoneException
223          *             if a parse error occurs, or the protocol is invalid
224          */
225         public Sone parseSone(Sone originalSone, InputStream soneInputStream) throws SoneException {
226                 /* TODO - impose a size limit? */
227
228                 Document document;
229                 /* XML parsing is not thread-safe. */
230                 synchronized (this) {
231                         document = XML.transformToDocument(soneInputStream);
232                 }
233                 if (document == null) {
234                         /* TODO - mark Sone as bad. */
235                         logger.log(Level.WARNING, String.format("Could not parse XML for Sone %s!", originalSone));
236                         return null;
237                 }
238
239                 Sone sone = new Sone(originalSone.getId()).setIdentity(originalSone.getIdentity());
240
241                 SimpleXML soneXml;
242                 try {
243                         soneXml = SimpleXML.fromDocument(document);
244                 } catch (NullPointerException npe1) {
245                         /* for some reason, invalid XML can cause NPEs. */
246                         logger.log(Level.WARNING, String.format("XML for Sone %s can not be parsed!", sone), npe1);
247                         return null;
248                 }
249
250                 Integer protocolVersion = null;
251                 String soneProtocolVersion = soneXml.getValue("protocol-version", null);
252                 if (soneProtocolVersion != null) {
253                         protocolVersion = Numbers.safeParseInteger(soneProtocolVersion);
254                 }
255                 if (protocolVersion == null) {
256                         logger.log(Level.INFO, "No protocol version found, assuming 0.");
257                         protocolVersion = 0;
258                 }
259
260                 if (protocolVersion < 0) {
261                         logger.log(Level.WARNING, String.format("Invalid protocol version: %d! Not parsing Sone.", protocolVersion));
262                         return null;
263                 }
264
265                 /* check for valid versions. */
266                 if (protocolVersion > MAX_PROTOCOL_VERSION) {
267                         logger.log(Level.WARNING, String.format("Unknown protocol version: %d! Not parsing Sone.", protocolVersion));
268                         return null;
269                 }
270
271                 String soneTime = soneXml.getValue("time", null);
272                 if (soneTime == null) {
273                         /* TODO - mark Sone as bad. */
274                         logger.log(Level.WARNING, String.format("Downloaded time for Sone %s was null!", sone));
275                         return null;
276                 }
277                 try {
278                         sone.setTime(Long.parseLong(soneTime));
279                 } catch (NumberFormatException nfe1) {
280                         /* TODO - mark Sone as bad. */
281                         logger.log(Level.WARNING, String.format("Downloaded Sone %s with invalid time: %s", sone, soneTime));
282                         return null;
283                 }
284
285                 SimpleXML clientXml = soneXml.getNode("client");
286                 if (clientXml != null) {
287                         String clientName = clientXml.getValue("name", null);
288                         String clientVersion = clientXml.getValue("version", null);
289                         if ((clientName == null) || (clientVersion == null)) {
290                                 logger.log(Level.WARNING, String.format("Download Sone %s with client XML but missing name or version!", sone));
291                                 return null;
292                         }
293                         sone.setClient(new Client(clientName, clientVersion));
294                 }
295
296                 String soneRequestUri = soneXml.getValue("request-uri", null);
297                 if (soneRequestUri != null) {
298                         try {
299                                 sone.setRequestUri(new FreenetURI(soneRequestUri));
300                         } catch (MalformedURLException mue1) {
301                                 /* TODO - mark Sone as bad. */
302                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has invalid request URI: %s", sone, soneRequestUri), mue1);
303                                 return null;
304                         }
305                 }
306
307                 String soneInsertUri = soneXml.getValue("insert-uri", null);
308                 if ((soneInsertUri != null) && (sone.getInsertUri() == null)) {
309                         try {
310                                 sone.setInsertUri(new FreenetURI(soneInsertUri));
311                                 sone.setLatestEdition(Math.max(sone.getRequestUri().getEdition(), sone.getInsertUri().getEdition()));
312                         } catch (MalformedURLException mue1) {
313                                 /* TODO - mark Sone as bad. */
314                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has invalid insert URI: %s", sone, soneInsertUri), mue1);
315                                 return null;
316                         }
317                 }
318
319                 SimpleXML profileXml = soneXml.getNode("profile");
320                 if (profileXml == null) {
321                         /* TODO - mark Sone as bad. */
322                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no profile!", sone));
323                         return null;
324                 }
325
326                 /* parse profile. */
327                 String profileFirstName = profileXml.getValue("first-name", null);
328                 String profileMiddleName = profileXml.getValue("middle-name", null);
329                 String profileLastName = profileXml.getValue("last-name", null);
330                 Integer profileBirthDay = Numbers.safeParseInteger(profileXml.getValue("birth-day", null));
331                 Integer profileBirthMonth = Numbers.safeParseInteger(profileXml.getValue("birth-month", null));
332                 Integer profileBirthYear = Numbers.safeParseInteger(profileXml.getValue("birth-year", null));
333                 Profile profile = new Profile(sone).setFirstName(profileFirstName).setMiddleName(profileMiddleName).setLastName(profileLastName);
334                 profile.setBirthDay(profileBirthDay).setBirthMonth(profileBirthMonth).setBirthYear(profileBirthYear);
335                 /* avatar is processed after images are loaded. */
336                 String avatarId = profileXml.getValue("avatar", null);
337
338                 /* parse profile fields. */
339                 SimpleXML profileFieldsXml = profileXml.getNode("fields");
340                 if (profileFieldsXml != null) {
341                         for (SimpleXML fieldXml : profileFieldsXml.getNodes("field")) {
342                                 String fieldName = fieldXml.getValue("field-name", null);
343                                 String fieldValue = fieldXml.getValue("field-value", "");
344                                 if (fieldName == null) {
345                                         logger.log(Level.WARNING, String.format("Downloaded profile field for Sone %s with missing data! Name: %s, Value: %s", sone, fieldName, fieldValue));
346                                         return null;
347                                 }
348                                 try {
349                                         profile.addField(fieldName).setValue(fieldValue);
350                                 } catch (IllegalArgumentException iae1) {
351                                         logger.log(Level.WARNING, String.format("Duplicate field: %s", fieldName), iae1);
352                                         return null;
353                                 }
354                         }
355                 }
356
357                 /* parse posts. */
358                 SimpleXML postsXml = soneXml.getNode("posts");
359                 Set<Post> posts = new HashSet<Post>();
360                 if (postsXml == null) {
361                         /* TODO - mark Sone as bad. */
362                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no posts!", sone));
363                 } else {
364                         for (SimpleXML postXml : postsXml.getNodes("post")) {
365                                 String postId = postXml.getValue("id", null);
366                                 String postRecipientId = postXml.getValue("recipient", null);
367                                 String postTime = postXml.getValue("time", null);
368                                 String postText = postXml.getValue("text", null);
369                                 if ((postId == null) || (postTime == null) || (postText == null)) {
370                                         /* TODO - mark Sone as bad. */
371                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with missing data! ID: %s, Time: %s, Text: %s", sone, postId, postTime, postText));
372                                         return null;
373                                 }
374                                 try {
375                                         Post post = core.getPost(postId).setSone(sone).setTime(Long.parseLong(postTime)).setText(postText);
376                                         if ((postRecipientId != null) && (postRecipientId.length() == 43)) {
377                                                 post.setRecipient(core.getSone(postRecipientId));
378                                         }
379                                         posts.add(post);
380                                 } catch (NumberFormatException nfe1) {
381                                         /* TODO - mark Sone as bad. */
382                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with invalid time: %s", sone, postTime));
383                                         return null;
384                                 }
385                         }
386                 }
387
388                 /* parse replies. */
389                 SimpleXML repliesXml = soneXml.getNode("replies");
390                 Set<PostReply> replies = new HashSet<PostReply>();
391                 if (repliesXml == null) {
392                         /* TODO - mark Sone as bad. */
393                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no replies!", sone));
394                 } else {
395                         for (SimpleXML replyXml : repliesXml.getNodes("reply")) {
396                                 String replyId = replyXml.getValue("id", null);
397                                 String replyPostId = replyXml.getValue("post-id", null);
398                                 String replyTime = replyXml.getValue("time", null);
399                                 String replyText = replyXml.getValue("text", null);
400                                 if ((replyId == null) || (replyPostId == null) || (replyTime == null) || (replyText == null)) {
401                                         /* TODO - mark Sone as bad. */
402                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with missing data! ID: %s, Post: %s, Time: %s, Text: %s", sone, replyId, replyPostId, replyTime, replyText));
403                                         return null;
404                                 }
405                                 try {
406                                         replies.add(core.getReply(replyId).setSone(sone).setPost(core.getPost(replyPostId)).setTime(Long.parseLong(replyTime)).setText(replyText));
407                                 } catch (NumberFormatException nfe1) {
408                                         /* TODO - mark Sone as bad. */
409                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with invalid time: %s", sone, replyTime));
410                                         return null;
411                                 }
412                         }
413                 }
414
415                 /* parse liked post IDs. */
416                 SimpleXML likePostIdsXml = soneXml.getNode("post-likes");
417                 Set<String> likedPostIds = new HashSet<String>();
418                 if (likePostIdsXml == null) {
419                         /* TODO - mark Sone as bad. */
420                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no post likes!", sone));
421                 } else {
422                         for (SimpleXML likedPostIdXml : likePostIdsXml.getNodes("post-like")) {
423                                 String postId = likedPostIdXml.getValue();
424                                 likedPostIds.add(postId);
425                         }
426                 }
427
428                 /* parse liked reply IDs. */
429                 SimpleXML likeReplyIdsXml = soneXml.getNode("reply-likes");
430                 Set<String> likedReplyIds = new HashSet<String>();
431                 if (likeReplyIdsXml == null) {
432                         /* TODO - mark Sone as bad. */
433                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no reply likes!", sone));
434                 } else {
435                         for (SimpleXML likedReplyIdXml : likeReplyIdsXml.getNodes("reply-like")) {
436                                 String replyId = likedReplyIdXml.getValue();
437                                 likedReplyIds.add(replyId);
438                         }
439                 }
440
441                 /* parse albums. */
442                 SimpleXML albumsXml = soneXml.getNode("albums");
443                 List<Album> topLevelAlbums = new ArrayList<Album>();
444                 if (albumsXml != null) {
445                         for (SimpleXML albumXml : albumsXml.getNodes("album")) {
446                                 String id = albumXml.getValue("id", null);
447                                 String parentId = albumXml.getValue("parent", null);
448                                 String title = albumXml.getValue("title", null);
449                                 String description = albumXml.getValue("description", "");
450                                 String albumImageId = albumXml.getValue("album-image", null);
451                                 if ((id == null) || (title == null) || (description == null)) {
452                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid album!", sone));
453                                         return null;
454                                 }
455                                 Album parent = null;
456                                 if (parentId != null) {
457                                         parent = core.getAlbum(parentId, false);
458                                         if (parent == null) {
459                                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has album with invalid parent!", sone));
460                                                 return null;
461                                         }
462                                 }
463                                 Album album = core.getAlbum(id).setSone(sone).setTitle(title).setDescription(description);
464                                 if (parent != null) {
465                                         parent.addAlbum(album);
466                                 } else {
467                                         topLevelAlbums.add(album);
468                                 }
469                                 SimpleXML imagesXml = albumXml.getNode("images");
470                                 if (imagesXml != null) {
471                                         for (SimpleXML imageXml : imagesXml.getNodes("image")) {
472                                                 String imageId = imageXml.getValue("id", null);
473                                                 String imageCreationTimeString = imageXml.getValue("creation-time", null);
474                                                 String imageKey = imageXml.getValue("key", null);
475                                                 String imageTitle = imageXml.getValue("title", null);
476                                                 String imageDescription = imageXml.getValue("description", "");
477                                                 String imageWidthString = imageXml.getValue("width", null);
478                                                 String imageHeightString = imageXml.getValue("height", null);
479                                                 if ((imageId == null) || (imageCreationTimeString == null) || (imageKey == null) || (imageTitle == null) || (imageWidthString == null) || (imageHeightString == null)) {
480                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid images!", sone));
481                                                         return null;
482                                                 }
483                                                 long creationTime = Numbers.safeParseLong(imageCreationTimeString, 0L);
484                                                 int imageWidth = Numbers.safeParseInteger(imageWidthString, 0);
485                                                 int imageHeight = Numbers.safeParseInteger(imageHeightString, 0);
486                                                 if ((imageWidth < 1) || (imageHeight < 1)) {
487                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains image %s with invalid dimensions (%s, %s)!", sone, imageId, imageWidthString, imageHeightString));
488                                                         return null;
489                                                 }
490                                                 Image image = core.getImage(imageId).setSone(sone).setKey(imageKey).setCreationTime(creationTime);
491                                                 image.setTitle(imageTitle).setDescription(imageDescription);
492                                                 image.setWidth(imageWidth).setHeight(imageHeight);
493                                                 album.addImage(image);
494                                         }
495                                 }
496                                 album.setAlbumImage(albumImageId);
497                         }
498                 }
499
500                 /* process avatar. */
501                 if (avatarId != null) {
502                         profile.setAvatar(core.getImage(avatarId, false));
503                 }
504
505                 /* okay, apparently everything was parsed correctly. Now import. */
506                 /* atomic setter operation on the Sone. */
507                 synchronized (sone) {
508                         sone.setProfile(profile);
509                         sone.setPosts(posts);
510                         sone.setReplies(replies);
511                         sone.setLikePostIds(likedPostIds);
512                         sone.setLikeReplyIds(likedReplyIds);
513                         sone.setAlbums(topLevelAlbums);
514                 }
515
516                 return sone;
517         }
518
519         //
520         // SERVICE METHODS
521         //
522
523         /**
524          * {@inheritDoc}
525          */
526         @Override
527         protected void serviceStop() {
528                 for (Sone sone : sones) {
529                         freenetInterface.unregisterUsk(sone);
530                 }
531         }
532
533 }