Restrict access to methods.
[Sone.git] / src / main / java / net / pterodactylus / sone / core / SoneDownloader.java
1 /*
2  * Sone - SoneDownloader.java - Copyright © 2010–2013 David Roden
3  *
4  * This program is free software: you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16  */
17
18 package net.pterodactylus.sone.core;
19
20 import java.io.InputStream;
21 import java.net.MalformedURLException;
22 import java.util.ArrayList;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Set;
26 import java.util.logging.Level;
27 import java.util.logging.Logger;
28
29 import net.pterodactylus.sone.core.FreenetInterface.Fetched;
30 import net.pterodactylus.sone.data.Album;
31 import net.pterodactylus.sone.data.Client;
32 import net.pterodactylus.sone.data.Image;
33 import net.pterodactylus.sone.data.Post;
34 import net.pterodactylus.sone.data.PostReply;
35 import net.pterodactylus.sone.data.Profile;
36 import net.pterodactylus.sone.data.Sone;
37 import net.pterodactylus.sone.data.Sone.SoneStatus;
38 import net.pterodactylus.sone.data.SoneImpl;
39 import net.pterodactylus.sone.database.PostBuilder;
40 import net.pterodactylus.sone.database.PostReplyBuilder;
41 import net.pterodactylus.util.io.Closer;
42 import net.pterodactylus.util.logging.Logging;
43 import net.pterodactylus.util.number.Numbers;
44 import net.pterodactylus.util.service.AbstractService;
45 import net.pterodactylus.util.xml.SimpleXML;
46 import net.pterodactylus.util.xml.XML;
47
48 import com.google.common.annotations.VisibleForTesting;
49 import org.w3c.dom.Document;
50
51 import freenet.client.FetchResult;
52 import freenet.keys.FreenetURI;
53 import freenet.support.api.Bucket;
54
55 /**
56  * The Sone downloader is responsible for download Sones as they are updated.
57  *
58  * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
59  */
60 public class SoneDownloader extends AbstractService {
61
62         /** The logger. */
63         private static final Logger logger = Logging.getLogger(SoneDownloader.class);
64
65         /** The maximum protocol version. */
66         private static final int MAX_PROTOCOL_VERSION = 0;
67
68         /** The core. */
69         private final Core core;
70
71         /** The Freenet interface. */
72         private final FreenetInterface freenetInterface;
73
74         /** The sones to update. */
75         private final Set<Sone> sones = new HashSet<Sone>();
76
77         /**
78          * Creates a new Sone downloader.
79          *
80          * @param core
81          *            The core
82          * @param freenetInterface
83          *            The Freenet interface
84          */
85         public SoneDownloader(Core core, FreenetInterface freenetInterface) {
86                 super("Sone Downloader", false);
87                 this.core = core;
88                 this.freenetInterface = freenetInterface;
89         }
90
91         //
92         // ACTIONS
93         //
94
95         /**
96          * Adds the given Sone to the set of Sones that will be watched for updates.
97          *
98          * @param sone
99          *            The Sone to add
100          */
101         public void addSone(Sone sone) {
102                 if (!sones.add(sone)) {
103                         freenetInterface.unregisterUsk(sone);
104                 }
105                 freenetInterface.registerUsk(sone, this);
106         }
107
108         /**
109          * Fetches the updated Sone. This method is a callback method for
110          * {@link FreenetInterface#registerUsk(Sone, SoneDownloader)}.
111          *
112          * @param sone
113          *            The Sone to fetch
114          */
115         public void fetchSone(Sone sone) {
116                 fetchSone(sone, sone.getRequestUri().sskForUSK());
117         }
118
119         /**
120          * Fetches the updated Sone. This method can be used to fetch a Sone from a
121          * specific URI.
122          *
123          * @param sone
124          *            The Sone to fetch
125          * @param soneUri
126          *            The URI to fetch the Sone from
127          */
128         public void fetchSone(Sone sone, FreenetURI soneUri) {
129                 fetchSone(sone, soneUri, false);
130         }
131
132         /**
133          * Fetches the Sone from the given URI.
134          *
135          * @param sone
136          *            The Sone to fetch
137          * @param soneUri
138          *            The URI of the Sone to fetch
139          * @param fetchOnly
140          *            {@code true} to only fetch and parse the Sone, {@code false}
141          *            to {@link Core#updateSone(Sone) update} it in the core
142          * @return The downloaded Sone, or {@code null} if the Sone could not be
143          *         downloaded
144          */
145         public Sone fetchSone(Sone sone, FreenetURI soneUri, boolean fetchOnly) {
146                 logger.log(Level.FINE, String.format("Starting fetch for Sone “%s” from %s…", sone, soneUri));
147                 FreenetURI requestUri = soneUri.setMetaString(new String[] { "sone.xml" });
148                 sone.setStatus(SoneStatus.downloading);
149                 try {
150                         Fetched fetchResults = freenetInterface.fetchUri(requestUri);
151                         if (fetchResults == null) {
152                                 /* TODO - mark Sone as bad. */
153                                 return null;
154                         }
155                         logger.log(Level.FINEST, String.format("Got %d bytes back.", fetchResults.getFetchResult().size()));
156                         Sone parsedSone = parseSone(sone, fetchResults.getFetchResult(), fetchResults.getFreenetUri());
157                         if (parsedSone != null) {
158                                 if (!fetchOnly) {
159                                         parsedSone.setStatus((parsedSone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
160                                         core.updateSone(parsedSone);
161                                         addSone(parsedSone);
162                                 }
163                         }
164                         return parsedSone;
165                 } finally {
166                         sone.setStatus((sone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
167                 }
168         }
169
170         /**
171          * Parses a Sone from a fetch result.
172          *
173          * @param originalSone
174          *            The sone to parse, or {@code null} if the Sone is yet unknown
175          * @param fetchResult
176          *            The fetch result
177          * @param requestUri
178          *            The requested URI
179          * @return The parsed Sone, or {@code null} if the Sone could not be parsed
180          */
181         private Sone parseSone(Sone originalSone, FetchResult fetchResult, FreenetURI requestUri) {
182                 logger.log(Level.FINEST, String.format("Parsing FetchResult (%d bytes, %s) for %s…", fetchResult.size(), fetchResult.getMimeType(), originalSone));
183                 Bucket soneBucket = fetchResult.asBucket();
184                 InputStream soneInputStream = null;
185                 try {
186                         soneInputStream = soneBucket.getInputStream();
187                         Sone parsedSone = parseSone(originalSone, soneInputStream);
188                         if (parsedSone != null) {
189                                 parsedSone.setLatestEdition(requestUri.getEdition());
190                                 if (requestUri.getKeyType().equals("USK")) {
191                                         parsedSone.setRequestUri(requestUri.setMetaString(new String[0]));
192                                 } else {
193                                         parsedSone.setRequestUri(requestUri.setKeyType("USK").setDocName("Sone").setMetaString(new String[0]));
194                                 }
195                         }
196                         return parsedSone;
197                 } catch (Exception e1) {
198                         logger.log(Level.WARNING, String.format("Could not parse Sone from %s!", requestUri), e1);
199                 } finally {
200                         Closer.close(soneInputStream);
201                         soneBucket.free();
202                 }
203                 return null;
204         }
205
206         /**
207          * Parses a Sone from the given input stream and creates a new Sone from the
208          * parsed data.
209          *
210          * @param originalSone
211          *            The Sone to update
212          * @param soneInputStream
213          *            The input stream to parse the Sone from
214          * @return The parsed Sone
215          * @throws SoneException
216          *             if a parse error occurs, or the protocol is invalid
217          */
218         @VisibleForTesting
219         protected Sone parseSone(Sone originalSone, InputStream soneInputStream) throws SoneException {
220                 /* TODO - impose a size limit? */
221
222                 Document document;
223                 /* XML parsing is not thread-safe. */
224                 synchronized (this) {
225                         document = XML.transformToDocument(soneInputStream);
226                 }
227                 if (document == null) {
228                         /* TODO - mark Sone as bad. */
229                         logger.log(Level.WARNING, String.format("Could not parse XML for Sone %s!", originalSone));
230                         return null;
231                 }
232
233                 Sone sone = new SoneImpl(originalSone.getId(), originalSone.isLocal()).setIdentity(originalSone.getIdentity());
234
235                 SimpleXML soneXml;
236                 try {
237                         soneXml = SimpleXML.fromDocument(document);
238                 } catch (NullPointerException npe1) {
239                         /* for some reason, invalid XML can cause NPEs. */
240                         logger.log(Level.WARNING, String.format("XML for Sone %s can not be parsed!", sone), npe1);
241                         return null;
242                 }
243
244                 Integer protocolVersion = null;
245                 String soneProtocolVersion = soneXml.getValue("protocol-version", null);
246                 if (soneProtocolVersion != null) {
247                         protocolVersion = Numbers.safeParseInteger(soneProtocolVersion);
248                 }
249                 if (protocolVersion == null) {
250                         logger.log(Level.INFO, "No protocol version found, assuming 0.");
251                         protocolVersion = 0;
252                 }
253
254                 if (protocolVersion < 0) {
255                         logger.log(Level.WARNING, String.format("Invalid protocol version: %d! Not parsing Sone.", protocolVersion));
256                         return null;
257                 }
258
259                 /* check for valid versions. */
260                 if (protocolVersion > MAX_PROTOCOL_VERSION) {
261                         logger.log(Level.WARNING, String.format("Unknown protocol version: %d! Not parsing Sone.", protocolVersion));
262                         return null;
263                 }
264
265                 String soneTime = soneXml.getValue("time", null);
266                 if (soneTime == null) {
267                         /* TODO - mark Sone as bad. */
268                         logger.log(Level.WARNING, String.format("Downloaded time for Sone %s was null!", sone));
269                         return null;
270                 }
271                 try {
272                         sone.setTime(Long.parseLong(soneTime));
273                 } catch (NumberFormatException nfe1) {
274                         /* TODO - mark Sone as bad. */
275                         logger.log(Level.WARNING, String.format("Downloaded Sone %s with invalid time: %s", sone, soneTime));
276                         return null;
277                 }
278
279                 SimpleXML clientXml = soneXml.getNode("client");
280                 if (clientXml != null) {
281                         String clientName = clientXml.getValue("name", null);
282                         String clientVersion = clientXml.getValue("version", null);
283                         if ((clientName == null) || (clientVersion == null)) {
284                                 logger.log(Level.WARNING, String.format("Download Sone %s with client XML but missing name or version!", sone));
285                                 return null;
286                         }
287                         sone.setClient(new Client(clientName, clientVersion));
288                 }
289
290                 String soneRequestUri = soneXml.getValue("request-uri", null);
291                 if (soneRequestUri != null) {
292                         try {
293                                 sone.setRequestUri(new FreenetURI(soneRequestUri));
294                         } catch (MalformedURLException mue1) {
295                                 /* TODO - mark Sone as bad. */
296                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has invalid request URI: %s", sone, soneRequestUri), mue1);
297                                 return null;
298                         }
299                 }
300
301                 if (originalSone.getInsertUri() != null) {
302                         sone.setInsertUri(originalSone.getInsertUri());
303                 }
304
305                 SimpleXML profileXml = soneXml.getNode("profile");
306                 if (profileXml == null) {
307                         /* TODO - mark Sone as bad. */
308                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no profile!", sone));
309                         return null;
310                 }
311
312                 /* parse profile. */
313                 String profileFirstName = profileXml.getValue("first-name", null);
314                 String profileMiddleName = profileXml.getValue("middle-name", null);
315                 String profileLastName = profileXml.getValue("last-name", null);
316                 Integer profileBirthDay = Numbers.safeParseInteger(profileXml.getValue("birth-day", null));
317                 Integer profileBirthMonth = Numbers.safeParseInteger(profileXml.getValue("birth-month", null));
318                 Integer profileBirthYear = Numbers.safeParseInteger(profileXml.getValue("birth-year", null));
319                 Profile profile = new Profile(sone).setFirstName(profileFirstName).setMiddleName(profileMiddleName).setLastName(profileLastName);
320                 profile.setBirthDay(profileBirthDay).setBirthMonth(profileBirthMonth).setBirthYear(profileBirthYear);
321                 /* avatar is processed after images are loaded. */
322                 String avatarId = profileXml.getValue("avatar", null);
323
324                 /* parse profile fields. */
325                 SimpleXML profileFieldsXml = profileXml.getNode("fields");
326                 if (profileFieldsXml != null) {
327                         for (SimpleXML fieldXml : profileFieldsXml.getNodes("field")) {
328                                 String fieldName = fieldXml.getValue("field-name", null);
329                                 String fieldValue = fieldXml.getValue("field-value", "");
330                                 if (fieldName == null) {
331                                         logger.log(Level.WARNING, String.format("Downloaded profile field for Sone %s with missing data! Name: %s, Value: %s", sone, fieldName, fieldValue));
332                                         return null;
333                                 }
334                                 try {
335                                         profile.addField(fieldName.trim()).setValue(fieldValue);
336                                 } catch (IllegalArgumentException iae1) {
337                                         logger.log(Level.WARNING, String.format("Duplicate field: %s", fieldName), iae1);
338                                         return null;
339                                 }
340                         }
341                 }
342
343                 /* parse posts. */
344                 SimpleXML postsXml = soneXml.getNode("posts");
345                 Set<Post> posts = new HashSet<Post>();
346                 if (postsXml == null) {
347                         /* TODO - mark Sone as bad. */
348                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no posts!", sone));
349                 } else {
350                         for (SimpleXML postXml : postsXml.getNodes("post")) {
351                                 String postId = postXml.getValue("id", null);
352                                 String postRecipientId = postXml.getValue("recipient", null);
353                                 String postTime = postXml.getValue("time", null);
354                                 String postText = postXml.getValue("text", null);
355                                 if ((postId == null) || (postTime == null) || (postText == null)) {
356                                         /* TODO - mark Sone as bad. */
357                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with missing data! ID: %s, Time: %s, Text: %s", sone, postId, postTime, postText));
358                                         return null;
359                                 }
360                                 try {
361                                         PostBuilder postBuilder = core.postBuilder();
362                                         /* TODO - parse time correctly. */
363                                         postBuilder.withId(postId).from(sone.getId()).withTime(Long.parseLong(postTime)).withText(postText);
364                                         if ((postRecipientId != null) && (postRecipientId.length() == 43)) {
365                                                 postBuilder.to(postRecipientId);
366                                         }
367                                         posts.add(postBuilder.build());
368                                 } catch (NumberFormatException nfe1) {
369                                         /* TODO - mark Sone as bad. */
370                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with invalid time: %s", sone, postTime));
371                                         return null;
372                                 }
373                         }
374                 }
375
376                 /* parse replies. */
377                 SimpleXML repliesXml = soneXml.getNode("replies");
378                 Set<PostReply> replies = new HashSet<PostReply>();
379                 if (repliesXml == null) {
380                         /* TODO - mark Sone as bad. */
381                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no replies!", sone));
382                 } else {
383                         for (SimpleXML replyXml : repliesXml.getNodes("reply")) {
384                                 String replyId = replyXml.getValue("id", null);
385                                 String replyPostId = replyXml.getValue("post-id", null);
386                                 String replyTime = replyXml.getValue("time", null);
387                                 String replyText = replyXml.getValue("text", null);
388                                 if ((replyId == null) || (replyPostId == null) || (replyTime == null) || (replyText == null)) {
389                                         /* TODO - mark Sone as bad. */
390                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with missing data! ID: %s, Post: %s, Time: %s, Text: %s", sone, replyId, replyPostId, replyTime, replyText));
391                                         return null;
392                                 }
393                                 try {
394                                         PostReplyBuilder postReplyBuilder = core.postReplyBuilder();
395                                         /* TODO - parse time correctly. */
396                                         postReplyBuilder.withId(replyId).from(sone.getId()).to(replyPostId).withTime(Long.parseLong(replyTime)).withText(replyText);
397                                         replies.add(postReplyBuilder.build());
398                                 } catch (NumberFormatException nfe1) {
399                                         /* TODO - mark Sone as bad. */
400                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with invalid time: %s", sone, replyTime));
401                                         return null;
402                                 }
403                         }
404                 }
405
406                 /* parse liked post IDs. */
407                 SimpleXML likePostIdsXml = soneXml.getNode("post-likes");
408                 Set<String> likedPostIds = new HashSet<String>();
409                 if (likePostIdsXml == null) {
410                         /* TODO - mark Sone as bad. */
411                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no post likes!", sone));
412                 } else {
413                         for (SimpleXML likedPostIdXml : likePostIdsXml.getNodes("post-like")) {
414                                 String postId = likedPostIdXml.getValue();
415                                 likedPostIds.add(postId);
416                         }
417                 }
418
419                 /* parse liked reply IDs. */
420                 SimpleXML likeReplyIdsXml = soneXml.getNode("reply-likes");
421                 Set<String> likedReplyIds = new HashSet<String>();
422                 if (likeReplyIdsXml == null) {
423                         /* TODO - mark Sone as bad. */
424                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no reply likes!", sone));
425                 } else {
426                         for (SimpleXML likedReplyIdXml : likeReplyIdsXml.getNodes("reply-like")) {
427                                 String replyId = likedReplyIdXml.getValue();
428                                 likedReplyIds.add(replyId);
429                         }
430                 }
431
432                 /* parse albums. */
433                 SimpleXML albumsXml = soneXml.getNode("albums");
434                 List<Album> topLevelAlbums = new ArrayList<Album>();
435                 if (albumsXml != null) {
436                         for (SimpleXML albumXml : albumsXml.getNodes("album")) {
437                                 String id = albumXml.getValue("id", null);
438                                 String parentId = albumXml.getValue("parent", null);
439                                 String title = albumXml.getValue("title", null);
440                                 String description = albumXml.getValue("description", "");
441                                 String albumImageId = albumXml.getValue("album-image", null);
442                                 if ((id == null) || (title == null)) {
443                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid album!", sone));
444                                         return null;
445                                 }
446                                 Album parent = null;
447                                 if (parentId != null) {
448                                         parent = core.getAlbum(parentId, false);
449                                         if (parent == null) {
450                                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has album with invalid parent!", sone));
451                                                 return null;
452                                         }
453                                 }
454                                 Album album = core.getAlbum(id).setSone(sone).modify().setTitle(title).setDescription(description).update();
455                                 if (parent != null) {
456                                         parent.addAlbum(album);
457                                 } else {
458                                         topLevelAlbums.add(album);
459                                 }
460                                 SimpleXML imagesXml = albumXml.getNode("images");
461                                 if (imagesXml != null) {
462                                         for (SimpleXML imageXml : imagesXml.getNodes("image")) {
463                                                 String imageId = imageXml.getValue("id", null);
464                                                 String imageCreationTimeString = imageXml.getValue("creation-time", null);
465                                                 String imageKey = imageXml.getValue("key", null);
466                                                 String imageTitle = imageXml.getValue("title", null);
467                                                 String imageDescription = imageXml.getValue("description", "");
468                                                 String imageWidthString = imageXml.getValue("width", null);
469                                                 String imageHeightString = imageXml.getValue("height", null);
470                                                 if ((imageId == null) || (imageCreationTimeString == null) || (imageKey == null) || (imageTitle == null) || (imageWidthString == null) || (imageHeightString == null)) {
471                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid images!", sone));
472                                                         return null;
473                                                 }
474                                                 long creationTime = Numbers.safeParseLong(imageCreationTimeString, 0L);
475                                                 int imageWidth = Numbers.safeParseInteger(imageWidthString, 0);
476                                                 int imageHeight = Numbers.safeParseInteger(imageHeightString, 0);
477                                                 if ((imageWidth < 1) || (imageHeight < 1)) {
478                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains image %s with invalid dimensions (%s, %s)!", sone, imageId, imageWidthString, imageHeightString));
479                                                         return null;
480                                                 }
481                                                 Image image = core.getImage(imageId).modify().setSone(sone).setKey(imageKey).setCreationTime(creationTime).update();
482                                                 image = image.modify().setTitle(imageTitle).setDescription(imageDescription).update();
483                                                 image = image.modify().setWidth(imageWidth).setHeight(imageHeight).update();
484                                                 album.addImage(image);
485                                         }
486                                 }
487                                 album.modify().setAlbumImage(albumImageId).update();
488                         }
489                 }
490
491                 /* process avatar. */
492                 if (avatarId != null) {
493                         profile.setAvatar(core.getImage(avatarId, false));
494                 }
495
496                 /* okay, apparently everything was parsed correctly. Now import. */
497                 /* atomic setter operation on the Sone. */
498                 synchronized (sone) {
499                         sone.setProfile(profile);
500                         sone.setPosts(posts);
501                         sone.setReplies(replies);
502                         sone.setLikePostIds(likedPostIds);
503                         sone.setLikeReplyIds(likedReplyIds);
504                         for (Album album : topLevelAlbums) {
505                                 sone.getRootAlbum().addAlbum(album);
506                         }
507                 }
508
509                 return sone;
510         }
511
512         //
513         // SERVICE METHODS
514         //
515
516         /**
517          * {@inheritDoc}
518          */
519         @Override
520         protected void serviceStop() {
521                 for (Sone sone : sones) {
522                         freenetInterface.unregisterUsk(sone);
523                 }
524         }
525
526         public class FetchSoneWithUri implements Runnable {
527
528                 private final Sone sone;
529
530                 public FetchSoneWithUri(Sone sone) {
531                         this.sone = sone;
532                 }
533
534                 @Override
535                 public void run() {
536                         fetchSone(sone, sone.getRequestUri());
537                 }
538
539         }
540
541         public class FetchSone implements Runnable {
542
543                 private final Sone sone;
544
545                 public FetchSone(Sone sone) {
546                         this.sone = sone;
547                 }
548
549                 @Override
550                 public void run() {
551                         fetchSone(sone);
552                 }
553
554         }
555
556 }