Add interface between freenet interface and Sone downloader.
[Sone.git] / src / main / java / net / pterodactylus / sone / core / SoneDownloaderImpl.java
1 /*
2  * Sone - SoneDownloader.java - Copyright © 2010–2013 David Roden
3  *
4  * This program is free software: you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16  */
17
18 package net.pterodactylus.sone.core;
19
20 import java.io.InputStream;
21 import java.net.MalformedURLException;
22 import java.util.ArrayList;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Set;
26 import java.util.logging.Level;
27 import java.util.logging.Logger;
28
29 import net.pterodactylus.sone.core.FreenetInterface.Fetched;
30 import net.pterodactylus.sone.data.Album;
31 import net.pterodactylus.sone.data.Client;
32 import net.pterodactylus.sone.data.Image;
33 import net.pterodactylus.sone.data.Post;
34 import net.pterodactylus.sone.data.PostReply;
35 import net.pterodactylus.sone.data.Profile;
36 import net.pterodactylus.sone.data.Sone;
37 import net.pterodactylus.sone.data.Sone.SoneStatus;
38 import net.pterodactylus.sone.data.SoneImpl;
39 import net.pterodactylus.sone.database.PostBuilder;
40 import net.pterodactylus.sone.database.PostReplyBuilder;
41 import net.pterodactylus.util.io.Closer;
42 import net.pterodactylus.util.logging.Logging;
43 import net.pterodactylus.util.number.Numbers;
44 import net.pterodactylus.util.service.AbstractService;
45 import net.pterodactylus.util.xml.SimpleXML;
46 import net.pterodactylus.util.xml.XML;
47
48 import freenet.client.FetchResult;
49 import freenet.keys.FreenetURI;
50 import freenet.support.api.Bucket;
51
52 import com.google.common.annotations.VisibleForTesting;
53 import org.w3c.dom.Document;
54
55 /**
56  * The Sone downloader is responsible for download Sones as they are updated.
57  *
58  * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
59  */
60 public class SoneDownloaderImpl extends AbstractService implements SoneDownloader {
61
62         /** The logger. */
63         private static final Logger logger = Logging.getLogger(SoneDownloaderImpl.class);
64
65         /** The maximum protocol version. */
66         private static final int MAX_PROTOCOL_VERSION = 0;
67
68         /** The core. */
69         private final Core core;
70
71         /** The Freenet interface. */
72         private final FreenetInterface freenetInterface;
73
74         /** The sones to update. */
75         private final Set<Sone> sones = new HashSet<Sone>();
76
77         /**
78          * Creates a new Sone downloader.
79          *
80          * @param core
81          *              The core
82          * @param freenetInterface
83          *              The Freenet interface
84          */
85         public SoneDownloaderImpl(Core core, FreenetInterface freenetInterface) {
86                 super("Sone Downloader", false);
87                 this.core = core;
88                 this.freenetInterface = freenetInterface;
89         }
90
91         //
92         // ACTIONS
93         //
94
95         /**
96          * Adds the given Sone to the set of Sones that will be watched for updates.
97          *
98          * @param sone
99          *              The Sone to add
100          */
101         @Override
102         public void addSone(Sone sone) {
103                 if (!sones.add(sone)) {
104                         freenetInterface.unregisterUsk(sone);
105                 }
106                 freenetInterface.registerUsk(sone, new SoneUpdater() {
107                         @Override
108                         public void updateSone(Sone sone, long edition) {
109                                 if (edition > sone.getLatestEdition()) {
110                                         sone.setLatestEdition(edition);
111                                         new Thread(fetchSoneAction(sone),
112                                                         "Sone Downloader").start();
113                                 }
114                         }
115                 });
116         }
117
118         private void fetchSone(Sone sone) {
119                 fetchSone(sone, sone.getRequestUri().sskForUSK());
120         }
121
122         /**
123          * Fetches the updated Sone. This method can be used to fetch a Sone from a
124          * specific URI.
125          *
126          * @param sone
127          *              The Sone to fetch
128          * @param soneUri
129          *              The URI to fetch the Sone from
130          */
131         @Override
132         public void fetchSone(Sone sone, FreenetURI soneUri) {
133                 fetchSone(sone, soneUri, false);
134         }
135
136         /**
137          * Fetches the Sone from the given URI.
138          *
139          * @param sone
140          *              The Sone to fetch
141          * @param soneUri
142          *              The URI of the Sone to fetch
143          * @param fetchOnly
144          *              {@code true} to only fetch and parse the Sone, {@code false}
145          *              to {@link Core#updateSone(Sone) update} it in the core
146          * @return The downloaded Sone, or {@code null} if the Sone could not be
147          *         downloaded
148          */
149         @Override
150         public Sone fetchSone(Sone sone, FreenetURI soneUri, boolean fetchOnly) {
151                 logger.log(Level.FINE, String.format("Starting fetch for Sone “%s” from %s…", sone, soneUri));
152                 FreenetURI requestUri = soneUri.setMetaString(new String[] { "sone.xml" });
153                 sone.setStatus(SoneStatus.downloading);
154                 try {
155                         Fetched fetchResults = freenetInterface.fetchUri(requestUri);
156                         if (fetchResults == null) {
157                                 /* TODO - mark Sone as bad. */
158                                 return null;
159                         }
160                         logger.log(Level.FINEST, String.format("Got %d bytes back.", fetchResults.getFetchResult().size()));
161                         Sone parsedSone = parseSone(sone, fetchResults.getFetchResult(), fetchResults.getFreenetUri());
162                         if (parsedSone != null) {
163                                 if (!fetchOnly) {
164                                         parsedSone.setStatus((parsedSone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
165                                         core.updateSone(parsedSone);
166                                         addSone(parsedSone);
167                                 }
168                         }
169                         return parsedSone;
170                 } finally {
171                         sone.setStatus((sone.getTime() == 0) ? SoneStatus.unknown : SoneStatus.idle);
172                 }
173         }
174
175         /**
176          * Parses a Sone from a fetch result.
177          *
178          * @param originalSone
179          *              The sone to parse, or {@code null} if the Sone is yet unknown
180          * @param fetchResult
181          *              The fetch result
182          * @param requestUri
183          *              The requested URI
184          * @return The parsed Sone, or {@code null} if the Sone could not be parsed
185          */
186         private Sone parseSone(Sone originalSone, FetchResult fetchResult, FreenetURI requestUri) {
187                 logger.log(Level.FINEST, String.format("Parsing FetchResult (%d bytes, %s) for %s…", fetchResult.size(), fetchResult.getMimeType(), originalSone));
188                 Bucket soneBucket = fetchResult.asBucket();
189                 InputStream soneInputStream = null;
190                 try {
191                         soneInputStream = soneBucket.getInputStream();
192                         Sone parsedSone = parseSone(originalSone, soneInputStream);
193                         if (parsedSone != null) {
194                                 parsedSone.setLatestEdition(requestUri.getEdition());
195                                 if (requestUri.getKeyType().equals("USK")) {
196                                         parsedSone.setRequestUri(requestUri.setMetaString(new String[0]));
197                                 } else {
198                                         parsedSone.setRequestUri(requestUri.setKeyType("USK").setDocName("Sone").setMetaString(new String[0]));
199                                 }
200                         }
201                         return parsedSone;
202                 } catch (Exception e1) {
203                         logger.log(Level.WARNING, String.format("Could not parse Sone from %s!", requestUri), e1);
204                 } finally {
205                         Closer.close(soneInputStream);
206                         soneBucket.free();
207                 }
208                 return null;
209         }
210
211         /**
212          * Parses a Sone from the given input stream and creates a new Sone from the
213          * parsed data.
214          *
215          * @param originalSone
216          *              The Sone to update
217          * @param soneInputStream
218          *              The input stream to parse the Sone from
219          * @return The parsed Sone
220          * @throws SoneException
221          *              if a parse error occurs, or the protocol is invalid
222          */
223         @VisibleForTesting
224         protected Sone parseSone(Sone originalSone, InputStream soneInputStream) throws SoneException {
225                 /* TODO - impose a size limit? */
226
227                 Document document;
228                 /* XML parsing is not thread-safe. */
229                 synchronized (this) {
230                         document = XML.transformToDocument(soneInputStream);
231                 }
232                 if (document == null) {
233                         /* TODO - mark Sone as bad. */
234                         logger.log(Level.WARNING, String.format("Could not parse XML for Sone %s!", originalSone));
235                         return null;
236                 }
237
238                 Sone sone = new SoneImpl(originalSone.getId(), originalSone.isLocal()).setIdentity(originalSone.getIdentity());
239
240                 SimpleXML soneXml;
241                 try {
242                         soneXml = SimpleXML.fromDocument(document);
243                 } catch (NullPointerException npe1) {
244                         /* for some reason, invalid XML can cause NPEs. */
245                         logger.log(Level.WARNING, String.format("XML for Sone %s can not be parsed!", sone), npe1);
246                         return null;
247                 }
248
249                 Integer protocolVersion = null;
250                 String soneProtocolVersion = soneXml.getValue("protocol-version", null);
251                 if (soneProtocolVersion != null) {
252                         protocolVersion = Numbers.safeParseInteger(soneProtocolVersion);
253                 }
254                 if (protocolVersion == null) {
255                         logger.log(Level.INFO, "No protocol version found, assuming 0.");
256                         protocolVersion = 0;
257                 }
258
259                 if (protocolVersion < 0) {
260                         logger.log(Level.WARNING, String.format("Invalid protocol version: %d! Not parsing Sone.", protocolVersion));
261                         return null;
262                 }
263
264                 /* check for valid versions. */
265                 if (protocolVersion > MAX_PROTOCOL_VERSION) {
266                         logger.log(Level.WARNING, String.format("Unknown protocol version: %d! Not parsing Sone.", protocolVersion));
267                         return null;
268                 }
269
270                 String soneTime = soneXml.getValue("time", null);
271                 if (soneTime == null) {
272                         /* TODO - mark Sone as bad. */
273                         logger.log(Level.WARNING, String.format("Downloaded time for Sone %s was null!", sone));
274                         return null;
275                 }
276                 try {
277                         sone.setTime(Long.parseLong(soneTime));
278                 } catch (NumberFormatException nfe1) {
279                         /* TODO - mark Sone as bad. */
280                         logger.log(Level.WARNING, String.format("Downloaded Sone %s with invalid time: %s", sone, soneTime));
281                         return null;
282                 }
283
284                 SimpleXML clientXml = soneXml.getNode("client");
285                 if (clientXml != null) {
286                         String clientName = clientXml.getValue("name", null);
287                         String clientVersion = clientXml.getValue("version", null);
288                         if ((clientName == null) || (clientVersion == null)) {
289                                 logger.log(Level.WARNING, String.format("Download Sone %s with client XML but missing name or version!", sone));
290                                 return null;
291                         }
292                         sone.setClient(new Client(clientName, clientVersion));
293                 }
294
295                 String soneRequestUri = soneXml.getValue("request-uri", null);
296                 if (soneRequestUri != null) {
297                         try {
298                                 sone.setRequestUri(new FreenetURI(soneRequestUri));
299                         } catch (MalformedURLException mue1) {
300                                 /* TODO - mark Sone as bad. */
301                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has invalid request URI: %s", sone, soneRequestUri), mue1);
302                                 return null;
303                         }
304                 }
305
306                 if (originalSone.getInsertUri() != null) {
307                         sone.setInsertUri(originalSone.getInsertUri());
308                 }
309
310                 SimpleXML profileXml = soneXml.getNode("profile");
311                 if (profileXml == null) {
312                         /* TODO - mark Sone as bad. */
313                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no profile!", sone));
314                         return null;
315                 }
316
317                 /* parse profile. */
318                 String profileFirstName = profileXml.getValue("first-name", null);
319                 String profileMiddleName = profileXml.getValue("middle-name", null);
320                 String profileLastName = profileXml.getValue("last-name", null);
321                 Integer profileBirthDay = Numbers.safeParseInteger(profileXml.getValue("birth-day", null));
322                 Integer profileBirthMonth = Numbers.safeParseInteger(profileXml.getValue("birth-month", null));
323                 Integer profileBirthYear = Numbers.safeParseInteger(profileXml.getValue("birth-year", null));
324                 Profile profile = new Profile(sone).setFirstName(profileFirstName).setMiddleName(profileMiddleName).setLastName(profileLastName);
325                 profile.setBirthDay(profileBirthDay).setBirthMonth(profileBirthMonth).setBirthYear(profileBirthYear);
326                 /* avatar is processed after images are loaded. */
327                 String avatarId = profileXml.getValue("avatar", null);
328
329                 /* parse profile fields. */
330                 SimpleXML profileFieldsXml = profileXml.getNode("fields");
331                 if (profileFieldsXml != null) {
332                         for (SimpleXML fieldXml : profileFieldsXml.getNodes("field")) {
333                                 String fieldName = fieldXml.getValue("field-name", null);
334                                 String fieldValue = fieldXml.getValue("field-value", "");
335                                 if (fieldName == null) {
336                                         logger.log(Level.WARNING, String.format("Downloaded profile field for Sone %s with missing data! Name: %s, Value: %s", sone, fieldName, fieldValue));
337                                         return null;
338                                 }
339                                 try {
340                                         profile.addField(fieldName.trim()).setValue(fieldValue);
341                                 } catch (IllegalArgumentException iae1) {
342                                         logger.log(Level.WARNING, String.format("Duplicate field: %s", fieldName), iae1);
343                                         return null;
344                                 }
345                         }
346                 }
347
348                 /* parse posts. */
349                 SimpleXML postsXml = soneXml.getNode("posts");
350                 Set<Post> posts = new HashSet<Post>();
351                 if (postsXml == null) {
352                         /* TODO - mark Sone as bad. */
353                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no posts!", sone));
354                 } else {
355                         for (SimpleXML postXml : postsXml.getNodes("post")) {
356                                 String postId = postXml.getValue("id", null);
357                                 String postRecipientId = postXml.getValue("recipient", null);
358                                 String postTime = postXml.getValue("time", null);
359                                 String postText = postXml.getValue("text", null);
360                                 if ((postId == null) || (postTime == null) || (postText == null)) {
361                                         /* TODO - mark Sone as bad. */
362                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with missing data! ID: %s, Time: %s, Text: %s", sone, postId, postTime, postText));
363                                         return null;
364                                 }
365                                 try {
366                                         PostBuilder postBuilder = core.postBuilder();
367                                         /* TODO - parse time correctly. */
368                                         postBuilder.withId(postId).from(sone.getId()).withTime(Long.parseLong(postTime)).withText(postText);
369                                         if ((postRecipientId != null) && (postRecipientId.length() == 43)) {
370                                                 postBuilder.to(postRecipientId);
371                                         }
372                                         posts.add(postBuilder.build());
373                                 } catch (NumberFormatException nfe1) {
374                                         /* TODO - mark Sone as bad. */
375                                         logger.log(Level.WARNING, String.format("Downloaded post for Sone %s with invalid time: %s", sone, postTime));
376                                         return null;
377                                 }
378                         }
379                 }
380
381                 /* parse replies. */
382                 SimpleXML repliesXml = soneXml.getNode("replies");
383                 Set<PostReply> replies = new HashSet<PostReply>();
384                 if (repliesXml == null) {
385                         /* TODO - mark Sone as bad. */
386                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no replies!", sone));
387                 } else {
388                         for (SimpleXML replyXml : repliesXml.getNodes("reply")) {
389                                 String replyId = replyXml.getValue("id", null);
390                                 String replyPostId = replyXml.getValue("post-id", null);
391                                 String replyTime = replyXml.getValue("time", null);
392                                 String replyText = replyXml.getValue("text", null);
393                                 if ((replyId == null) || (replyPostId == null) || (replyTime == null) || (replyText == null)) {
394                                         /* TODO - mark Sone as bad. */
395                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with missing data! ID: %s, Post: %s, Time: %s, Text: %s", sone, replyId, replyPostId, replyTime, replyText));
396                                         return null;
397                                 }
398                                 try {
399                                         PostReplyBuilder postReplyBuilder = core.postReplyBuilder();
400                                         /* TODO - parse time correctly. */
401                                         postReplyBuilder.withId(replyId).from(sone.getId()).to(replyPostId).withTime(Long.parseLong(replyTime)).withText(replyText);
402                                         replies.add(postReplyBuilder.build());
403                                 } catch (NumberFormatException nfe1) {
404                                         /* TODO - mark Sone as bad. */
405                                         logger.log(Level.WARNING, String.format("Downloaded reply for Sone %s with invalid time: %s", sone, replyTime));
406                                         return null;
407                                 }
408                         }
409                 }
410
411                 /* parse liked post IDs. */
412                 SimpleXML likePostIdsXml = soneXml.getNode("post-likes");
413                 Set<String> likedPostIds = new HashSet<String>();
414                 if (likePostIdsXml == null) {
415                         /* TODO - mark Sone as bad. */
416                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no post likes!", sone));
417                 } else {
418                         for (SimpleXML likedPostIdXml : likePostIdsXml.getNodes("post-like")) {
419                                 String postId = likedPostIdXml.getValue();
420                                 likedPostIds.add(postId);
421                         }
422                 }
423
424                 /* parse liked reply IDs. */
425                 SimpleXML likeReplyIdsXml = soneXml.getNode("reply-likes");
426                 Set<String> likedReplyIds = new HashSet<String>();
427                 if (likeReplyIdsXml == null) {
428                         /* TODO - mark Sone as bad. */
429                         logger.log(Level.WARNING, String.format("Downloaded Sone %s has no reply likes!", sone));
430                 } else {
431                         for (SimpleXML likedReplyIdXml : likeReplyIdsXml.getNodes("reply-like")) {
432                                 String replyId = likedReplyIdXml.getValue();
433                                 likedReplyIds.add(replyId);
434                         }
435                 }
436
437                 /* parse albums. */
438                 SimpleXML albumsXml = soneXml.getNode("albums");
439                 List<Album> topLevelAlbums = new ArrayList<Album>();
440                 if (albumsXml != null) {
441                         for (SimpleXML albumXml : albumsXml.getNodes("album")) {
442                                 String id = albumXml.getValue("id", null);
443                                 String parentId = albumXml.getValue("parent", null);
444                                 String title = albumXml.getValue("title", null);
445                                 String description = albumXml.getValue("description", "");
446                                 String albumImageId = albumXml.getValue("album-image", null);
447                                 if ((id == null) || (title == null)) {
448                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid album!", sone));
449                                         return null;
450                                 }
451                                 Album parent = null;
452                                 if (parentId != null) {
453                                         parent = core.getAlbum(parentId);
454                                         if (parent == null) {
455                                                 logger.log(Level.WARNING, String.format("Downloaded Sone %s has album with invalid parent!", sone));
456                                                 return null;
457                                         }
458                                 }
459                                 Album album = core.albumBuilder()
460                                                 .withId(id)
461                                                 .by(sone)
462                                                 .build()
463                                                 .modify()
464                                                 .setTitle(title)
465                                                 .setDescription(description)
466                                                 .update();
467                                 if (parent != null) {
468                                         parent.addAlbum(album);
469                                 } else {
470                                         topLevelAlbums.add(album);
471                                 }
472                                 SimpleXML imagesXml = albumXml.getNode("images");
473                                 if (imagesXml != null) {
474                                         for (SimpleXML imageXml : imagesXml.getNodes("image")) {
475                                                 String imageId = imageXml.getValue("id", null);
476                                                 String imageCreationTimeString = imageXml.getValue("creation-time", null);
477                                                 String imageKey = imageXml.getValue("key", null);
478                                                 String imageTitle = imageXml.getValue("title", null);
479                                                 String imageDescription = imageXml.getValue("description", "");
480                                                 String imageWidthString = imageXml.getValue("width", null);
481                                                 String imageHeightString = imageXml.getValue("height", null);
482                                                 if ((imageId == null) || (imageCreationTimeString == null) || (imageKey == null) || (imageTitle == null) || (imageWidthString == null) || (imageHeightString == null)) {
483                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains invalid images!", sone));
484                                                         return null;
485                                                 }
486                                                 long creationTime = Numbers.safeParseLong(imageCreationTimeString, 0L);
487                                                 int imageWidth = Numbers.safeParseInteger(imageWidthString, 0);
488                                                 int imageHeight = Numbers.safeParseInteger(imageHeightString, 0);
489                                                 if ((imageWidth < 1) || (imageHeight < 1)) {
490                                                         logger.log(Level.WARNING, String.format("Downloaded Sone %s contains image %s with invalid dimensions (%s, %s)!", sone, imageId, imageWidthString, imageHeightString));
491                                                         return null;
492                                                 }
493                                                 Image image = core.getImage(imageId).modify().setSone(sone).setKey(imageKey).setCreationTime(creationTime).update();
494                                                 image = image.modify().setTitle(imageTitle).setDescription(imageDescription).update();
495                                                 image = image.modify().setWidth(imageWidth).setHeight(imageHeight).update();
496                                                 album.addImage(image);
497                                         }
498                                 }
499                                 album.modify().setAlbumImage(albumImageId).update();
500                         }
501                 }
502
503                 /* process avatar. */
504                 if (avatarId != null) {
505                         profile.setAvatar(core.getImage(avatarId, false));
506                 }
507
508                 /* okay, apparently everything was parsed correctly. Now import. */
509                 /* atomic setter operation on the Sone. */
510                 synchronized (sone) {
511                         sone.setProfile(profile);
512                         sone.setPosts(posts);
513                         sone.setReplies(replies);
514                         sone.setLikePostIds(likedPostIds);
515                         sone.setLikeReplyIds(likedReplyIds);
516                         for (Album album : topLevelAlbums) {
517                                 sone.getRootAlbum().addAlbum(album);
518                         }
519                 }
520
521                 return sone;
522         }
523
524         @Override
525         public Runnable fetchSoneWithUriAction(final Sone sone) {
526                 return new Runnable() {
527                         @Override
528                         public void run() {
529                                 fetchSone(sone, sone.getRequestUri());
530                         }
531                 };
532         }
533
534         @Override
535         public Runnable fetchSoneAction(final Sone sone) {
536                 return new Runnable() {
537                         @Override
538                         public void run() {
539                                 fetchSone(sone);
540                         }
541                 };
542         }
543
544         /** {@inheritDoc} */
545         @Override
546         protected void serviceStop() {
547                 for (Sone sone : sones) {
548                         freenetInterface.unregisterUsk(sone);
549                 }
550         }
551
552 }