Make all fields private.
[jSite.git] / src / main / java / de / todesbaum / jsite / application / ProjectInserter.java
1 /*
2  * jSite - ProjectInserter.java - Copyright © 2006–2012 David Roden
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17  */
18
19 package de.todesbaum.jsite.application;
20
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.FileNotFoundException;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.HashSet;
29 import java.util.Iterator;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Map.Entry;
33 import java.util.Set;
34 import java.util.concurrent.CountDownLatch;
35 import java.util.logging.Level;
36 import java.util.logging.Logger;
37
38 import net.pterodactylus.util.io.StreamCopier.ProgressListener;
39
40 import com.google.common.base.Optional;
41 import de.todesbaum.jsite.gui.FileScanner;
42 import de.todesbaum.jsite.gui.FileScanner.ScannedFile;
43 import de.todesbaum.jsite.gui.FileScannerListener;
44 import de.todesbaum.util.freenet.fcp2.Client;
45 import de.todesbaum.util.freenet.fcp2.ClientPutComplexDir;
46 import de.todesbaum.util.freenet.fcp2.ClientPutDir.ManifestPutter;
47 import de.todesbaum.util.freenet.fcp2.Connection;
48 import de.todesbaum.util.freenet.fcp2.DirectFileEntry;
49 import de.todesbaum.util.freenet.fcp2.FileEntry;
50 import de.todesbaum.util.freenet.fcp2.Message;
51 import de.todesbaum.util.freenet.fcp2.PriorityClass;
52 import de.todesbaum.util.freenet.fcp2.RedirectFileEntry;
53 import de.todesbaum.util.freenet.fcp2.Verbosity;
54
55 /**
56  * Manages project inserts.
57  *
58  * @author David ‘Bombe’ Roden <bombe@freenetproject.org>
59  */
60 public class ProjectInserter implements FileScannerListener, Runnable {
61
62         /** The logger. */
63         private static final Logger logger = Logger.getLogger(ProjectInserter.class.getName());
64
65         /** Random number for FCP instances. */
66         private static final int random = (int) (Math.random() * Integer.MAX_VALUE);
67
68         /** Counter for FCP connection identifier. */
69         private static int counter = 0;
70
71         /** The list of insert listeners. */
72         private List<InsertListener> insertListeners = new ArrayList<InsertListener>();
73
74         /** The freenet interface. */
75         private Freenet7Interface freenetInterface;
76
77         /** The project to insert. */
78         private Project project;
79
80         /** The file scanner. */
81         private FileScanner fileScanner;
82
83         /** Object used for synchronization. */
84         private final Object lockObject = new Object();
85
86         /** The temp directory. */
87         private String tempDirectory;
88
89         /** The current connection. */
90         private Connection connection;
91
92         /** Whether the insert is cancelled. */
93         private volatile boolean cancelled = false;
94
95         /** Progress listener for payload transfers. */
96         private ProgressListener progressListener;
97
98         /** Whether to use “early encode.” */
99         private boolean useEarlyEncode;
100
101         /** The insert priority. */
102         private PriorityClass priority;
103
104         /** The manifest putter. */
105         private ManifestPutter manifestPutter;
106
107         /**
108          * Adds a listener to the list of registered listeners.
109          *
110          * @param insertListener
111          *            The listener to add
112          */
113         public void addInsertListener(InsertListener insertListener) {
114                 insertListeners.add(insertListener);
115         }
116
117         /**
118          * Removes a listener from the list of registered listeners.
119          *
120          * @param insertListener
121          *            The listener to remove
122          */
123         public void removeInsertListener(InsertListener insertListener) {
124                 insertListeners.remove(insertListener);
125         }
126
127         /**
128          * Notifies all listeners that the project insert has started.
129          *
130          * @see InsertListener#projectInsertStarted(Project)
131          */
132         protected void fireProjectInsertStarted() {
133                 for (InsertListener insertListener : insertListeners) {
134                         insertListener.projectInsertStarted(project);
135                 }
136         }
137
138         /**
139          * Notifies all listeners that the insert has generated a URI.
140          *
141          * @see InsertListener#projectURIGenerated(Project, String)
142          * @param uri
143          *            The generated URI
144          */
145         protected void fireProjectURIGenerated(String uri) {
146                 for (InsertListener insertListener : insertListeners) {
147                         insertListener.projectURIGenerated(project, uri);
148                 }
149         }
150
151         /**
152          * Notifies all listeners that the insert has made some progress.
153          *
154          * @see InsertListener#projectUploadFinished(Project)
155          */
156         protected void fireProjectUploadFinished() {
157                 for (InsertListener insertListener : insertListeners) {
158                         insertListener.projectUploadFinished(project);
159                 }
160         }
161
162         /**
163          * Notifies all listeners that the insert has made some progress.
164          *
165          * @see InsertListener#projectInsertProgress(Project, int, int, int, int,
166          *      boolean)
167          * @param succeeded
168          *            The number of succeeded blocks
169          * @param failed
170          *            The number of failed blocks
171          * @param fatal
172          *            The number of fatally failed blocks
173          * @param total
174          *            The total number of blocks
175          * @param finalized
176          *            <code>true</code> if the total number of blocks has already
177          *            been finalized, <code>false</code> otherwise
178          */
179         protected void fireProjectInsertProgress(int succeeded, int failed, int fatal, int total, boolean finalized) {
180                 for (InsertListener insertListener : insertListeners) {
181                         insertListener.projectInsertProgress(project, succeeded, failed, fatal, total, finalized);
182                 }
183         }
184
185         /**
186          * Notifies all listeners the project insert has finished.
187          *
188          * @see InsertListener#projectInsertFinished(Project, boolean, Throwable)
189          * @param success
190          *            <code>true</code> if the project was inserted successfully,
191          *            <code>false</code> if it failed
192          * @param cause
193          *            The cause of the failure, if any
194          */
195         protected void fireProjectInsertFinished(boolean success, Throwable cause) {
196                 for (InsertListener insertListener : insertListeners) {
197                         insertListener.projectInsertFinished(project, success, cause);
198                 }
199         }
200
201         /**
202          * Sets the project to insert.
203          *
204          * @param project
205          *            The project to insert
206          */
207         public void setProject(Project project) {
208                 this.project = project;
209         }
210
211         /**
212          * Sets the freenet interface to use.
213          *
214          * @param freenetInterface
215          *            The freenet interface to use
216          */
217         public void setFreenetInterface(Freenet7Interface freenetInterface) {
218                 this.freenetInterface = freenetInterface;
219         }
220
221         /**
222          * Sets the temp directory to use.
223          *
224          * @param tempDirectory
225          *            The temp directory to use, or {@code null} to use the system
226          *            default
227          */
228         public void setTempDirectory(String tempDirectory) {
229                 this.tempDirectory = tempDirectory;
230         }
231
232         /**
233          * Sets whether to use the “early encode“ flag for the insert.
234          *
235          * @param useEarlyEncode
236          *            {@code true} to set the “early encode” flag for the insert,
237          *            {@code false} otherwise
238          */
239         public void setUseEarlyEncode(boolean useEarlyEncode) {
240                 this.useEarlyEncode = useEarlyEncode;
241         }
242
243         /**
244          * Sets the insert priority.
245          *
246          * @param priority
247          *            The insert priority
248          */
249         public void setPriority(PriorityClass priority) {
250                 this.priority = priority;
251         }
252
253         /**
254          * Sets the manifest putter to use for inserts.
255          *
256          * @param manifestPutter
257          *            The manifest putter to use
258          */
259         public void setManifestPutter(ManifestPutter manifestPutter) {
260                 this.manifestPutter = manifestPutter;
261         }
262
263         /**
264          * Starts the insert.
265          *
266          * @param progressListener
267          *            Listener to notify on progress events
268          */
269         public void start(ProgressListener progressListener) {
270                 cancelled = false;
271                 this.progressListener = progressListener;
272                 fileScanner = new FileScanner(project);
273                 fileScanner.addFileScannerListener(this);
274                 new Thread(fileScanner).start();
275         }
276
277         /**
278          * Stops the current insert.
279          */
280         public void stop() {
281                 cancelled = true;
282                 synchronized (lockObject) {
283                         if (connection != null) {
284                                 connection.disconnect();
285                         }
286                 }
287         }
288
289         /**
290          * Creates a file entry suitable for handing in to
291          * {@link ClientPutComplexDir#addFileEntry(FileEntry)}.
292          *
293          * @param file
294          *            The name and hash of the file to insert
295          * @param edition
296          *            The current edition
297          * @return A file entry for the given file
298          */
299         private FileEntry createFileEntry(ScannedFile file, int edition) {
300                 String filename = file.getFilename();
301                 FileOption fileOption = project.getFileOption(filename);
302                 if (fileOption.isInsert()) {
303                         fileOption.setCurrentHash(file.getHash());
304                         /* check if file was modified. */
305                         if (!project.isAlwaysForceInsert() && !fileOption.isForceInsert() && file.getHash().equals(fileOption.getLastInsertHash())) {
306                                 /* only insert a redirect. */
307                                 logger.log(Level.FINE, String.format("Inserting redirect to edition %d for %s.", fileOption.getLastInsertEdition(), filename));
308                                 return new RedirectFileEntry(fileOption.getChangedName().or(filename), fileOption.getMimeType(), "SSK@" + project.getRequestURI() + "/" + project.getPath() + "-" + fileOption.getLastInsertEdition() + "/" + fileOption.getLastInsertFilename());
309                         }
310                         try {
311                                 return createFileEntry(filename, fileOption.getChangedName(), fileOption.getMimeType());
312                         } catch (IOException ioe1) {
313                                 /* ignore, null is returned. */
314                         }
315                 } else {
316                         if (fileOption.isInsertRedirect()) {
317                                 return new RedirectFileEntry(fileOption.getChangedName().or(filename), fileOption.getMimeType(), fileOption.getCustomKey());
318                         }
319                 }
320                 return null;
321         }
322
323         private FileEntry createFileEntry(String filename, Optional<String> changedName, String mimeType) throws FileNotFoundException {
324                 File physicalFile = new File(project.getLocalPath(), filename);
325                 InputStream fileEntryInputStream = new FileInputStream(physicalFile);
326                 return new DirectFileEntry(changedName.or(filename), mimeType, fileEntryInputStream, physicalFile.length());
327         }
328
329         /**
330          * Validates the given project. The project will be checked for any invalid
331          * conditions, such as invalid insert or request keys, missing path names,
332          * missing default file, and so on.
333          *
334          * @param project
335          *            The project to check
336          * @return The encountered warnings and errors
337          */
338         public static CheckReport validateProject(Project project) {
339                 CheckReport checkReport = new CheckReport();
340                 if ((project.getLocalPath() == null) || (project.getLocalPath().trim().length() == 0)) {
341                         checkReport.addIssue("error.no-local-path", true);
342                 }
343                 if ((project.getPath() == null) || (project.getPath().trim().length() == 0)) {
344                         checkReport.addIssue("error.no-path", true);
345                 }
346                 if ((project.getIndexFile() == null) || (project.getIndexFile().length() == 0)) {
347                         checkReport.addIssue("warning.empty-index", false);
348                 } else {
349                         File indexFile = new File(project.getLocalPath(), project.getIndexFile());
350                         if (!indexFile.exists()) {
351                                 checkReport.addIssue("error.index-missing", true);
352                         }
353                 }
354                 String indexFile = project.getIndexFile();
355                 boolean hasIndexFile = (indexFile != null) && (indexFile.length() > 0);
356                 List<String> allowedIndexContentTypes = Arrays.asList("text/html", "application/xhtml+xml");
357                 if (hasIndexFile && !allowedIndexContentTypes.contains(project.getFileOption(indexFile).getMimeType())) {
358                         checkReport.addIssue("warning.index-not-html", false);
359                 }
360                 Map<String, FileOption> fileOptions = project.getFileOptions();
361                 Set<Entry<String, FileOption>> fileOptionEntries = fileOptions.entrySet();
362                 boolean insert = fileOptionEntries.isEmpty();
363                 for (Entry<String, FileOption> fileOptionEntry : fileOptionEntries) {
364                         String fileName = fileOptionEntry.getKey();
365                         FileOption fileOption = fileOptionEntry.getValue();
366                         insert |= fileOption.isInsert() || fileOption.isInsertRedirect();
367                         if (fileName.equals(project.getIndexFile()) && !fileOption.isInsert() && !fileOption.isInsertRedirect()) {
368                                 checkReport.addIssue("error.index-not-inserted", true);
369                         }
370                         if (!fileOption.isInsert() && fileOption.isInsertRedirect() && ((fileOption.getCustomKey().length() == 0) || "CHK@".equals(fileOption.getCustomKey()))) {
371                                 checkReport.addIssue("error.no-custom-key", true, fileName);
372                         }
373                 }
374                 if (!insert) {
375                         checkReport.addIssue("error.no-files-to-insert", true);
376                 }
377                 Set<String> fileNames = new HashSet<String>();
378                 for (Entry<String, FileOption> fileOptionEntry : fileOptionEntries) {
379                         FileOption fileOption = fileOptionEntry.getValue();
380                         if (!fileOption.isInsert() && !fileOption.isInsertRedirect()) {
381                                 logger.log(Level.FINEST, "Ignoring {0}.", fileOptionEntry.getKey());
382                                 continue;
383                         }
384                         String fileName = fileOption.getChangedName().or(fileOptionEntry.getKey());
385                         logger.log(Level.FINEST, "Adding “{0}” for {1}.", new Object[] { fileName, fileOptionEntry.getKey() });
386                         if (!fileNames.add(fileName)) {
387                                 checkReport.addIssue("error.duplicate-file", true, fileName);
388                         }
389                 }
390                 long totalSize = 0;
391                 FileScanner fileScanner = new FileScanner(project);
392                 final CountDownLatch completionLatch = new CountDownLatch(1);
393                 fileScanner.addFileScannerListener(new FileScannerListener() {
394
395                         @Override
396                         public void fileScannerFinished(FileScanner fileScanner) {
397                                 completionLatch.countDown();
398                         }
399                 });
400                 new Thread(fileScanner).start();
401                 while (completionLatch.getCount() > 0) {
402                         try {
403                                 completionLatch.await();
404                         } catch (InterruptedException ie1) {
405                                 /* TODO: logging */
406                         }
407                 }
408                 for (ScannedFile scannedFile : fileScanner.getFiles()) {
409                         String fileName = scannedFile.getFilename();
410                         FileOption fileOption = project.getFileOption(fileName);
411                         if ((fileOption != null) && !fileOption.isInsert()) {
412                                 continue;
413                         }
414                         totalSize += new File(project.getLocalPath(), fileName).length();
415                 }
416                 if (totalSize > 2 * 1024 * 1024) {
417                         checkReport.addIssue("warning.site-larger-than-2-mib", false);
418                 }
419                 return checkReport;
420         }
421
422         /**
423          * {@inheritDoc}
424          */
425         @Override
426         public void run() {
427                 fireProjectInsertStarted();
428                 List<ScannedFile> files = fileScanner.getFiles();
429
430                 /* create connection to node */
431                 synchronized (lockObject) {
432                         connection = freenetInterface.getConnection("project-insert-" + random + counter++);
433                 }
434                 connection.setTempDirectory(tempDirectory);
435                 boolean connected = false;
436                 Throwable cause = null;
437                 try {
438                         connected = connection.connect();
439                 } catch (IOException e1) {
440                         cause = e1;
441                 }
442
443                 if (!connected || cancelled) {
444                         fireProjectInsertFinished(false, cancelled ? new AbortedException() : cause);
445                         return;
446                 }
447
448                 Client client = new Client(connection);
449
450                 /* collect files */
451                 int edition = project.getEdition();
452                 String dirURI = "USK@" + project.getInsertURI() + "/" + project.getPath() + "/" + edition + "/";
453                 ClientPutComplexDir putDir = new ClientPutComplexDir("dir-" + counter++, dirURI, tempDirectory);
454                 if ((project.getIndexFile() != null) && (project.getIndexFile().length() > 0)) {
455                         putDir.setDefaultName(project.getIndexFile());
456                 }
457                 putDir.setVerbosity(Verbosity.ALL);
458                 putDir.setMaxRetries(-1);
459                 putDir.setEarlyEncode(useEarlyEncode);
460                 putDir.setPriorityClass(priority);
461                 putDir.setManifestPutter(manifestPutter);
462                 for (ScannedFile file : files) {
463                         FileEntry fileEntry = createFileEntry(file, edition);
464                         if (fileEntry != null) {
465                                 try {
466                                         putDir.addFileEntry(fileEntry);
467                                 } catch (IOException ioe1) {
468                                         fireProjectInsertFinished(false, ioe1);
469                                         return;
470                                 }
471                         }
472                 }
473
474                 /* start request */
475                 try {
476                         client.execute(putDir, progressListener);
477                         fireProjectUploadFinished();
478                 } catch (IOException ioe1) {
479                         fireProjectInsertFinished(false, ioe1);
480                         return;
481                 }
482
483                 /* parse progress and success messages */
484                 String finalURI = null;
485                 boolean success = false;
486                 boolean finished = false;
487                 boolean disconnected = false;
488                 while (!finished && !cancelled) {
489                         Message message = client.readMessage();
490                         finished = (message == null) || (disconnected = client.isDisconnected());
491                         logger.log(Level.FINE, "Received message: " + message);
492                         if (!finished) {
493                                 @SuppressWarnings("null")
494                                 String messageName = message.getName();
495                                 if ("URIGenerated".equals(messageName)) {
496                                         finalURI = message.get("URI");
497                                         fireProjectURIGenerated(finalURI);
498                                 }
499                                 if ("SimpleProgress".equals(messageName)) {
500                                         int total = Integer.parseInt(message.get("Total"));
501                                         int succeeded = Integer.parseInt(message.get("Succeeded"));
502                                         int fatal = Integer.parseInt(message.get("FatallyFailed"));
503                                         int failed = Integer.parseInt(message.get("Failed"));
504                                         boolean finalized = Boolean.parseBoolean(message.get("FinalizedTotal"));
505                                         fireProjectInsertProgress(succeeded, failed, fatal, total, finalized);
506                                 }
507                                 success |= "PutSuccessful".equals(messageName);
508                                 finished = (success && (finalURI != null)) || "PutFailed".equals(messageName) || messageName.endsWith("Error");
509                         }
510                 }
511
512                 /* post-insert work */
513                 if (success) {
514                         @SuppressWarnings("null")
515                         String editionPart = finalURI.substring(finalURI.lastIndexOf('/') + 1);
516                         int newEdition = Integer.parseInt(editionPart);
517                         project.setEdition(newEdition);
518                         project.setLastInsertionTime(System.currentTimeMillis());
519                         project.onSuccessfulInsert();
520                 }
521                 fireProjectInsertFinished(success, cancelled ? new AbortedException() : (disconnected ? new IOException("Connection terminated") : null));
522         }
523
524         //
525         // INTERFACE FileScannerListener
526         //
527
528         /**
529          * {@inheritDoc}
530          */
531         @Override
532         public void fileScannerFinished(FileScanner fileScanner) {
533                 if (!fileScanner.isError()) {
534                         new Thread(this).start();
535                 } else {
536                         fireProjectInsertFinished(false, null);
537                 }
538                 fileScanner.removeFileScannerListener(this);
539         }
540
541         /**
542          * Container class that collects all warnings and errors that occured during
543          * {@link ProjectInserter#validateProject(Project) project validation}.
544          *
545          * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
546          */
547         public static class CheckReport implements Iterable<Issue> {
548
549                 /** The issures that occured. */
550                 private final List<Issue> issues = new ArrayList<Issue>();
551
552                 /**
553                  * Adds an issue.
554                  *
555                  * @param issue
556                  *            The issue to add
557                  */
558                 public void addIssue(Issue issue) {
559                         issues.add(issue);
560                 }
561
562                 /**
563                  * Creates an {@link Issue} from the given error key and fatality flag
564                  * and {@link #addIssue(Issue) adds} it.
565                  *
566                  * @param errorKey
567                  *            The error key
568                  * @param fatal
569                  *            {@code true} if the error is fatal, {@code false} if only
570                  *            a warning should be generated
571                  * @param parameters
572                  *            Any additional parameters
573                  */
574                 public void addIssue(String errorKey, boolean fatal, String... parameters) {
575                         addIssue(new Issue(errorKey, fatal, parameters));
576                 }
577
578                 /**
579                  * {@inheritDoc}
580                  */
581                 @Override
582                 public Iterator<Issue> iterator() {
583                         return issues.iterator();
584                 }
585
586                 /**
587                  * Returns whether this check report does not contain any errors.
588                  *
589                  * @return {@code true} if this check report does not contain any
590                  *         errors, {@code false} if this check report does contain
591                  *         errors
592                  */
593                 public boolean isEmpty() {
594                         return issues.isEmpty();
595                 }
596
597                 /**
598                  * Returns the number of issues in this check report.
599                  *
600                  * @return The number of issues
601                  */
602                 public int size() {
603                         return issues.size();
604                 }
605
606         }
607
608         /**
609          * Container class for a single issue. An issue contains an error key
610          * that describes the error, and a fatality flag that determines whether
611          * the insert has to be aborted (if the flag is {@code true}) or if it
612          * can still be performed and only a warning should be generated (if the
613          * flag is {@code false}).
614          *
615          * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’
616          *         Roden</a>
617          */
618         public static class Issue {
619
620                 /** The error key. */
621                 private final String errorKey;
622
623                 /** The fatality flag. */
624                 private final boolean fatal;
625
626                 /** Additional parameters. */
627                 private String[] parameters;
628
629                 /**
630                  * Creates a new issue.
631                  *
632                  * @param errorKey
633                  *            The error key
634                  * @param fatal
635                  *            The fatality flag
636                  * @param parameters
637                  *            Any additional parameters
638                  */
639                 protected Issue(String errorKey, boolean fatal, String... parameters) {
640                         this.errorKey = errorKey;
641                         this.fatal = fatal;
642                         this.parameters = parameters;
643                 }
644
645                 /**
646                  * Returns the key of the encountered error.
647                  *
648                  * @return The error key
649                  */
650                 public String getErrorKey() {
651                         return errorKey;
652                 }
653
654                 /**
655                  * Returns whether the issue is fatal and the insert has to be
656                  * aborted. Otherwise only a warning should be shown.
657                  *
658                  * @return {@code true} if the insert needs to be aborted, {@code
659                  *         false} otherwise
660                  */
661                 public boolean isFatal() {
662                         return fatal;
663                 }
664
665                 /**
666                  * Returns any additional parameters.
667                  *
668                  * @return The additional parameters
669                  */
670                 public String[] getParameters() {
671                         return parameters;
672                 }
673
674         }
675
676 }