Factor out project validation, re-order warning and error messages.
[jSite.git] / src / de / todesbaum / jsite / application / ProjectInserter.java
1 /*
2  * jSite - a tool for uploading websites into Freenet
3  * Copyright (C) 2006 David Roden
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18  */
19
20 package de.todesbaum.jsite.application;
21
22 import java.io.ByteArrayInputStream;
23 import java.io.ByteArrayOutputStream;
24 import java.io.File;
25 import java.io.FileInputStream;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.HashMap;
32 import java.util.Iterator;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.Set;
36 import java.util.Map.Entry;
37 import java.util.logging.Level;
38 import java.util.logging.Logger;
39 import java.util.zip.ZipEntry;
40 import java.util.zip.ZipOutputStream;
41
42 import de.todesbaum.jsite.application.ProjectInserter.CheckReport.Issue;
43 import de.todesbaum.jsite.gui.FileScanner;
44 import de.todesbaum.jsite.gui.FileScannerListener;
45 import de.todesbaum.util.freenet.fcp2.Client;
46 import de.todesbaum.util.freenet.fcp2.ClientPutComplexDir;
47 import de.todesbaum.util.freenet.fcp2.Connection;
48 import de.todesbaum.util.freenet.fcp2.DirectFileEntry;
49 import de.todesbaum.util.freenet.fcp2.FileEntry;
50 import de.todesbaum.util.freenet.fcp2.Message;
51 import de.todesbaum.util.freenet.fcp2.RedirectFileEntry;
52 import de.todesbaum.util.freenet.fcp2.Verbosity;
53 import de.todesbaum.util.io.Closer;
54 import de.todesbaum.util.io.ReplacingOutputStream;
55 import de.todesbaum.util.io.StreamCopier;
56
57 /**
58  * Manages project inserts.
59  *
60  * @author David ‘Bombe’ Roden <bombe@freenetproject.org>
61  */
62 public class ProjectInserter implements FileScannerListener, Runnable {
63
64         /** The logger. */
65         private static final Logger logger = Logger.getLogger(ProjectInserter.class.getName());
66
67         /** Random number for FCP instances. */
68         private static final int random = (int) (Math.random() * Integer.MAX_VALUE);
69
70         /** Counter for FCP connection identifier. */
71         private static int counter = 0;
72
73         /** The list of insert listeners. */
74         private List<InsertListener> insertListeners = new ArrayList<InsertListener>();
75
76         /** The freenet interface. */
77         protected Freenet7Interface freenetInterface;
78
79         /** The project to insert. */
80         protected Project project;
81
82         /** The file scanner. */
83         private FileScanner fileScanner;
84
85         /** Object used for synchronization. */
86         protected final Object lockObject = new Object();
87
88         /** The temp directory. */
89         private String tempDirectory;
90
91         /**
92          * Adds a listener to the list of registered listeners.
93          *
94          * @param insertListener
95          *            The listener to add
96          */
97         public void addInsertListener(InsertListener insertListener) {
98                 insertListeners.add(insertListener);
99         }
100
101         /**
102          * Removes a listener from the list of registered listeners.
103          *
104          * @param insertListener
105          *            The listener to remove
106          */
107         public void removeInsertListener(InsertListener insertListener) {
108                 insertListeners.remove(insertListener);
109         }
110
111         /**
112          * Notifies all listeners that the project insert has started.
113          *
114          * @see InsertListener#projectInsertStarted(Project)
115          */
116         protected void fireProjectInsertStarted() {
117                 for (InsertListener insertListener : insertListeners) {
118                         insertListener.projectInsertStarted(project);
119                 }
120         }
121
122         /**
123          * Notifies all listeners that the insert has generated a URI.
124          *
125          * @see InsertListener#projectURIGenerated(Project, String)
126          * @param uri
127          *            The generated URI
128          */
129         protected void fireProjectURIGenerated(String uri) {
130                 for (InsertListener insertListener : insertListeners) {
131                         insertListener.projectURIGenerated(project, uri);
132                 }
133         }
134
135         /**
136          * Notifies all listeners that the insert has made some progress.
137          *
138          * @see InsertListener#projectUploadFinished(Project)
139          */
140         protected void fireProjectUploadFinished() {
141                 for (InsertListener insertListener : insertListeners) {
142                         insertListener.projectUploadFinished(project);
143                 }
144         }
145
146         /**
147          * Notifies all listeners that the insert has made some progress.
148          *
149          * @see InsertListener#projectInsertProgress(Project, int, int, int, int,
150          *      boolean)
151          * @param succeeded
152          *            The number of succeeded blocks
153          * @param failed
154          *            The number of failed blocks
155          * @param fatal
156          *            The number of fatally failed blocks
157          * @param total
158          *            The total number of blocks
159          * @param finalized
160          *            <code>true</code> if the total number of blocks has already
161          *            been finalized, <code>false</code> otherwise
162          */
163         protected void fireProjectInsertProgress(int succeeded, int failed, int fatal, int total, boolean finalized) {
164                 for (InsertListener insertListener : insertListeners) {
165                         insertListener.projectInsertProgress(project, succeeded, failed, fatal, total, finalized);
166                 }
167         }
168
169         /**
170          * Notifies all listeners the project insert has finished.
171          *
172          * @see InsertListener#projectInsertFinished(Project, boolean, Throwable)
173          * @param success
174          *            <code>true</code> if the project was inserted successfully,
175          *            <code>false</code> if it failed
176          * @param cause
177          *            The cause of the failure, if any
178          */
179         protected void fireProjectInsertFinished(boolean success, Throwable cause) {
180                 for (InsertListener insertListener : insertListeners) {
181                         insertListener.projectInsertFinished(project, success, cause);
182                 }
183         }
184
185         /**
186          * Sets the project to insert.
187          *
188          * @param project
189          *            The project to insert
190          */
191         public void setProject(Project project) {
192                 this.project = project;
193         }
194
195         /**
196          * Sets the freenet interface to use.
197          *
198          * @param freenetInterface
199          *            The freenet interface to use
200          */
201         public void setFreenetInterface(Freenet7Interface freenetInterface) {
202                 this.freenetInterface = freenetInterface;
203         }
204
205         /**
206          * Sets the temp directory to use.
207          *
208          * @param tempDirectory
209          *            The temp directory to use, or {@code null} to use the system
210          *            default
211          */
212         public void setTempDirectory(String tempDirectory) {
213                 this.tempDirectory = tempDirectory;
214         }
215
216         /**
217          * Starts the insert.
218          */
219         public void start() {
220                 fileScanner = new FileScanner(project);
221                 fileScanner.addFileScannerListener(this);
222                 new Thread(fileScanner).start();
223         }
224
225         /**
226          * Creates an input stream that delivers the given file, replacing edition
227          * tokens in the file’s content, if necessary.
228          *
229          * @param filename
230          *            The name of the file
231          * @param fileOption
232          *            The file options
233          * @param edition
234          *            The current edition
235          * @param length
236          *            An array containing a single long which is used to
237          *            <em>return</em> the final length of the file, after all
238          *            replacements
239          * @return The input stream for the file
240          * @throws IOException
241          *             if an I/O error occurs
242          */
243         private InputStream createFileInputStream(String filename, FileOption fileOption, int edition, long[] length) throws IOException {
244                 File file = new File(project.getLocalPath(), filename);
245                 length[0] = file.length();
246                 if (!fileOption.getReplaceEdition()) {
247                         return new FileInputStream(file);
248                 }
249                 ByteArrayOutputStream filteredByteOutputStream = new ByteArrayOutputStream(Math.min(Integer.MAX_VALUE, (int) length[0]));
250                 ReplacingOutputStream outputStream = new ReplacingOutputStream(filteredByteOutputStream);
251                 FileInputStream fileInput = new FileInputStream(file);
252                 try {
253                         outputStream.addReplacement("$[EDITION]", String.valueOf(edition));
254                         outputStream.addReplacement("$[URI]", project.getFinalRequestURI(0));
255                         for (int index = 1; index <= fileOption.getEditionRange(); index++) {
256                                 outputStream.addReplacement("$[URI+" + index + "]", project.getFinalRequestURI(index));
257                                 outputStream.addReplacement("$[EDITION+" + index + "]", String.valueOf(edition + index));
258                         }
259                         StreamCopier.copy(fileInput, outputStream, length[0]);
260                 } finally {
261                         Closer.close(fileInput);
262                         Closer.close(outputStream);
263                         Closer.close(filteredByteOutputStream);
264                 }
265                 byte[] filteredBytes = filteredByteOutputStream.toByteArray();
266                 length[0] = filteredBytes.length;
267                 return new ByteArrayInputStream(filteredBytes);
268         }
269
270         /**
271          * Creates an input stream for a container.
272          *
273          * @param containerFiles
274          *            All container definitions
275          * @param containerName
276          *            The name of the container to create
277          * @param edition
278          *            The current edition
279          * @param containerLength
280          *            An array containing a single long which is used to
281          *            <em>return</em> the final length of the container stream,
282          *            after all replacements
283          * @return The input stream for the container
284          * @throws IOException
285          *             if an I/O error occurs
286          */
287         private InputStream createContainerInputStream(Map<String, List<String>> containerFiles, String containerName, int edition, long[] containerLength) throws IOException {
288                 File tempFile = File.createTempFile("jsite", ".zip", (tempDirectory == null) ? null : new File(tempDirectory));
289                 tempFile.deleteOnExit();
290                 FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
291                 ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
292                 try {
293                         for (String filename : containerFiles.get(containerName)) {
294                                 File dataFile = new File(project.getLocalPath(), filename);
295                                 if (dataFile.exists()) {
296                                         ZipEntry zipEntry = new ZipEntry(filename);
297                                         long[] fileLength = new long[1];
298                                         InputStream wrappedInputStream = createFileInputStream(filename, project.getFileOption(filename), edition, fileLength);
299                                         try {
300                                                 zipOutputStream.putNextEntry(zipEntry);
301                                                 StreamCopier.copy(wrappedInputStream, zipOutputStream, fileLength[0]);
302                                         } finally {
303                                                 zipOutputStream.closeEntry();
304                                                 wrappedInputStream.close();
305                                         }
306                                 }
307                         }
308                 } finally {
309                         zipOutputStream.closeEntry();
310                         Closer.close(zipOutputStream);
311                         Closer.close(fileOutputStream);
312                 }
313
314                 containerLength[0] = tempFile.length();
315                 return new FileInputStream(tempFile);
316         }
317
318         /**
319          * Creates a file entry suitable for handing in to
320          * {@link ClientPutComplexDir#addFileEntry(FileEntry)}.
321          *
322          * @param filename
323          *            The name of the file to insert
324          * @param edition
325          *            The current edition
326          * @param containerFiles
327          *            The container definitions
328          * @return A file entry for the given file
329          */
330         private FileEntry createFileEntry(String filename, int edition, Map<String, List<String>> containerFiles) {
331                 FileEntry fileEntry = null;
332                 FileOption fileOption = project.getFileOption(filename);
333                 if (filename.startsWith("/container/:")) {
334                         String containerName = filename.substring("/container/:".length());
335                         try {
336                                 long[] containerLength = new long[1];
337                                 InputStream containerInputStream = createContainerInputStream(containerFiles, containerName, edition, containerLength);
338                                 fileEntry = new DirectFileEntry(containerName + ".zip", "application/zip", containerInputStream, containerLength[0]);
339                         } catch (IOException ioe1) {
340                                 /* ignore, null is returned. */
341                         }
342                 } else {
343                         if (fileOption.isInsert()) {
344                                 try {
345                                         long[] fileLength = new long[1];
346                                         InputStream fileEntryInputStream = createFileInputStream(filename, fileOption, edition, fileLength);
347                                         fileEntry = new DirectFileEntry(filename, project.getFileOption(filename).getMimeType(), fileEntryInputStream, fileLength[0]);
348                                 } catch (IOException ioe1) {
349                                         /* ignore, null is returned. */
350                                 }
351                         } else {
352                                 if (fileOption.isInsertRedirect()) {
353                                         fileEntry = new RedirectFileEntry(filename, fileOption.getMimeType(), fileOption.getCustomKey());
354                                 }
355                         }
356                 }
357                 return fileEntry;
358         }
359
360         /**
361          * Creates container definitions.
362          *
363          * @param files
364          *            The list of all files
365          * @param containers
366          *            The list of all containers
367          * @param containerFiles
368          *            Empty map that will be filled with container definitions
369          */
370         private void createContainers(List<String> files, List<String> containers, Map<String, List<String>> containerFiles) {
371                 for (String filename : new ArrayList<String>(files)) {
372                         FileOption fileOption = project.getFileOption(filename);
373                         String containerName = fileOption.getContainer();
374                         if (!containerName.equals("")) {
375                                 if (!containers.contains(containerName)) {
376                                         containers.add(containerName);
377                                         containerFiles.put(containerName, new ArrayList<String>());
378                                         /* hmm. looks like a hack to me. */
379                                         files.add("/container/:" + containerName);
380                                 }
381                                 containerFiles.get(containerName).add(filename);
382                                 files.remove(filename);
383                         }
384                 }
385         }
386
387         /**
388          * Validates the given project. The project will be checked for any invalid
389          * conditions, such as invalid insert or request keys, missing path names,
390          * missing default file, and so on.
391          *
392          * @param project
393          *            The project to check
394          * @return The encountered warnings and errors
395          */
396         public static CheckReport validateProject(Project project) {
397                 CheckReport checkReport = new CheckReport();
398                 if ((project.getLocalPath() == null) || (project.getLocalPath().trim().length() == 0)) {
399                         checkReport.addIssue("error.no-local-path", true);
400                 }
401                 if ((project.getPath() == null) || (project.getPath().trim().length() == 0)) {
402                         checkReport.addIssue("error.no-path", true);
403                 }
404                 if ((project.getIndexFile() == null) || (project.getIndexFile().length() == 0)) {
405                         checkReport.addIssue("warning.empty-index", false);
406                 } else {
407                         File indexFile = new File(project.getLocalPath(), project.getIndexFile());
408                         if (!indexFile.exists()) {
409                                 checkReport.addIssue("error.index-missing", true);
410                         }
411                 }
412                 String indexFile = project.getIndexFile();
413                 boolean hasIndexFile = (indexFile != null);
414                 if (hasIndexFile && !project.getFileOption(indexFile).getContainer().equals("")) {
415                         checkReport.addIssue("warning.container-index", false);
416                 }
417                 List<String> allowedIndexContentTypes = Arrays.asList("text/html", "application/xhtml+xml");
418                 if (hasIndexFile && !allowedIndexContentTypes.contains(project.getFileOption(indexFile).getMimeType())) {
419                         checkReport.addIssue("warning.index-not-html", false);
420                 }
421                 Map<String, FileOption> fileOptions = project.getFileOptions();
422                 Set<Entry<String, FileOption>> fileOptionEntries = fileOptions.entrySet();
423                 boolean insert = false;
424                 for (Entry<String, FileOption> fileOptionEntry : fileOptionEntries) {
425                         String fileName = fileOptionEntry.getKey();
426                         FileOption fileOption = fileOptionEntry.getValue();
427                         insert |= fileOption.isInsert() || fileOption.isInsertRedirect();
428                         if (fileName.equals(project.getIndexFile()) && !fileOption.isInsert() && !fileOption.isInsertRedirect()) {
429                                 checkReport.addIssue("error.index-not-inserted", true);
430                         }
431                         if (!fileOption.isInsert() && fileOption.isInsertRedirect() && ((fileOption.getCustomKey().length() == 0) || "CHK@".equals(fileOption.getCustomKey()))) {
432                                 checkReport.addIssue("error.no-custom-key", true, fileName);
433                         }
434                 }
435                 if (!insert) {
436                         checkReport.addIssue("error.no-files-to-insert", true);
437                 }
438                 return checkReport;
439         }
440
441         /**
442          * {@inheritDoc}
443          */
444         public void run() {
445                 fireProjectInsertStarted();
446                 List<String> files = fileScanner.getFiles();
447
448                 /* create connection to node */
449                 Connection connection = freenetInterface.getConnection("project-insert-" + random + counter++);
450                 connection.setTempDirectory(tempDirectory);
451                 boolean connected = false;
452                 Throwable cause = null;
453                 try {
454                         connected = connection.connect();
455                 } catch (IOException e1) {
456                         cause = e1;
457                 }
458
459                 if (!connected) {
460                         fireProjectInsertFinished(false, cause);
461                         return;
462                 }
463
464                 Client client = new Client(connection);
465
466                 /* create containers */
467                 final List<String> containers = new ArrayList<String>();
468                 final Map<String, List<String>> containerFiles = new HashMap<String, List<String>>();
469                 createContainers(files, containers, containerFiles);
470
471                 /* collect files */
472                 int edition = project.getEdition();
473                 String dirURI = "USK@" + project.getInsertURI() + "/" + project.getPath() + "/" + edition + "/";
474                 ClientPutComplexDir putDir = new ClientPutComplexDir("dir-" + counter++, dirURI, tempDirectory);
475                 if ((project.getIndexFile() != null) && (project.getIndexFile().length() > 0)) {
476                         putDir.setDefaultName(project.getIndexFile());
477                 }
478                 putDir.setVerbosity(Verbosity.ALL);
479                 putDir.setMaxRetries(-1);
480                 for (String filename : files) {
481                         FileEntry fileEntry = createFileEntry(filename, edition, containerFiles);
482                         if (fileEntry != null) {
483                                 try {
484                                         putDir.addFileEntry(fileEntry);
485                                 } catch (IOException ioe1) {
486                                         fireProjectInsertFinished(false, ioe1);
487                                         return;
488                                 }
489                         }
490                 }
491
492                 /* start request */
493                 try {
494                         client.execute(putDir);
495                 } catch (IOException ioe1) {
496                         fireProjectInsertFinished(false, ioe1);
497                         return;
498                 }
499
500                 /* parse progress and success messages */
501                 String finalURI = null;
502                 boolean firstMessage = true;
503                 boolean success = false;
504                 boolean finished = false;
505                 boolean disconnected = false;
506                 while (!finished) {
507                         Message message = client.readMessage();
508                         finished = (message == null) || (disconnected = client.isDisconnected());
509                         if (firstMessage) {
510                                 fireProjectUploadFinished();
511                                 firstMessage = false;
512                         }
513                         logger.log(Level.FINE, "Received message: " + message);
514                         if (!finished) {
515                                 @SuppressWarnings("null")
516                                 String messageName = message.getName();
517                                 if ("URIGenerated".equals(messageName)) {
518                                         finalURI = message.get("URI");
519                                         fireProjectURIGenerated(finalURI);
520                                 }
521                                 if ("SimpleProgress".equals(messageName)) {
522                                         int total = Integer.parseInt(message.get("Total"));
523                                         int succeeded = Integer.parseInt(message.get("Succeeded"));
524                                         int fatal = Integer.parseInt(message.get("FatallyFailed"));
525                                         int failed = Integer.parseInt(message.get("Failed"));
526                                         boolean finalized = Boolean.parseBoolean(message.get("FinalizedTotal"));
527                                         fireProjectInsertProgress(succeeded, failed, fatal, total, finalized);
528                                 }
529                                 success = "PutSuccessful".equals(messageName);
530                                 finished = success || "PutFailed".equals(messageName) || messageName.endsWith("Error");
531                         }
532                 }
533
534                 /* post-insert work */
535                 fireProjectInsertFinished(success, disconnected ? new IOException("Connection terminated") : null);
536                 if (success) {
537                         @SuppressWarnings("null")
538                         String editionPart = finalURI.substring(finalURI.lastIndexOf('/') + 1);
539                         int newEdition = Integer.parseInt(editionPart);
540                         project.setEdition(newEdition);
541                         project.setLastInsertionTime(System.currentTimeMillis());
542                 }
543         }
544
545         //
546         // INTERFACE FileScannerListener
547         //
548
549         /**
550          * {@inheritDoc}
551          */
552         public void fileScannerFinished(FileScanner fileScanner) {
553                 if (!fileScanner.isError()) {
554                         new Thread(this).start();
555                 } else {
556                         fireProjectInsertFinished(false, null);
557                 }
558                 fileScanner.removeFileScannerListener(this);
559         }
560
561         /**
562          * Container class that collects all warnings and errors that occured during
563          * {@link ProjectInserter#validateProject(Project) project validation}.
564          *
565          * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
566          */
567         public static class CheckReport implements Iterable<Issue> {
568
569                 /** The issures that occured. */
570                 private final List<Issue> issues = new ArrayList<Issue>();
571
572                 /**
573                  * Adds an issue.
574                  *
575                  * @param issue
576                  *            The issue to add
577                  */
578                 public void addIssue(Issue issue) {
579                         issues.add(issue);
580                 }
581
582                 /**
583                  * Creates an {@link Issue} from the given error key and fatality flag
584                  * and {@link #addIssue(Issue) adds} it.
585                  *
586                  * @param errorKey
587                  *            The error key
588                  * @param fatal
589                  *            {@code true} if the error is fatal, {@code false} if only
590                  *            a warning should be generated
591                  * @param parameters
592                  *            Any additional parameters
593                  */
594                 public void addIssue(String errorKey, boolean fatal, String... parameters) {
595                         addIssue(new Issue(errorKey, fatal, parameters));
596                 }
597
598                 /**
599                  * {@inheritDoc}
600                  */
601                 public Iterator<Issue> iterator() {
602                         return issues.iterator();
603                 }
604
605                 /**
606                  * Returns whether this check report does not contain any errors.
607                  *
608                  * @return {@code true} if this check report does not contain any
609                  *         errors, {@code false} if this check report does contain
610                  *         errors
611                  */
612                 public boolean isEmpty() {
613                         return issues.isEmpty();
614                 }
615
616                 /**
617                  * Returns the number of issues in this check report.
618                  *
619                  * @return The number of issues
620                  */
621                 public int size() {
622                         return issues.size();
623                 }
624
625                 /**
626                  * Container class for a single issue. An issue contains an error key
627                  * that describes the error, and a fatality flag that determines whether
628                  * the insert has to be aborted (if the flag is {@code true}) or if it
629                  * can still be performed and only a warning should be generated (if the
630                  * flag is {@code false}).
631                  *
632                  * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’
633                  *         Roden</a>
634                  */
635                 public static class Issue {
636
637                         /** The error key. */
638                         private final String errorKey;
639
640                         /** The fatality flag. */
641                         private final boolean fatal;
642
643                         /** Additional parameters. */
644                         private String[] parameters;
645
646                         /**
647                          * Creates a new issue.
648                          *
649                          * @param errorKey
650                          *            The error key
651                          * @param fatal
652                          *            The fatality flag
653                          * @param parameters
654                          *            Any additional parameters
655                          */
656                         protected Issue(String errorKey, boolean fatal, String... parameters) {
657                                 this.errorKey = errorKey;
658                                 this.fatal = fatal;
659                                 this.parameters = parameters;
660                         }
661
662                         /**
663                          * Returns the key of the encountered error.
664                          *
665                          * @return The error key
666                          */
667                         public String getErrorKey() {
668                                 return errorKey;
669                         }
670
671                         /**
672                          * Returns whether the issue is fatal and the insert has to be
673                          * aborted. Otherwise only a warning should be shown.
674                          *
675                          * @return {@code true} if the insert needs to be aborted, {@code
676                          *         false} otherwise
677                          */
678                         public boolean isFatal() {
679                                 return fatal;
680                         }
681
682                         /**
683                          * Returns any additional parameters.
684                          *
685                          * @return The additional parameters
686                          */
687                         public String[] getParameters() {
688                                 return parameters;
689                         }
690
691                 }
692
693         }
694
695 }