src/de/todesbaum/jsite/application/ProjectInserter.java

   1 /*
   2  * jSite - ProjectInserter.java - Copyright © 2006–2012 David Roden
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  */
  18
  19 package de.todesbaum.jsite.application;
  20
  21 import java.io.File;
  22 import java.io.FileInputStream;
  23 import java.io.IOException;
  24 import java.io.InputStream;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.HashSet;
  28 import java.util.Iterator;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Map.Entry;
  32 import java.util.Set;
  33 import java.util.logging.Level;
  34 import java.util.logging.Logger;
  35
  36 import de.todesbaum.jsite.gui.FileScanner;
  37 import de.todesbaum.jsite.gui.FileScanner.ScannedFile;
  38 import de.todesbaum.jsite.gui.FileScannerListener;
  39 import de.todesbaum.util.freenet.fcp2.Client;
  40 import de.todesbaum.util.freenet.fcp2.ClientPutComplexDir;
  41 import de.todesbaum.util.freenet.fcp2.ClientPutDir.ManifestPutter;
  42 import de.todesbaum.util.freenet.fcp2.Connection;
  43 import de.todesbaum.util.freenet.fcp2.DirectFileEntry;
  44 import de.todesbaum.util.freenet.fcp2.FileEntry;
  45 import de.todesbaum.util.freenet.fcp2.Message;
  46 import de.todesbaum.util.freenet.fcp2.RedirectFileEntry;
  47 import de.todesbaum.util.freenet.fcp2.Verbosity;
  48 import de.todesbaum.util.io.StreamCopier.ProgressListener;
  49
  50 /**
  51  * Manages project inserts.
  52  *
  53  * @author David ‘Bombe’ Roden &lt;bombe@freenetproject.org&gt;
  54  */
  55 public class ProjectInserter implements FileScannerListener, Runnable {
  56
  57         /** The logger. */
  58         private static final Logger logger = Logger.getLogger(ProjectInserter.class.getName());
  59
  60         /** Random number for FCP instances. */
  61         private static final int random = (int) (Math.random() * Integer.MAX_VALUE);
  62
  63         /** Counter for FCP connection identifier. */
  64         private static int counter = 0;
  65
  66         /** The list of insert listeners. */
  67         private List<InsertListener> insertListeners = new ArrayList<InsertListener>();
  68
  69         /** The freenet interface. */
  70         protected Freenet7Interface freenetInterface;
  71
  72         /** The project to insert. */
  73         protected Project project;
  74
  75         /** The file scanner. */
  76         private FileScanner fileScanner;
  77
  78         /** Object used for synchronization. */
  79         protected final Object lockObject = new Object();
  80
  81         /** The temp directory. */
  82         private String tempDirectory;
  83
  84         /** The current connection. */
  85         private Connection connection;
  86
  87         /** Whether the insert is cancelled. */
  88         private volatile boolean cancelled = false;
  89
  90         /** Progress listener for payload transfers. */
  91         private ProgressListener progressListener;
  92
  93         /** Whether to use “early encode.” */
  94         private boolean useEarlyEncode;
  95
  96         /**
  97          * Adds a listener to the list of registered listeners.
  98          *
  99          * @param insertListener
 100          *            The listener to add
 101          */
 102         public void addInsertListener(InsertListener insertListener) {
 103                 insertListeners.add(insertListener);
 104         }
 105
 106         /**
 107          * Removes a listener from the list of registered listeners.
 108          *
 109          * @param insertListener
 110          *            The listener to remove
 111          */
 112         public void removeInsertListener(InsertListener insertListener) {
 113                 insertListeners.remove(insertListener);
 114         }
 115
 116         /**
 117          * Notifies all listeners that the project insert has started.
 118          *
 119          * @see InsertListener#projectInsertStarted(Project)
 120          */
 121         protected void fireProjectInsertStarted() {
 122                 for (InsertListener insertListener : insertListeners) {
 123                         insertListener.projectInsertStarted(project);
 124                 }
 125         }
 126
 127         /**
 128          * Notifies all listeners that the insert has generated a URI.
 129          *
 130          * @see InsertListener#projectURIGenerated(Project, String)
 131          * @param uri
 132          *            The generated URI
 133          */
 134         protected void fireProjectURIGenerated(String uri) {
 135                 for (InsertListener insertListener : insertListeners) {
 136                         insertListener.projectURIGenerated(project, uri);
 137                 }
 138         }
 139
 140         /**
 141          * Notifies all listeners that the insert has made some progress.
 142          *
 143          * @see InsertListener#projectUploadFinished(Project)
 144          */
 145         protected void fireProjectUploadFinished() {
 146                 for (InsertListener insertListener : insertListeners) {
 147                         insertListener.projectUploadFinished(project);
 148                 }
 149         }
 150
 151         /**
 152          * Notifies all listeners that the insert has made some progress.
 153          *
 154          * @see InsertListener#projectInsertProgress(Project, int, int, int, int,
 155          *      boolean)
 156          * @param succeeded
 157          *            The number of succeeded blocks
 158          * @param failed
 159          *            The number of failed blocks
 160          * @param fatal
 161          *            The number of fatally failed blocks
 162          * @param total
 163          *            The total number of blocks
 164          * @param finalized
 165          *            <code>true</code> if the total number of blocks has already
 166          *            been finalized, <code>false</code> otherwise
 167          */
 168         protected void fireProjectInsertProgress(int succeeded, int failed, int fatal, int total, boolean finalized) {
 169                 for (InsertListener insertListener : insertListeners) {
 170                         insertListener.projectInsertProgress(project, succeeded, failed, fatal, total, finalized);
 171                 }
 172         }
 173
 174         /**
 175          * Notifies all listeners the project insert has finished.
 176          *
 177          * @see InsertListener#projectInsertFinished(Project, boolean, Throwable)
 178          * @param success
 179          *            <code>true</code> if the project was inserted successfully,
 180          *            <code>false</code> if it failed
 181          * @param cause
 182          *            The cause of the failure, if any
 183          */
 184         protected void fireProjectInsertFinished(boolean success, Throwable cause) {
 185                 for (InsertListener insertListener : insertListeners) {
 186                         insertListener.projectInsertFinished(project, success, cause);
 187                 }
 188         }
 189
 190         /**
 191          * Sets the project to insert.
 192          *
 193          * @param project
 194          *            The project to insert
 195          */
 196         public void setProject(Project project) {
 197                 this.project = project;
 198         }
 199
 200         /**
 201          * Sets the freenet interface to use.
 202          *
 203          * @param freenetInterface
 204          *            The freenet interface to use
 205          */
 206         public void setFreenetInterface(Freenet7Interface freenetInterface) {
 207                 this.freenetInterface = freenetInterface;
 208         }
 209
 210         /**
 211          * Sets the temp directory to use.
 212          *
 213          * @param tempDirectory
 214          *            The temp directory to use, or {@code null} to use the system
 215          *            default
 216          */
 217         public void setTempDirectory(String tempDirectory) {
 218                 this.tempDirectory = tempDirectory;
 219         }
 220
 221         /**
 222          * Sets whether to use the “early encode“ flag for the insert.
 223          *
 224          * @param useEarlyEncode
 225          *            {@code true} to set the “early encode” flag for the insert,
 226          *            {@code false} otherwise
 227          */
 228         public void setUseEarlyEncode(boolean useEarlyEncode) {
 229                 this.useEarlyEncode = useEarlyEncode;
 230         }
 231
 232         /**
 233          * Starts the insert.
 234          *
 235          * @param progressListener
 236          *            Listener to notify on progress events
 237          */
 238         public void start(ProgressListener progressListener) {
 239                 cancelled = false;
 240                 this.progressListener = progressListener;
 241                 fileScanner = new FileScanner(project);
 242                 fileScanner.addFileScannerListener(this);
 243                 new Thread(fileScanner).start();
 244         }
 245
 246         /**
 247          * Stops the current insert.
 248          */
 249         public void stop() {
 250                 cancelled = true;
 251                 synchronized (lockObject) {
 252                         if (connection != null) {
 253                                 connection.disconnect();
 254                         }
 255                 }
 256         }
 257
 258         /**
 259          * Creates an input stream that delivers the given file, replacing edition
 260          * tokens in the file’s content, if necessary.
 261          *
 262          * @param filename
 263          *            The name of the file
 264          * @param fileOption
 265          *            The file options
 266          * @param edition
 267          *            The current edition
 268          * @param length
 269          *            An array containing a single long which is used to
 270          *            <em>return</em> the final length of the file, after all
 271          *            replacements
 272          * @return The input stream for the file
 273          * @throws IOException
 274          *             if an I/O error occurs
 275          */
 276         private InputStream createFileInputStream(String filename, FileOption fileOption, int edition, long[] length) throws IOException {
 277                 File file = new File(project.getLocalPath(), filename);
 278                 length[0] = file.length();
 279                 return new FileInputStream(file);
 280         }
 281
 282         /**
 283          * Creates a file entry suitable for handing in to
 284          * {@link ClientPutComplexDir#addFileEntry(FileEntry)}.
 285          *
 286          * @param file
 287          *            The name and hash of the file to insert
 288          * @param edition
 289          *            The current edition
 290          * @return A file entry for the given file
 291          */
 292         private FileEntry createFileEntry(ScannedFile file, int edition) {
 293                 FileEntry fileEntry = null;
 294                 String filename = file.getFilename();
 295                 FileOption fileOption = project.getFileOption(filename);
 296                 if (fileOption.isInsert()) {
 297                         fileOption.setCurrentHash(file.getHash());
 298                         /* check if file was modified. */
 299                         if (!fileOption.isForceInsert() && file.getHash().equals(fileOption.getLastInsertHash())) {
 300                                 /* only insert a redirect. */
 301                                 logger.log(Level.FINE, String.format("Inserting redirect to edition %d for %s.", fileOption.getLastInsertEdition(), filename));
 302                                 return new RedirectFileEntry(filename, fileOption.getMimeType(), "SSK@" + project.getRequestURI() + "/" + project.getPath() + "-" + fileOption.getLastInsertEdition() + "/" + filename);
 303                         }
 304                         try {
 305                                 long[] fileLength = new long[1];
 306                                 InputStream fileEntryInputStream = createFileInputStream(filename, fileOption, edition, fileLength);
 307                                 fileEntry = new DirectFileEntry(filename, fileOption.getMimeType(), fileEntryInputStream, fileLength[0]);
 308                         } catch (IOException ioe1) {
 309                                 /* ignore, null is returned. */
 310                         }
 311                 } else {
 312                         if (fileOption.isInsertRedirect()) {
 313                                 fileEntry = new RedirectFileEntry(filename, fileOption.getMimeType(), fileOption.getCustomKey());
 314                         }
 315                 }
 316                 return fileEntry;
 317         }
 318
 319         /**
 320          * Validates the given project. The project will be checked for any invalid
 321          * conditions, such as invalid insert or request keys, missing path names,
 322          * missing default file, and so on.
 323          *
 324          * @param project
 325          *            The project to check
 326          * @return The encountered warnings and errors
 327          */
 328         public static CheckReport validateProject(Project project) {
 329                 CheckReport checkReport = new CheckReport();
 330                 if ((project.getLocalPath() == null) || (project.getLocalPath().trim().length() == 0)) {
 331                         checkReport.addIssue("error.no-local-path", true);
 332                 }
 333                 if ((project.getPath() == null) || (project.getPath().trim().length() == 0)) {
 334                         checkReport.addIssue("error.no-path", true);
 335                 }
 336                 if ((project.getIndexFile() == null) || (project.getIndexFile().length() == 0)) {
 337                         checkReport.addIssue("warning.empty-index", false);
 338                 } else {
 339                         File indexFile = new File(project.getLocalPath(), project.getIndexFile());
 340                         if (!indexFile.exists()) {
 341                                 checkReport.addIssue("error.index-missing", true);
 342                         }
 343                 }
 344                 String indexFile = project.getIndexFile();
 345                 boolean hasIndexFile = (indexFile != null) && (indexFile.length() > 0);
 346                 List<String> allowedIndexContentTypes = Arrays.asList("text/html", "application/xhtml+xml");
 347                 if (hasIndexFile && !allowedIndexContentTypes.contains(project.getFileOption(indexFile).getMimeType())) {
 348                         checkReport.addIssue("warning.index-not-html", false);
 349                 }
 350                 Map<String, FileOption> fileOptions = project.getFileOptions();
 351                 Set<Entry<String, FileOption>> fileOptionEntries = fileOptions.entrySet();
 352                 boolean insert = fileOptionEntries.isEmpty();
 353                 for (Entry<String, FileOption> fileOptionEntry : fileOptionEntries) {
 354                         String fileName = fileOptionEntry.getKey();
 355                         FileOption fileOption = fileOptionEntry.getValue();
 356                         insert |= fileOption.isInsert() || fileOption.isInsertRedirect();
 357                         if (fileName.equals(project.getIndexFile()) && !fileOption.isInsert() && !fileOption.isInsertRedirect()) {
 358                                 checkReport.addIssue("error.index-not-inserted", true);
 359                         }
 360                         if (!fileOption.isInsert() && fileOption.isInsertRedirect() && ((fileOption.getCustomKey().length() == 0) || "CHK@".equals(fileOption.getCustomKey()))) {
 361                                 checkReport.addIssue("error.no-custom-key", true, fileName);
 362                         }
 363                 }
 364                 if (!insert) {
 365                         checkReport.addIssue("error.no-files-to-insert", true);
 366                 }
 367                 Set<String> fileNames = new HashSet<String>();
 368                 for (Entry<String, FileOption> fileOptionEntry : fileOptionEntries) {
 369                         FileOption fileOption = fileOptionEntry.getValue();
 370                         if (!fileOption.isInsert() && !fileOption.isInsertRedirect()) {
 371                                 logger.log(Level.FINEST, "Ignoring {0}.", fileOptionEntry.getKey());
 372                                 continue;
 373                         }
 374                         String fileName = fileOptionEntry.getKey();
 375                         if (fileOption.hasChangedName()) {
 376                                 fileName = fileOption.getChangedName();
 377                         }
 378                         logger.log(Level.FINEST, "Adding “{0}” for {1}.", new Object[] { fileName, fileOptionEntry.getKey() });
 379                         if (!fileNames.add(fileName)) {
 380                                 checkReport.addIssue("error.duplicate-file", true, fileName);
 381                         }
 382                 }
 383                 return checkReport;
 384         }
 385
 386         /**
 387          * {@inheritDoc}
 388          */
 389         public void run() {
 390                 fireProjectInsertStarted();
 391                 List<ScannedFile> files = fileScanner.getFiles();
 392
 393                 /* create connection to node */
 394                 synchronized (lockObject) {
 395                         connection = freenetInterface.getConnection("project-insert-" + random + counter++);
 396                 }
 397                 connection.setTempDirectory(tempDirectory);
 398                 boolean connected = false;
 399                 Throwable cause = null;
 400                 try {
 401                         connected = connection.connect();
 402                 } catch (IOException e1) {
 403                         cause = e1;
 404                 }
 405
 406                 if (!connected || cancelled) {
 407                         fireProjectInsertFinished(false, cancelled ? new AbortedException() : cause);
 408                         return;
 409                 }
 410
 411                 Client client = new Client(connection);
 412
 413                 /* collect files */
 414                 int edition = project.getEdition();
 415                 String dirURI = "USK@" + project.getInsertURI() + "/" + project.getPath() + "/" + edition + "/";
 416                 ClientPutComplexDir putDir = new ClientPutComplexDir("dir-" + counter++, dirURI, tempDirectory);
 417                 if ((project.getIndexFile() != null) && (project.getIndexFile().length() > 0)) {
 418                         putDir.setDefaultName(project.getIndexFile());
 419                 }
 420                 putDir.setVerbosity(Verbosity.ALL);
 421                 putDir.setMaxRetries(-1);
 422                 putDir.setEarlyEncode(useEarlyEncode);
 423                 putDir.setManifestPutter(ManifestPutter.DEFAULT);
 424                 for (ScannedFile file : files) {
 425                         FileEntry fileEntry = createFileEntry(file, edition);
 426                         if (fileEntry != null) {
 427                                 try {
 428                                         putDir.addFileEntry(fileEntry);
 429                                 } catch (IOException ioe1) {
 430                                         fireProjectInsertFinished(false, ioe1);
 431                                         return;
 432                                 }
 433                         }
 434                 }
 435
 436                 /* start request */
 437                 try {
 438                         client.execute(putDir, progressListener);
 439                         fireProjectUploadFinished();
 440                 } catch (IOException ioe1) {
 441                         fireProjectInsertFinished(false, ioe1);
 442                         return;
 443                 }
 444
 445                 /* parse progress and success messages */
 446                 String finalURI = null;
 447                 boolean success = false;
 448                 boolean finished = false;
 449                 boolean disconnected = false;
 450                 while (!finished && !cancelled) {
 451                         Message message = client.readMessage();
 452                         finished = (message == null) || (disconnected = client.isDisconnected());
 453                         logger.log(Level.FINE, "Received message: " + message);
 454                         if (!finished) {
 455                                 @SuppressWarnings("null")
 456                                 String messageName = message.getName();
 457                                 if ("URIGenerated".equals(messageName)) {
 458                                         finalURI = message.get("URI");
 459                                         fireProjectURIGenerated(finalURI);
 460                                 }
 461                                 if ("SimpleProgress".equals(messageName)) {
 462                                         int total = Integer.parseInt(message.get("Total"));
 463                                         int succeeded = Integer.parseInt(message.get("Succeeded"));
 464                                         int fatal = Integer.parseInt(message.get("FatallyFailed"));
 465                                         int failed = Integer.parseInt(message.get("Failed"));
 466                                         boolean finalized = Boolean.parseBoolean(message.get("FinalizedTotal"));
 467                                         fireProjectInsertProgress(succeeded, failed, fatal, total, finalized);
 468                                 }
 469                                 success |= "PutSuccessful".equals(messageName);
 470                                 finished = (success && (finalURI != null)) || "PutFailed".equals(messageName) || messageName.endsWith("Error");
 471                         }
 472                 }
 473
 474                 /* post-insert work */
 475                 if (success) {
 476                         @SuppressWarnings("null")
 477                         String editionPart = finalURI.substring(finalURI.lastIndexOf('/') + 1);
 478                         int newEdition = Integer.parseInt(editionPart);
 479                         project.setEdition(newEdition);
 480                         project.setLastInsertionTime(System.currentTimeMillis());
 481                         project.onSuccessfulInsert();
 482                 }
 483                 fireProjectInsertFinished(success, cancelled ? new AbortedException() : (disconnected ? new IOException("Connection terminated") : null));
 484         }
 485
 486         //
 487         // INTERFACE FileScannerListener
 488         //
 489
 490         /**
 491          * {@inheritDoc}
 492          */
 493         public void fileScannerFinished(FileScanner fileScanner) {
 494                 if (!fileScanner.isError()) {
 495                         new Thread(this).start();
 496                 } else {
 497                         fireProjectInsertFinished(false, null);
 498                 }
 499                 fileScanner.removeFileScannerListener(this);
 500         }
 501
 502         /**
 503          * Container class that collects all warnings and errors that occured during
 504          * {@link ProjectInserter#validateProject(Project) project validation}.
 505          *
 506          * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’ Roden</a>
 507          */
 508         public static class CheckReport implements Iterable<Issue> {
 509
 510                 /** The issures that occured. */
 511                 private final List<Issue> issues = new ArrayList<Issue>();
 512
 513                 /**
 514                  * Adds an issue.
 515                  *
 516                  * @param issue
 517                  *            The issue to add
 518                  */
 519                 public void addIssue(Issue issue) {
 520                         issues.add(issue);
 521                 }
 522
 523                 /**
 524                  * Creates an {@link Issue} from the given error key and fatality flag
 525                  * and {@link #addIssue(Issue) adds} it.
 526                  *
 527                  * @param errorKey
 528                  *            The error key
 529                  * @param fatal
 530                  *            {@code true} if the error is fatal, {@code false} if only
 531                  *            a warning should be generated
 532                  * @param parameters
 533                  *            Any additional parameters
 534                  */
 535                 public void addIssue(String errorKey, boolean fatal, String... parameters) {
 536                         addIssue(new Issue(errorKey, fatal, parameters));
 537                 }
 538
 539                 /**
 540                  * {@inheritDoc}
 541                  */
 542                 public Iterator<Issue> iterator() {
 543                         return issues.iterator();
 544                 }
 545
 546                 /**
 547                  * Returns whether this check report does not contain any errors.
 548                  *
 549                  * @return {@code true} if this check report does not contain any
 550                  *         errors, {@code false} if this check report does contain
 551                  *         errors
 552                  */
 553                 public boolean isEmpty() {
 554                         return issues.isEmpty();
 555                 }
 556
 557                 /**
 558                  * Returns the number of issues in this check report.
 559                  *
 560                  * @return The number of issues
 561                  */
 562                 public int size() {
 563                         return issues.size();
 564                 }
 565
 566         }
 567
 568         /**
 569          * Container class for a single issue. An issue contains an error key
 570          * that describes the error, and a fatality flag that determines whether
 571          * the insert has to be aborted (if the flag is {@code true}) or if it
 572          * can still be performed and only a warning should be generated (if the
 573          * flag is {@code false}).
 574          *
 575          * @author <a href="mailto:bombe@pterodactylus.net">David ‘Bombe’
 576          *         Roden</a>
 577          */
 578         public static class Issue {
 579
 580                 /** The error key. */
 581                 private final String errorKey;
 582
 583                 /** The fatality flag. */
 584                 private final boolean fatal;
 585
 586                 /** Additional parameters. */
 587                 private String[] parameters;
 588
 589                 /**
 590                  * Creates a new issue.
 591                  *
 592                  * @param errorKey
 593                  *            The error key
 594                  * @param fatal
 595                  *            The fatality flag
 596                  * @param parameters
 597                  *            Any additional parameters
 598                  */
 599                 protected Issue(String errorKey, boolean fatal, String... parameters) {
 600                         this.errorKey = errorKey;
 601                         this.fatal = fatal;
 602                         this.parameters = parameters;
 603                 }
 604
 605                 /**
 606                  * Returns the key of the encountered error.
 607                  *
 608                  * @return The error key
 609                  */
 610                 public String getErrorKey() {
 611                         return errorKey;
 612                 }
 613
 614                 /**
 615                  * Returns whether the issue is fatal and the insert has to be
 616                  * aborted. Otherwise only a warning should be shown.
 617                  *
 618                  * @return {@code true} if the insert needs to be aborted, {@code
 619                  *         false} otherwise
 620                  */
 621                 public boolean isFatal() {
 622                         return fatal;
 623                 }
 624
 625                 /**
 626                  * Returns any additional parameters.
 627                  *
 628                  * @return The additional parameters
 629                  */
 630                 public String[] getParameters() {
 631                         return parameters;
 632                 }
 633
 634         }
 635
 636 }