src/de/todesbaum/jsite/application/ProjectInserter.java

   1 /*
   2  * jSite - a tool for uploading websites into Freenet
   3  * Copyright (C) 2006 David Roden
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  */
  19
  20 package de.todesbaum.jsite.application;
  21
  22 import java.io.ByteArrayInputStream;
  23 import java.io.ByteArrayOutputStream;
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.util.ArrayList;
  30 import java.util.HashMap;
  31 import java.util.List;
  32 import java.util.Map;
  33 import java.util.logging.Level;
  34 import java.util.logging.Logger;
  35 import java.util.zip.ZipEntry;
  36 import java.util.zip.ZipOutputStream;
  37
  38 import de.todesbaum.jsite.gui.FileScanner;
  39 import de.todesbaum.jsite.gui.FileScannerListener;
  40 import de.todesbaum.util.freenet.fcp2.Client;
  41 import de.todesbaum.util.freenet.fcp2.ClientPutComplexDir;
  42 import de.todesbaum.util.freenet.fcp2.Connection;
  43 import de.todesbaum.util.freenet.fcp2.DirectFileEntry;
  44 import de.todesbaum.util.freenet.fcp2.FileEntry;
  45 import de.todesbaum.util.freenet.fcp2.Message;
  46 import de.todesbaum.util.freenet.fcp2.RedirectFileEntry;
  47 import de.todesbaum.util.freenet.fcp2.Verbosity;
  48 import de.todesbaum.util.io.Closer;
  49 import de.todesbaum.util.io.ReplacingOutputStream;
  50 import de.todesbaum.util.io.StreamCopier;
  51
  52 /**
  53  * Manages project inserts.
  54  *
  55  * @author David ‘Bombe’ Roden &lt;bombe@freenetproject.org&gt;
  56  */
  57 public class ProjectInserter implements FileScannerListener, Runnable {
  58
  59         /** The logger. */
  60         private static final Logger logger = Logger.getLogger(ProjectInserter.class.getName());
  61
  62         /** Random number for FCP instances. */
  63         private static final int random = (int) (Math.random() * Integer.MAX_VALUE);
  64
  65         /** Counter for FCP connection identifier. */
  66         private static int counter = 0;
  67
  68         /** The list of insert listeners. */
  69         private List<InsertListener> insertListeners = new ArrayList<InsertListener>();
  70
  71         /** The freenet interface. */
  72         protected Freenet7Interface freenetInterface;
  73
  74         /** The project to insert. */
  75         protected Project project;
  76
  77         /** The file scanner. */
  78         private FileScanner fileScanner;
  79
  80         /** Object used for synchronization. */
  81         protected final Object lockObject = new Object();
  82
  83         /** The temp directory. */
  84         private String tempDirectory;
  85
  86         /**
  87          * Adds a listener to the list of registered listeners.
  88          *
  89          * @param insertListener
  90          *            The listener to add
  91          */
  92         public void addInsertListener(InsertListener insertListener) {
  93                 insertListeners.add(insertListener);
  94         }
  95
  96         /**
  97          * Removes a listener from the list of registered listeners.
  98          *
  99          * @param insertListener
 100          *            The listener to remove
 101          */
 102         public void removeInsertListener(InsertListener insertListener) {
 103                 insertListeners.remove(insertListener);
 104         }
 105
 106         /**
 107          * Notifies all listeners that the project insert has started.
 108          *
 109          * @see InsertListener#projectInsertStarted(Project)
 110          */
 111         protected void fireProjectInsertStarted() {
 112                 for (InsertListener insertListener : insertListeners) {
 113                         insertListener.projectInsertStarted(project);
 114                 }
 115         }
 116
 117         /**
 118          * Notifies all listeners that the insert has generated a URI.
 119          *
 120          * @see InsertListener#projectURIGenerated(Project, String)
 121          * @param uri
 122          *            The generated URI
 123          */
 124         protected void fireProjectURIGenerated(String uri) {
 125                 for (InsertListener insertListener : insertListeners) {
 126                         insertListener.projectURIGenerated(project, uri);
 127                 }
 128         }
 129
 130         /**
 131          * Notifies all listeners that the insert has made some progress.
 132          *
 133          * @see InsertListener#projectUploadFinished(Project)
 134          */
 135         protected void fireProjectUploadFinished() {
 136                 for (InsertListener insertListener : insertListeners) {
 137                         insertListener.projectUploadFinished(project);
 138                 }
 139         }
 140
 141         /**
 142          * Notifies all listeners that the insert has made some progress.
 143          *
 144          * @see InsertListener#projectInsertProgress(Project, int, int, int, int,
 145          *      boolean)
 146          * @param succeeded
 147          *            The number of succeeded blocks
 148          * @param failed
 149          *            The number of failed blocks
 150          * @param fatal
 151          *            The number of fatally failed blocks
 152          * @param total
 153          *            The total number of blocks
 154          * @param finalized
 155          *            <code>true</code> if the total number of blocks has already
 156          *            been finalized, <code>false</code> otherwise
 157          */
 158         protected void fireProjectInsertProgress(int succeeded, int failed, int fatal, int total, boolean finalized) {
 159                 for (InsertListener insertListener : insertListeners) {
 160                         insertListener.projectInsertProgress(project, succeeded, failed, fatal, total, finalized);
 161                 }
 162         }
 163
 164         /**
 165          * Notifies all listeners the project insert has finished.
 166          *
 167          * @see InsertListener#projectInsertFinished(Project, boolean, Throwable)
 168          * @param success
 169          *            <code>true</code> if the project was inserted successfully,
 170          *            <code>false</code> if it failed
 171          * @param cause
 172          *            The cause of the failure, if any
 173          */
 174         protected void fireProjectInsertFinished(boolean success, Throwable cause) {
 175                 for (InsertListener insertListener : insertListeners) {
 176                         insertListener.projectInsertFinished(project, success, cause);
 177                 }
 178         }
 179
 180         /**
 181          * Sets the project to insert.
 182          *
 183          * @param project
 184          *            The project to insert
 185          */
 186         public void setProject(Project project) {
 187                 this.project = project;
 188         }
 189
 190         /**
 191          * Sets the freenet interface to use.
 192          *
 193          * @param freenetInterface
 194          *            The freenet interface to use
 195          */
 196         public void setFreenetInterface(Freenet7Interface freenetInterface) {
 197                 this.freenetInterface = freenetInterface;
 198         }
 199
 200         /**
 201          * Sets the temp directory to use.
 202          *
 203          * @param tempDirectory
 204          *            The temp directory to use, or {@code null} to use the system
 205          *            default
 206          */
 207         public void setTempDirectory(String tempDirectory) {
 208                 this.tempDirectory = tempDirectory;
 209         }
 210
 211         /**
 212          * Starts the insert.
 213          */
 214         public void start() {
 215                 fileScanner = new FileScanner(project);
 216                 fileScanner.addFileScannerListener(this);
 217                 new Thread(fileScanner).start();
 218         }
 219
 220         /**
 221          * Creates an input stream that delivers the given file, replacing edition
 222          * tokens in the file’s content, if necessary.
 223          *
 224          * @param filename
 225          *            The name of the file
 226          * @param fileOption
 227          *            The file options
 228          * @param edition
 229          *            The current edition
 230          * @param length
 231          *            An array containing a single long which is used to
 232          *            <em>return</em> the final length of the file, after all
 233          *            replacements
 234          * @return The input stream for the file
 235          * @throws IOException
 236          *             if an I/O error occurs
 237          */
 238         private InputStream createFileInputStream(String filename, FileOption fileOption, int edition, long[] length) throws IOException {
 239                 File file = new File(project.getLocalPath(), filename);
 240                 length[0] = file.length();
 241                 if (!fileOption.getReplaceEdition()) {
 242                         return new FileInputStream(file);
 243                 }
 244                 ByteArrayOutputStream filteredByteOutputStream = new ByteArrayOutputStream(Math.min(Integer.MAX_VALUE, (int) length[0]));
 245                 ReplacingOutputStream outputStream = new ReplacingOutputStream(filteredByteOutputStream);
 246                 FileInputStream fileInput = new FileInputStream(file);
 247                 try {
 248                         outputStream.addReplacement("$[EDITION]", String.valueOf(edition));
 249                         outputStream.addReplacement("$[URI]", project.getFinalRequestURI(0));
 250                         for (int index = 1; index <= fileOption.getEditionRange(); index++) {
 251                                 outputStream.addReplacement("$[URI+" + index + "]", project.getFinalRequestURI(index));
 252                                 outputStream.addReplacement("$[EDITION+" + index + "]", String.valueOf(edition + index));
 253                         }
 254                         StreamCopier.copy(fileInput, outputStream, length[0]);
 255                 } finally {
 256                         Closer.close(fileInput);
 257                         Closer.close(outputStream);
 258                         Closer.close(filteredByteOutputStream);
 259                 }
 260                 byte[] filteredBytes = filteredByteOutputStream.toByteArray();
 261                 length[0] = filteredBytes.length;
 262                 return new ByteArrayInputStream(filteredBytes);
 263         }
 264
 265         /**
 266          * Creates an input stream for a container.
 267          *
 268          * @param containerFiles
 269          *            All container definitions
 270          * @param containerName
 271          *            The name of the container to create
 272          * @param edition
 273          *            The current edition
 274          * @param containerLength
 275          *            An array containing a single long which is used to
 276          *            <em>return</em> the final length of the container stream,
 277          *            after all replacements
 278          * @return The input stream for the container
 279          * @throws IOException
 280          *             if an I/O error occurs
 281          */
 282         private InputStream createContainerInputStream(Map<String, List<String>> containerFiles, String containerName, int edition, long[] containerLength) throws IOException {
 283                 File tempFile = File.createTempFile("jsite", ".zip", (tempDirectory == null) ? null : new File(tempDirectory));
 284                 tempFile.deleteOnExit();
 285                 FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
 286                 ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
 287                 try {
 288                         for (String filename : containerFiles.get(containerName)) {
 289                                 File dataFile = new File(project.getLocalPath(), filename);
 290                                 if (dataFile.exists()) {
 291                                         ZipEntry zipEntry = new ZipEntry(filename);
 292                                         long[] fileLength = new long[1];
 293                                         InputStream wrappedInputStream = createFileInputStream(filename, project.getFileOption(filename), edition, fileLength);
 294                                         try {
 295                                                 zipOutputStream.putNextEntry(zipEntry);
 296                                                 StreamCopier.copy(wrappedInputStream, zipOutputStream, fileLength[0]);
 297                                         } finally {
 298                                                 zipOutputStream.closeEntry();
 299                                                 wrappedInputStream.close();
 300                                         }
 301                                 }
 302                         }
 303                 } finally {
 304                         zipOutputStream.closeEntry();
 305                         Closer.close(zipOutputStream);
 306                         Closer.close(fileOutputStream);
 307                 }
 308
 309                 containerLength[0] = tempFile.length();
 310                 return new FileInputStream(tempFile);
 311         }
 312
 313         /**
 314          * Creates a file entry suitable for handing in to
 315          * {@link ClientPutComplexDir#addFileEntry(FileEntry)}.
 316          *
 317          * @param filename
 318          *            The name of the file to insert
 319          * @param edition
 320          *            The current edition
 321          * @param containerFiles
 322          *            The container definitions
 323          * @return A file entry for the given file
 324          */
 325         private FileEntry createFileEntry(String filename, int edition, Map<String, List<String>> containerFiles) {
 326                 FileEntry fileEntry = null;
 327                 FileOption fileOption = project.getFileOption(filename);
 328                 if (filename.startsWith("/container/:")) {
 329                         String containerName = filename.substring("/container/:".length());
 330                         try {
 331                                 long[] containerLength = new long[1];
 332                                 InputStream containerInputStream = createContainerInputStream(containerFiles, containerName, edition, containerLength);
 333                                 fileEntry = new DirectFileEntry(containerName + ".zip", "application/zip", containerInputStream, containerLength[0]);
 334                         } catch (IOException ioe1) {
 335                                 /* ignore, null is returned. */
 336                         }
 337                 } else {
 338                         if (fileOption.isInsert()) {
 339                                 try {
 340                                         long[] fileLength = new long[1];
 341                                         InputStream fileEntryInputStream = createFileInputStream(filename, fileOption, edition, fileLength);
 342                                         fileEntry = new DirectFileEntry(filename, project.getFileOption(filename).getMimeType(), fileEntryInputStream, fileLength[0]);
 343                                 } catch (IOException ioe1) {
 344                                         /* ignore, null is returned. */
 345                                 }
 346                         } else {
 347                                 fileEntry = new RedirectFileEntry(filename, fileOption.getMimeType(), fileOption.getCustomKey());
 348                         }
 349                 }
 350                 return fileEntry;
 351         }
 352
 353         /**
 354          * Creates container definitions.
 355          *
 356          * @param files
 357          *            The list of all files
 358          * @param containers
 359          *            The list of all containers
 360          * @param containerFiles
 361          *            Empty map that will be filled with container definitions
 362          */
 363         private void createContainers(List<String> files, List<String> containers, Map<String, List<String>> containerFiles) {
 364                 for (String filename : new ArrayList<String>(files)) {
 365                         FileOption fileOption = project.getFileOption(filename);
 366                         String containerName = fileOption.getContainer();
 367                         if (!containerName.equals("")) {
 368                                 if (!containers.contains(containerName)) {
 369                                         containers.add(containerName);
 370                                         containerFiles.put(containerName, new ArrayList<String>());
 371                                         /* hmm. looks like a hack to me. */
 372                                         files.add("/container/:" + containerName);
 373                                 }
 374                                 containerFiles.get(containerName).add(filename);
 375                                 files.remove(filename);
 376                         }
 377                 }
 378         }
 379
 380         /**
 381          * {@inheritDoc}
 382          */
 383         public void run() {
 384                 fireProjectInsertStarted();
 385                 List<String> files = fileScanner.getFiles();
 386
 387                 /* create connection to node */
 388                 Connection connection = freenetInterface.getConnection("project-insert-" + random + counter++);
 389                 connection.setTempDirectory(tempDirectory);
 390                 boolean connected = false;
 391                 Throwable cause = null;
 392                 try {
 393                         connected = connection.connect();
 394                 } catch (IOException e1) {
 395                         cause = e1;
 396                 }
 397
 398                 if (!connected) {
 399                         fireProjectInsertFinished(false, cause);
 400                         return;
 401                 }
 402
 403                 Client client = new Client(connection);
 404
 405                 /* create containers */
 406                 final List<String> containers = new ArrayList<String>();
 407                 final Map<String, List<String>> containerFiles = new HashMap<String, List<String>>();
 408                 createContainers(files, containers, containerFiles);
 409
 410                 /* collect files */
 411                 int edition = project.getEdition();
 412                 String dirURI = "USK@" + project.getInsertURI() + "/" + project.getPath() + "/" + edition + "/";
 413                 ClientPutComplexDir putDir = new ClientPutComplexDir("dir-" + counter++, dirURI, tempDirectory);
 414                 if ((project.getIndexFile() != null) && (project.getIndexFile().length() > 0)) {
 415                         putDir.setDefaultName(project.getIndexFile());
 416                 }
 417                 putDir.setVerbosity(Verbosity.ALL);
 418                 putDir.setMaxRetries(-1);
 419                 for (String filename : files) {
 420                         FileEntry fileEntry = createFileEntry(filename, edition, containerFiles);
 421                         if (fileEntry != null) {
 422                                 putDir.addFileEntry(fileEntry);
 423                         }
 424                 }
 425
 426                 /* start request */
 427                 try {
 428                         client.execute(putDir);
 429                 } catch (IOException ioe1) {
 430                         fireProjectInsertFinished(false, ioe1);
 431                         return;
 432                 }
 433
 434                 /* parse progress and success messages */
 435                 String finalURI = null;
 436                 boolean firstMessage = true;
 437                 boolean success = false;
 438                 boolean finished = false;
 439                 boolean disconnected = false;
 440                 while (!finished) {
 441                         Message message = client.readMessage();
 442                         finished = (message == null) || (disconnected = client.isDisconnected());
 443                         if (firstMessage) {
 444                                 fireProjectUploadFinished();
 445                                 firstMessage = false;
 446                         }
 447                         logger.log(Level.FINE, "Received message: " + message);
 448                         if (!finished) {
 449                                 @SuppressWarnings("null")
 450                                 String messageName = message.getName();
 451                                 if ("URIGenerated".equals(messageName)) {
 452                                         finalURI = message.get("URI");
 453                                         fireProjectURIGenerated(finalURI);
 454                                 }
 455                                 if ("SimpleProgress".equals(messageName)) {
 456                                         int total = Integer.parseInt(message.get("Total"));
 457                                         int succeeded = Integer.parseInt(message.get("Succeeded"));
 458                                         int fatal = Integer.parseInt(message.get("FatallyFailed"));
 459                                         int failed = Integer.parseInt(message.get("Failed"));
 460                                         boolean finalized = Boolean.parseBoolean(message.get("FinalizedTotal"));
 461                                         fireProjectInsertProgress(succeeded, failed, fatal, total, finalized);
 462                                 }
 463                                 success = "PutSuccessful".equals(messageName);
 464                                 finished = success || "PutFailed".equals(messageName) || messageName.endsWith("Error");
 465                         }
 466                 }
 467
 468                 /* post-insert work */
 469                 fireProjectInsertFinished(success, disconnected ? new IOException("Connection terminated") : null);
 470                 if (success) {
 471                         @SuppressWarnings("null")
 472                         String editionPart = finalURI.substring(finalURI.lastIndexOf('/') + 1);
 473                         int newEdition = Integer.parseInt(editionPart);
 474                         project.setEdition(newEdition);
 475                         project.setLastInsertionTime(System.currentTimeMillis());
 476                 }
 477         }
 478
 479         //
 480         // INTERFACE FileScannerListener
 481         //
 482
 483         /**
 484          * {@inheritDoc}
 485          */
 486         public void fileScannerFinished(FileScanner fileScanner) {
 487                 if (!fileScanner.isError()) {
 488                         new Thread(this).start();
 489                 } else {
 490                         fireProjectInsertFinished(false, null);
 491                 }
 492                 fileScanner.removeFileScannerListener(this);
 493         }
 494
 495 }