src/de/todesbaum/jsite/application/ProjectInserter.java

   1 /*
   2  * jSite - a tool for uploading websites into Freenet
   3  * Copyright (C) 2006 David Roden
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  */
  19
  20 package de.todesbaum.jsite.application;
  21
  22 import java.io.ByteArrayInputStream;
  23 import java.io.ByteArrayOutputStream;
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.util.ArrayList;
  30 import java.util.HashMap;
  31 import java.util.List;
  32 import java.util.Map;
  33 import java.util.zip.ZipEntry;
  34 import java.util.zip.ZipOutputStream;
  35
  36 import de.todesbaum.jsite.gui.FileScanner;
  37 import de.todesbaum.jsite.gui.FileScannerListener;
  38 import de.todesbaum.util.freenet.fcp2.Client;
  39 import de.todesbaum.util.freenet.fcp2.ClientPutComplexDir;
  40 import de.todesbaum.util.freenet.fcp2.Connection;
  41 import de.todesbaum.util.freenet.fcp2.DirectFileEntry;
  42 import de.todesbaum.util.freenet.fcp2.FileEntry;
  43 import de.todesbaum.util.freenet.fcp2.Message;
  44 import de.todesbaum.util.freenet.fcp2.RedirectFileEntry;
  45 import de.todesbaum.util.freenet.fcp2.Verbosity;
  46 import de.todesbaum.util.io.Closer;
  47 import de.todesbaum.util.io.ReplacingOutputStream;
  48 import de.todesbaum.util.io.StreamCopier;
  49
  50 /**
  51  * Manages project inserts.
  52  *
  53  * @author David ‘Bombe’ Roden &lt;bombe@freenetproject.org&gt;
  54  */
  55 public class ProjectInserter implements FileScannerListener, Runnable {
  56
  57         /** Counter for FCP connection identifier. */
  58         private static int counter = 0;
  59
  60         /** Whether debug mode is set. */
  61         private boolean debug = false;
  62
  63         /** The list of insert listeners. */
  64         private List<InsertListener> insertListeners = new ArrayList<InsertListener>();
  65
  66         /** The freenet interface. */
  67         protected Freenet7Interface freenetInterface;
  68
  69         /** The project to insert. */
  70         protected Project project;
  71
  72         /** The file scanner. */
  73         private FileScanner fileScanner;
  74
  75         /** Object used for synchronization. */
  76         protected final Object lockObject = new Object();
  77
  78         /**
  79          * Adds a listener to the list of registered listeners.
  80          *
  81          * @param insertListener
  82          *            The listener to add
  83          */
  84         public void addInsertListener(InsertListener insertListener) {
  85                 insertListeners.add(insertListener);
  86         }
  87
  88         /**
  89          * Removes a listener from the list of registered listeners.
  90          *
  91          * @param insertListener
  92          *            The listener to remove
  93          */
  94         public void removeInsertListener(InsertListener insertListener) {
  95                 insertListeners.remove(insertListener);
  96         }
  97
  98         /**
  99          * Notifies all listeners that the project insert has started.
 100          *
 101          * @see InsertListener#projectInsertStarted(Project)
 102          */
 103         protected void fireProjectInsertStarted() {
 104                 for (InsertListener insertListener : insertListeners) {
 105                         insertListener.projectInsertStarted(project);
 106                 }
 107         }
 108
 109         /**
 110          * Notifies all listeners that the insert has generated a URI.
 111          *
 112          * @see InsertListener#projectURIGenerated(Project, String)
 113          * @param uri
 114          *            The generated URI
 115          */
 116         protected void fireProjectURIGenerated(String uri) {
 117                 for (InsertListener insertListener : insertListeners) {
 118                         insertListener.projectURIGenerated(project, uri);
 119                 }
 120         }
 121
 122         /**
 123          * Notifies all listeners that the insert has made some progress.
 124          *
 125          * @see InsertListener#projectInsertProgress(Project, int, int, int, int,
 126          *      boolean)
 127          * @param succeeded
 128          *            The number of succeeded blocks
 129          * @param failed
 130          *            The number of failed blocks
 131          * @param fatal
 132          *            The number of fatally failed blocks
 133          * @param total
 134          *            The total number of blocks
 135          * @param finalized
 136          *            <code>true</code> if the total number of blocks has already
 137          *            been finalized, <code>false</code> otherwise
 138          */
 139         protected void fireProjectInsertProgress(int succeeded, int failed, int fatal, int total, boolean finalized) {
 140                 for (InsertListener insertListener : insertListeners) {
 141                         insertListener.projectInsertProgress(project, succeeded, failed, fatal, total, finalized);
 142                 }
 143         }
 144
 145         /**
 146          * Notifies all listeners the project insert has finished.
 147          *
 148          * @see InsertListener#projectInsertFinished(Project, boolean, Throwable)
 149          * @param success
 150          *            <code>true</code> if the project was inserted successfully,
 151          *            <code>false</code> if it failed
 152          * @param cause
 153          *            The cause of the failure, if any
 154          */
 155         protected void fireProjectInsertFinished(boolean success, Throwable cause) {
 156                 for (InsertListener insertListener : insertListeners) {
 157                         insertListener.projectInsertFinished(project, success, cause);
 158                 }
 159         }
 160
 161         /**
 162          * Sets the debug mode.
 163          *
 164          * @param debug
 165          *            <code>true</code> to activate debug mode, <code>false</code>
 166          *            to deactivate
 167          */
 168         public void setDebug(boolean debug) {
 169                 this.debug = debug;
 170         }
 171
 172         /**
 173          * Sets the project to insert.
 174          *
 175          * @param project
 176          *            The project to insert
 177          */
 178         public void setProject(Project project) {
 179                 this.project = project;
 180         }
 181
 182         /**
 183          * Sets the freenet interface to use.
 184          *
 185          * @param freenetInterface
 186          *            The freenet interface to use
 187          */
 188         public void setFreenetInterface(Freenet7Interface freenetInterface) {
 189                 this.freenetInterface = freenetInterface;
 190         }
 191
 192         /**
 193          * Starts the insert.
 194          */
 195         public void start() {
 196                 fileScanner = new FileScanner(project);
 197                 fileScanner.addFileScannerListener(this);
 198                 new Thread(fileScanner).start();
 199         }
 200
 201         /**
 202          * Creates an input stream that delivers the given file, replacing edition
 203          * tokens in the file’s content, if necessary.
 204          *
 205          * @param filename
 206          *            The name of the file
 207          * @param fileOption
 208          *            The file options
 209          * @param edition
 210          *            The current edition
 211          * @param length
 212          *            An array containing a single long which is used to
 213          *            <em>return</em> the final length of the file, after all
 214          *            replacements
 215          * @return The input stream for the file
 216          * @throws IOException
 217          *             if an I/O error occurs
 218          */
 219         private InputStream createFileInputStream(String filename, FileOption fileOption, int edition, long[] length) throws IOException {
 220                 File file = new File(project.getLocalPath(), filename);
 221                 length[0] = file.length();
 222                 if (!fileOption.getReplaceEdition()) {
 223                         return new FileInputStream(file);
 224                 }
 225                 ByteArrayOutputStream filteredByteOutputStream = new ByteArrayOutputStream(Math.min(Integer.MAX_VALUE, (int) length[0]));
 226                 ReplacingOutputStream outputStream = new ReplacingOutputStream(filteredByteOutputStream);
 227                 FileInputStream fileInput = new FileInputStream(file);
 228                 try {
 229                         outputStream.addReplacement("$[EDITION]", String.valueOf(edition));
 230                         outputStream.addReplacement("$[URI]", project.getFinalRequestURI(0));
 231                         for (int index = 1; index <= fileOption.getEditionRange(); index++) {
 232                                 outputStream.addReplacement("$[URI+" + index + "]", project.getFinalRequestURI(index));
 233                                 outputStream.addReplacement("$[EDITION+" + index + "]", String.valueOf(edition + index));
 234                         }
 235                         StreamCopier.copy(fileInput, outputStream, length[0]);
 236                 } finally {
 237                         Closer.close(fileInput);
 238                         Closer.close(outputStream);
 239                         Closer.close(filteredByteOutputStream);
 240                 }
 241                 byte[] filteredBytes = filteredByteOutputStream.toByteArray();
 242                 length[0] = filteredBytes.length;
 243                 return new ByteArrayInputStream(filteredBytes);
 244         }
 245
 246         /**
 247          * Creates an input stream for a container.
 248          *
 249          * @param containerFiles
 250          *            All container definitions
 251          * @param containerName
 252          *            The name of the container to create
 253          * @param edition
 254          *            The current edition
 255          * @param containerLength
 256          *            An array containing a single long which is used to
 257          *            <em>return</em> the final length of the container stream,
 258          *            after all replacements
 259          * @return The input stream for the container
 260          * @throws IOException
 261          *             if an I/O error occurs
 262          */
 263         private InputStream createContainerInputStream(Map<String, List<String>> containerFiles, String containerName, int edition, long[] containerLength) throws IOException {
 264                 File tempFile = File.createTempFile("jsite", ".zip");
 265                 tempFile.deleteOnExit();
 266                 FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
 267                 ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
 268                 try {
 269                         for (String filename : containerFiles.get(containerName)) {
 270                                 File dataFile = new File(project.getLocalPath(), filename);
 271                                 if (dataFile.exists()) {
 272                                         ZipEntry zipEntry = new ZipEntry(filename);
 273                                         long[] fileLength = new long[1];
 274                                         InputStream wrappedInputStream = createFileInputStream(filename, project.getFileOption(filename), edition, fileLength);
 275                                         try {
 276                                                 zipOutputStream.putNextEntry(zipEntry);
 277                                                 StreamCopier.copy(wrappedInputStream, zipOutputStream, fileLength[0]);
 278                                         } finally {
 279                                                 zipOutputStream.closeEntry();
 280                                                 wrappedInputStream.close();
 281                                         }
 282                                 }
 283                         }
 284                 } finally {
 285                         zipOutputStream.closeEntry();
 286                         Closer.close(zipOutputStream);
 287                         Closer.close(fileOutputStream);
 288                 }
 289
 290                 containerLength[0] = tempFile.length();
 291                 return new FileInputStream(tempFile);
 292         }
 293
 294         /**
 295          * Creates a file entry suitable for handing in to
 296          * {@link ClientPutComplexDir#addFileEntry(FileEntry)}.
 297          *
 298          * @param filename
 299          *            The name of the file to insert
 300          * @param edition
 301          *            The current edition
 302          * @param containerFiles
 303          *            The container definitions
 304          * @return A file entry for the given file
 305          */
 306         private FileEntry createFileEntry(String filename, int edition, Map<String, List<String>> containerFiles) {
 307                 FileEntry fileEntry = null;
 308                 FileOption fileOption = project.getFileOption(filename);
 309                 if (filename.startsWith("/container/:")) {
 310                         String containerName = filename.substring("/container/:".length());
 311                         try {
 312                                 long[] containerLength = new long[1];
 313                                 InputStream containerInputStream = createContainerInputStream(containerFiles, containerName, edition, containerLength);
 314                                 fileEntry = new DirectFileEntry(containerName + ".zip", "application/zip", containerInputStream, containerLength[0]);
 315                         } catch (IOException ioe1) {
 316                                 /* ignore, null is returned. */
 317                         }
 318                 } else {
 319                         if (fileOption.isInsert()) {
 320                                 try {
 321                                         long[] fileLength = new long[1];
 322                                         InputStream fileEntryInputStream = createFileInputStream(filename, fileOption, edition, fileLength);
 323                                         fileEntry = new DirectFileEntry(filename, project.getFileOption(filename).getMimeType(), fileEntryInputStream, fileLength[0]);
 324                                 } catch (IOException ioe1) {
 325                                         /* ignore, null is returned. */
 326                                 }
 327                         } else {
 328                                 fileEntry = new RedirectFileEntry(filename, fileOption.getMimeType(), fileOption.getCustomKey());
 329                         }
 330                 }
 331                 return fileEntry;
 332         }
 333
 334         /**
 335          * Creates container definitions.
 336          *
 337          * @param files
 338          *            The list of all files
 339          * @param containers
 340          *            The list of all containers
 341          * @param containerFiles
 342          *            Empty map that will be filled with container definitions
 343          */
 344         private void createContainers(List<String> files, List<String> containers, Map<String, List<String>> containerFiles) {
 345                 for (String filename : new ArrayList<String>(files)) {
 346                         FileOption fileOption = project.getFileOption(filename);
 347                         String containerName = fileOption.getContainer();
 348                         if (!containerName.equals("")) {
 349                                 if (!containers.contains(containerName)) {
 350                                         containers.add(containerName);
 351                                         containerFiles.put(containerName, new ArrayList<String>());
 352                                         /* hmm. looks like a hack to me. */
 353                                         files.add("/container/:" + containerName);
 354                                 }
 355                                 containerFiles.get(containerName).add(filename);
 356                                 files.remove(filename);
 357                         }
 358                 }
 359         }
 360
 361         /**
 362          * {@inheritDoc}
 363          */
 364         public void run() {
 365                 fireProjectInsertStarted();
 366                 List<String> files = fileScanner.getFiles();
 367
 368                 /* create connection to node */
 369                 Connection connection = freenetInterface.getConnection("project-insert-" + counter++);
 370                 boolean connected = false;
 371                 Throwable cause = null;
 372                 try {
 373                         connected = connection.connect();
 374                 } catch (IOException e1) {
 375                         cause = e1;
 376                 }
 377
 378                 if (!connected) {
 379                         fireProjectInsertFinished(false, cause);
 380                         return;
 381                 }
 382
 383                 Client client = new Client(connection);
 384
 385                 /* create containers */
 386                 final List<String> containers = new ArrayList<String>();
 387                 final Map<String, List<String>> containerFiles = new HashMap<String, List<String>>();
 388                 createContainers(files, containers, containerFiles);
 389
 390                 /* collect files */
 391                 int edition = project.getEdition();
 392                 String dirURI = "USK@" + project.getInsertURI() + "/" + project.getPath() + "/" + edition + "/";
 393                 ClientPutComplexDir putDir = new ClientPutComplexDir("dir-" + counter++, dirURI);
 394                 putDir.setDefaultName(project.getIndexFile());
 395                 putDir.setVerbosity(Verbosity.ALL);
 396                 putDir.setMaxRetries(-1);
 397                 for (String filename : files) {
 398                         FileEntry fileEntry = createFileEntry(filename, edition, containerFiles);
 399                         if (fileEntry != null) {
 400                                 putDir.addFileEntry(fileEntry);
 401                         }
 402                 }
 403
 404                 /* start request */
 405                 try {
 406                         client.execute(putDir);
 407                 } catch (IOException ioe1) {
 408                         fireProjectInsertFinished(false, ioe1);
 409                         return;
 410                 }
 411
 412                 /* parse progress and success messages */
 413                 String finalURI = null;
 414                 boolean success = false;
 415                 boolean finished = false;
 416                 boolean disconnected = false;
 417                 while (!finished) {
 418                         Message message = client.readMessage();
 419                         finished = (message == null) || (disconnected = client.isDisconnected());
 420                         if (debug) {
 421                                 System.out.println(message);
 422                         }
 423                         if (!finished) {
 424                                 String messageName = message.getName();
 425                                 if ("URIGenerated".equals(messageName)) {
 426                                         finalURI = message.get("URI");
 427                                         fireProjectURIGenerated(finalURI);
 428                                 }
 429                                 if ("SimpleProgress".equals(messageName)) {
 430                                         int total = Integer.parseInt(message.get("Total"));
 431                                         int succeeded = Integer.parseInt(message.get("Succeeded"));
 432                                         int fatal = Integer.parseInt(message.get("FatallyFailed"));
 433                                         int failed = Integer.parseInt(message.get("Failed"));
 434                                         boolean finalized = Boolean.parseBoolean(message.get("FinalizedTotal"));
 435                                         fireProjectInsertProgress(succeeded, failed, fatal, total, finalized);
 436                                 }
 437                                 success = "PutSuccessful".equals(messageName);
 438                                 finished = success || "PutFailed".equals(messageName) || messageName.endsWith("Error");
 439                         }
 440                 }
 441
 442                 /* post-insert work */
 443                 fireProjectInsertFinished(success, disconnected ? new IOException("Connection terminated") : null);
 444                 if (success) {
 445                         String editionPart = finalURI.substring(finalURI.lastIndexOf('/') + 1);
 446                         int newEdition = Integer.parseInt(editionPart);
 447                         project.setEdition(newEdition);
 448                         project.setLastInsertionTime(System.currentTimeMillis());
 449                 }
 450         }
 451
 452         //
 453         // INTERFACE FileScannerListener
 454         //
 455
 456         /**
 457          * {@inheritDoc}
 458          */
 459         public void fileScannerFinished(FileScanner fileScanner) {
 460                 if (!fileScanner.isError()) {
 461                         new Thread(this).start();
 462                 } else {
 463                         fireProjectInsertFinished(false, null);
 464                 }
 465                 fileScanner.removeFileScannerListener(this);
 466         }
 467
 468 }