src/de/todesbaum/jsite/application/ProjectInserter.java

   1 /*
   2  * jSite - a tool for uploading websites into Freenet
   3  * Copyright (C) 2006 David Roden
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  */
  19
  20 package de.todesbaum.jsite.application;
  21
  22 import java.io.ByteArrayInputStream;
  23 import java.io.ByteArrayOutputStream;
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.util.ArrayList;
  30 import java.util.HashMap;
  31 import java.util.List;
  32 import java.util.Map;
  33 import java.util.zip.ZipEntry;
  34 import java.util.zip.ZipOutputStream;
  35
  36 import de.todesbaum.jsite.gui.FileScanner;
  37 import de.todesbaum.jsite.gui.FileScannerListener;
  38 import de.todesbaum.util.freenet.fcp2.Client;
  39 import de.todesbaum.util.freenet.fcp2.ClientPutComplexDir;
  40 import de.todesbaum.util.freenet.fcp2.Connection;
  41 import de.todesbaum.util.freenet.fcp2.DirectFileEntry;
  42 import de.todesbaum.util.freenet.fcp2.FileEntry;
  43 import de.todesbaum.util.freenet.fcp2.Message;
  44 import de.todesbaum.util.freenet.fcp2.RedirectFileEntry;
  45 import de.todesbaum.util.freenet.fcp2.Verbosity;
  46 import de.todesbaum.util.io.Closer;
  47 import de.todesbaum.util.io.ReplacingOutputStream;
  48 import de.todesbaum.util.io.StreamCopier;
  49
  50 /**
  51  * Manages project inserts.
  52  *
  53  * @author David ‘Bombe’ Roden &lt;bombe@freenetproject.org&gt;
  54  */
  55 public class ProjectInserter implements FileScannerListener, Runnable {
  56
  57         /** Random number for FCP instances. */
  58         private static final int random = (int) (Math.random() * Integer.MAX_VALUE);
  59
  60         /** Counter for FCP connection identifier. */
  61         private static int counter = 0;
  62
  63         /** Whether debug mode is set. */
  64         private boolean debug = false;
  65
  66         /** The list of insert listeners. */
  67         private List<InsertListener> insertListeners = new ArrayList<InsertListener>();
  68
  69         /** The freenet interface. */
  70         protected Freenet7Interface freenetInterface;
  71
  72         /** The project to insert. */
  73         protected Project project;
  74
  75         /** The file scanner. */
  76         private FileScanner fileScanner;
  77
  78         /** Object used for synchronization. */
  79         protected final Object lockObject = new Object();
  80
  81         /**
  82          * Adds a listener to the list of registered listeners.
  83          *
  84          * @param insertListener
  85          *            The listener to add
  86          */
  87         public void addInsertListener(InsertListener insertListener) {
  88                 insertListeners.add(insertListener);
  89         }
  90
  91         /**
  92          * Removes a listener from the list of registered listeners.
  93          *
  94          * @param insertListener
  95          *            The listener to remove
  96          */
  97         public void removeInsertListener(InsertListener insertListener) {
  98                 insertListeners.remove(insertListener);
  99         }
 100
 101         /**
 102          * Notifies all listeners that the project insert has started.
 103          *
 104          * @see InsertListener#projectInsertStarted(Project)
 105          */
 106         protected void fireProjectInsertStarted() {
 107                 for (InsertListener insertListener : insertListeners) {
 108                         insertListener.projectInsertStarted(project);
 109                 }
 110         }
 111
 112         /**
 113          * Notifies all listeners that the insert has generated a URI.
 114          *
 115          * @see InsertListener#projectURIGenerated(Project, String)
 116          * @param uri
 117          *            The generated URI
 118          */
 119         protected void fireProjectURIGenerated(String uri) {
 120                 for (InsertListener insertListener : insertListeners) {
 121                         insertListener.projectURIGenerated(project, uri);
 122                 }
 123         }
 124
 125         /**
 126          * Notifies all listeners that the insert has made some progress.
 127          *
 128          * @see InsertListener#projectInsertProgress(Project, int, int, int, int,
 129          *      boolean)
 130          * @param succeeded
 131          *            The number of succeeded blocks
 132          * @param failed
 133          *            The number of failed blocks
 134          * @param fatal
 135          *            The number of fatally failed blocks
 136          * @param total
 137          *            The total number of blocks
 138          * @param finalized
 139          *            <code>true</code> if the total number of blocks has already
 140          *            been finalized, <code>false</code> otherwise
 141          */
 142         protected void fireProjectInsertProgress(int succeeded, int failed, int fatal, int total, boolean finalized) {
 143                 for (InsertListener insertListener : insertListeners) {
 144                         insertListener.projectInsertProgress(project, succeeded, failed, fatal, total, finalized);
 145                 }
 146         }
 147
 148         /**
 149          * Notifies all listeners the project insert has finished.
 150          *
 151          * @see InsertListener#projectInsertFinished(Project, boolean, Throwable)
 152          * @param success
 153          *            <code>true</code> if the project was inserted successfully,
 154          *            <code>false</code> if it failed
 155          * @param cause
 156          *            The cause of the failure, if any
 157          */
 158         protected void fireProjectInsertFinished(boolean success, Throwable cause) {
 159                 for (InsertListener insertListener : insertListeners) {
 160                         insertListener.projectInsertFinished(project, success, cause);
 161                 }
 162         }
 163
 164         /**
 165          * Sets the debug mode.
 166          *
 167          * @param debug
 168          *            <code>true</code> to activate debug mode, <code>false</code>
 169          *            to deactivate
 170          */
 171         public void setDebug(boolean debug) {
 172                 this.debug = debug;
 173         }
 174
 175         /**
 176          * Sets the project to insert.
 177          *
 178          * @param project
 179          *            The project to insert
 180          */
 181         public void setProject(Project project) {
 182                 this.project = project;
 183         }
 184
 185         /**
 186          * Sets the freenet interface to use.
 187          *
 188          * @param freenetInterface
 189          *            The freenet interface to use
 190          */
 191         public void setFreenetInterface(Freenet7Interface freenetInterface) {
 192                 this.freenetInterface = freenetInterface;
 193         }
 194
 195         /**
 196          * Starts the insert.
 197          */
 198         public void start() {
 199                 fileScanner = new FileScanner(project);
 200                 fileScanner.addFileScannerListener(this);
 201                 new Thread(fileScanner).start();
 202         }
 203
 204         /**
 205          * Creates an input stream that delivers the given file, replacing edition
 206          * tokens in the file’s content, if necessary.
 207          *
 208          * @param filename
 209          *            The name of the file
 210          * @param fileOption
 211          *            The file options
 212          * @param edition
 213          *            The current edition
 214          * @param length
 215          *            An array containing a single long which is used to
 216          *            <em>return</em> the final length of the file, after all
 217          *            replacements
 218          * @return The input stream for the file
 219          * @throws IOException
 220          *             if an I/O error occurs
 221          */
 222         private InputStream createFileInputStream(String filename, FileOption fileOption, int edition, long[] length) throws IOException {
 223                 File file = new File(project.getLocalPath(), filename);
 224                 length[0] = file.length();
 225                 if (!fileOption.getReplaceEdition()) {
 226                         return new FileInputStream(file);
 227                 }
 228                 ByteArrayOutputStream filteredByteOutputStream = new ByteArrayOutputStream(Math.min(Integer.MAX_VALUE, (int) length[0]));
 229                 ReplacingOutputStream outputStream = new ReplacingOutputStream(filteredByteOutputStream);
 230                 FileInputStream fileInput = new FileInputStream(file);
 231                 try {
 232                         outputStream.addReplacement("$[EDITION]", String.valueOf(edition));
 233                         outputStream.addReplacement("$[URI]", project.getFinalRequestURI(0));
 234                         for (int index = 1; index <= fileOption.getEditionRange(); index++) {
 235                                 outputStream.addReplacement("$[URI+" + index + "]", project.getFinalRequestURI(index));
 236                                 outputStream.addReplacement("$[EDITION+" + index + "]", String.valueOf(edition + index));
 237                         }
 238                         StreamCopier.copy(fileInput, outputStream, length[0]);
 239                 } finally {
 240                         Closer.close(fileInput);
 241                         Closer.close(outputStream);
 242                         Closer.close(filteredByteOutputStream);
 243                 }
 244                 byte[] filteredBytes = filteredByteOutputStream.toByteArray();
 245                 length[0] = filteredBytes.length;
 246                 return new ByteArrayInputStream(filteredBytes);
 247         }
 248
 249         /**
 250          * Creates an input stream for a container.
 251          *
 252          * @param containerFiles
 253          *            All container definitions
 254          * @param containerName
 255          *            The name of the container to create
 256          * @param edition
 257          *            The current edition
 258          * @param containerLength
 259          *            An array containing a single long which is used to
 260          *            <em>return</em> the final length of the container stream,
 261          *            after all replacements
 262          * @return The input stream for the container
 263          * @throws IOException
 264          *             if an I/O error occurs
 265          */
 266         private InputStream createContainerInputStream(Map<String, List<String>> containerFiles, String containerName, int edition, long[] containerLength) throws IOException {
 267                 File tempFile = File.createTempFile("jsite", ".zip");
 268                 tempFile.deleteOnExit();
 269                 FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
 270                 ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
 271                 try {
 272                         for (String filename : containerFiles.get(containerName)) {
 273                                 File dataFile = new File(project.getLocalPath(), filename);
 274                                 if (dataFile.exists()) {
 275                                         ZipEntry zipEntry = new ZipEntry(filename);
 276                                         long[] fileLength = new long[1];
 277                                         InputStream wrappedInputStream = createFileInputStream(filename, project.getFileOption(filename), edition, fileLength);
 278                                         try {
 279                                                 zipOutputStream.putNextEntry(zipEntry);
 280                                                 StreamCopier.copy(wrappedInputStream, zipOutputStream, fileLength[0]);
 281                                         } finally {
 282                                                 zipOutputStream.closeEntry();
 283                                                 wrappedInputStream.close();
 284                                         }
 285                                 }
 286                         }
 287                 } finally {
 288                         zipOutputStream.closeEntry();
 289                         Closer.close(zipOutputStream);
 290                         Closer.close(fileOutputStream);
 291                 }
 292
 293                 containerLength[0] = tempFile.length();
 294                 return new FileInputStream(tempFile);
 295         }
 296
 297         /**
 298          * Creates a file entry suitable for handing in to
 299          * {@link ClientPutComplexDir#addFileEntry(FileEntry)}.
 300          *
 301          * @param filename
 302          *            The name of the file to insert
 303          * @param edition
 304          *            The current edition
 305          * @param containerFiles
 306          *            The container definitions
 307          * @return A file entry for the given file
 308          */
 309         private FileEntry createFileEntry(String filename, int edition, Map<String, List<String>> containerFiles) {
 310                 FileEntry fileEntry = null;
 311                 FileOption fileOption = project.getFileOption(filename);
 312                 if (filename.startsWith("/container/:")) {
 313                         String containerName = filename.substring("/container/:".length());
 314                         try {
 315                                 long[] containerLength = new long[1];
 316                                 InputStream containerInputStream = createContainerInputStream(containerFiles, containerName, edition, containerLength);
 317                                 fileEntry = new DirectFileEntry(containerName + ".zip", "application/zip", containerInputStream, containerLength[0]);
 318                         } catch (IOException ioe1) {
 319                                 /* ignore, null is returned. */
 320                         }
 321                 } else {
 322                         if (fileOption.isInsert()) {
 323                                 try {
 324                                         long[] fileLength = new long[1];
 325                                         InputStream fileEntryInputStream = createFileInputStream(filename, fileOption, edition, fileLength);
 326                                         fileEntry = new DirectFileEntry(filename, project.getFileOption(filename).getMimeType(), fileEntryInputStream, fileLength[0]);
 327                                 } catch (IOException ioe1) {
 328                                         /* ignore, null is returned. */
 329                                 }
 330                         } else {
 331                                 fileEntry = new RedirectFileEntry(filename, fileOption.getMimeType(), fileOption.getCustomKey());
 332                         }
 333                 }
 334                 return fileEntry;
 335         }
 336
 337         /**
 338          * Creates container definitions.
 339          *
 340          * @param files
 341          *            The list of all files
 342          * @param containers
 343          *            The list of all containers
 344          * @param containerFiles
 345          *            Empty map that will be filled with container definitions
 346          */
 347         private void createContainers(List<String> files, List<String> containers, Map<String, List<String>> containerFiles) {
 348                 for (String filename : new ArrayList<String>(files)) {
 349                         FileOption fileOption = project.getFileOption(filename);
 350                         String containerName = fileOption.getContainer();
 351                         if (!containerName.equals("")) {
 352                                 if (!containers.contains(containerName)) {
 353                                         containers.add(containerName);
 354                                         containerFiles.put(containerName, new ArrayList<String>());
 355                                         /* hmm. looks like a hack to me. */
 356                                         files.add("/container/:" + containerName);
 357                                 }
 358                                 containerFiles.get(containerName).add(filename);
 359                                 files.remove(filename);
 360                         }
 361                 }
 362         }
 363
 364         /**
 365          * {@inheritDoc}
 366          */
 367         public void run() {
 368                 fireProjectInsertStarted();
 369                 List<String> files = fileScanner.getFiles();
 370
 371                 /* create connection to node */
 372                 Connection connection = freenetInterface.getConnection("project-insert-" + random + counter++);
 373                 boolean connected = false;
 374                 Throwable cause = null;
 375                 try {
 376                         connected = connection.connect();
 377                 } catch (IOException e1) {
 378                         cause = e1;
 379                 }
 380
 381                 if (!connected) {
 382                         fireProjectInsertFinished(false, cause);
 383                         return;
 384                 }
 385
 386                 Client client = new Client(connection);
 387
 388                 /* create containers */
 389                 final List<String> containers = new ArrayList<String>();
 390                 final Map<String, List<String>> containerFiles = new HashMap<String, List<String>>();
 391                 createContainers(files, containers, containerFiles);
 392
 393                 /* collect files */
 394                 int edition = project.getEdition();
 395                 String dirURI = "USK@" + project.getInsertURI() + "/" + project.getPath() + "/" + edition + "/";
 396                 ClientPutComplexDir putDir = new ClientPutComplexDir("dir-" + counter++, dirURI);
 397                 if ((project.getIndexFile() != null) && (project.getIndexFile().length() > 0)) {
 398                         putDir.setDefaultName(project.getIndexFile());
 399                 }
 400                 putDir.setVerbosity(Verbosity.ALL);
 401                 putDir.setMaxRetries(-1);
 402                 for (String filename : files) {
 403                         FileEntry fileEntry = createFileEntry(filename, edition, containerFiles);
 404                         if (fileEntry != null) {
 405                                 putDir.addFileEntry(fileEntry);
 406                         }
 407                 }
 408
 409                 /* start request */
 410                 try {
 411                         client.execute(putDir);
 412                 } catch (IOException ioe1) {
 413                         fireProjectInsertFinished(false, ioe1);
 414                         return;
 415                 }
 416
 417                 /* parse progress and success messages */
 418                 String finalURI = null;
 419                 boolean success = false;
 420                 boolean finished = false;
 421                 boolean disconnected = false;
 422                 while (!finished) {
 423                         Message message = client.readMessage();
 424                         finished = (message == null) || (disconnected = client.isDisconnected());
 425                         if (debug) {
 426                                 System.out.println(message);
 427                         }
 428                         if (!finished) {
 429                                 @SuppressWarnings("null")
 430                                 String messageName = message.getName();
 431                                 if ("URIGenerated".equals(messageName)) {
 432                                         finalURI = message.get("URI");
 433                                         fireProjectURIGenerated(finalURI);
 434                                 }
 435                                 if ("SimpleProgress".equals(messageName)) {
 436                                         int total = Integer.parseInt(message.get("Total"));
 437                                         int succeeded = Integer.parseInt(message.get("Succeeded"));
 438                                         int fatal = Integer.parseInt(message.get("FatallyFailed"));
 439                                         int failed = Integer.parseInt(message.get("Failed"));
 440                                         boolean finalized = Boolean.parseBoolean(message.get("FinalizedTotal"));
 441                                         fireProjectInsertProgress(succeeded, failed, fatal, total, finalized);
 442                                 }
 443                                 success = "PutSuccessful".equals(messageName);
 444                                 finished = success || "PutFailed".equals(messageName) || messageName.endsWith("Error");
 445                         }
 446                 }
 447
 448                 /* post-insert work */
 449                 fireProjectInsertFinished(success, disconnected ? new IOException("Connection terminated") : null);
 450                 if (success) {
 451                         @SuppressWarnings("null")
 452                         String editionPart = finalURI.substring(finalURI.lastIndexOf('/') + 1);
 453                         int newEdition = Integer.parseInt(editionPart);
 454                         project.setEdition(newEdition);
 455                         project.setLastInsertionTime(System.currentTimeMillis());
 456                 }
 457         }
 458
 459         //
 460         // INTERFACE FileScannerListener
 461         //
 462
 463         /**
 464          * {@inheritDoc}
 465          */
 466         public void fileScannerFinished(FileScanner fileScanner) {
 467                 if (!fileScanner.isError()) {
 468                         new Thread(this).start();
 469                 } else {
 470                         fireProjectInsertFinished(false, null);
 471                 }
 472                 fileScanner.removeFileScannerListener(this);
 473         }
 474
 475 }