1 | /* |
2 | * Copyright 2006-2012 the original author or authors. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | package org.springframework.batch.item.file; |
18 | |
19 | import java.io.BufferedWriter; |
20 | import java.io.File; |
21 | import java.io.FileOutputStream; |
22 | import java.io.IOException; |
23 | import java.io.Writer; |
24 | import java.nio.channels.Channels; |
25 | import java.nio.channels.FileChannel; |
26 | import java.nio.charset.UnsupportedCharsetException; |
27 | import java.util.List; |
28 | |
29 | import org.apache.commons.logging.Log; |
30 | import org.apache.commons.logging.LogFactory; |
31 | import org.springframework.batch.item.ExecutionContext; |
32 | import org.springframework.batch.item.ItemStream; |
33 | import org.springframework.batch.item.ItemStreamException; |
34 | import org.springframework.batch.item.WriteFailedException; |
35 | import org.springframework.batch.item.WriterNotOpenException; |
36 | import org.springframework.batch.item.file.transform.LineAggregator; |
37 | import org.springframework.batch.item.support.AbstractItemStreamItemWriter; |
38 | import org.springframework.batch.item.util.FileUtils; |
39 | import org.springframework.batch.support.transaction.TransactionAwareBufferedWriter; |
40 | import org.springframework.beans.factory.InitializingBean; |
41 | import org.springframework.core.io.Resource; |
42 | import org.springframework.util.Assert; |
43 | import org.springframework.util.ClassUtils; |
44 | |
45 | /** |
46 | * This class is an item writer that writes data to a file or stream. The writer |
47 | * also provides restart. The location of the output file is defined by a |
48 | * {@link Resource} and must represent a writable file.<br/> |
49 | * |
50 | * Uses buffered writer to improve performance.<br/> |
51 | * |
52 | * The implementation is *not* thread-safe. |
53 | * |
54 | * @author Waseem Malik |
55 | * @author Tomas Slanina |
56 | * @author Robert Kasanicky |
57 | * @author Dave Syer |
58 | * @author Michael Minella |
59 | */ |
60 | public class FlatFileItemWriter<T> extends AbstractItemStreamItemWriter<T> implements ResourceAwareItemWriterItemStream<T>, |
61 | InitializingBean { |
62 | |
63 | private static final boolean DEFAULT_TRANSACTIONAL = true; |
64 | |
65 | protected static final Log logger = LogFactory.getLog(FlatFileItemWriter.class); |
66 | |
67 | private static final String DEFAULT_LINE_SEPARATOR = System.getProperty("line.separator"); |
68 | |
69 | private static final String WRITTEN_STATISTICS_NAME = "written"; |
70 | |
71 | private static final String RESTART_DATA_NAME = "current.count"; |
72 | |
73 | private Resource resource; |
74 | |
75 | private OutputState state = null; |
76 | |
77 | private LineAggregator<T> lineAggregator; |
78 | |
79 | private boolean saveState = true; |
80 | |
81 | private boolean forceSync = false; |
82 | |
83 | private boolean shouldDeleteIfExists = true; |
84 | |
85 | private boolean shouldDeleteIfEmpty = false; |
86 | |
87 | private String encoding = OutputState.DEFAULT_CHARSET; |
88 | |
89 | private FlatFileHeaderCallback headerCallback; |
90 | |
91 | private FlatFileFooterCallback footerCallback; |
92 | |
93 | private String lineSeparator = DEFAULT_LINE_SEPARATOR; |
94 | |
95 | private boolean transactional = DEFAULT_TRANSACTIONAL; |
96 | |
97 | private boolean append = false; |
98 | |
99 | public FlatFileItemWriter() { |
100 | this.setExecutionContextName(ClassUtils.getShortName(FlatFileItemWriter.class)); |
101 | } |
102 | |
103 | /** |
104 | * Assert that mandatory properties (lineAggregator) are set. |
105 | * |
106 | * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() |
107 | */ |
108 | @Override |
109 | public void afterPropertiesSet() throws Exception { |
110 | Assert.notNull(lineAggregator, "A LineAggregator must be provided."); |
111 | if (append) { |
112 | shouldDeleteIfExists = false; |
113 | } |
114 | } |
115 | |
116 | /** |
117 | * Flag to indicate that changes should be force-synced to disk on flush. |
118 | * Defaults to false, which means that even with a local disk changes could |
119 | * be lost if the OS crashes in between a write and a cache flush. Setting |
120 | * to true may result in slower performance for usage patterns involving many |
121 | * frequent writes. |
122 | * |
123 | * @param forceSync the flag value to set |
124 | */ |
125 | public void setForceSync(boolean forceSync) { |
126 | this.forceSync = forceSync; |
127 | } |
128 | |
129 | /** |
130 | * Public setter for the line separator. Defaults to the System property |
131 | * line.separator. |
132 | * @param lineSeparator the line separator to set |
133 | */ |
134 | public void setLineSeparator(String lineSeparator) { |
135 | this.lineSeparator = lineSeparator; |
136 | } |
137 | |
138 | /** |
139 | * Public setter for the {@link LineAggregator}. This will be used to |
140 | * translate the item into a line for output. |
141 | * |
142 | * @param lineAggregator the {@link LineAggregator} to set |
143 | */ |
144 | public void setLineAggregator(LineAggregator<T> lineAggregator) { |
145 | this.lineAggregator = lineAggregator; |
146 | } |
147 | |
148 | /** |
149 | * Setter for resource. Represents a file that can be written. |
150 | * |
151 | * @param resource |
152 | */ |
153 | @Override |
154 | public void setResource(Resource resource) { |
155 | this.resource = resource; |
156 | } |
157 | |
158 | /** |
159 | * Sets encoding for output template. |
160 | */ |
161 | public void setEncoding(String newEncoding) { |
162 | this.encoding = newEncoding; |
163 | } |
164 | |
165 | /** |
166 | * Flag to indicate that the target file should be deleted if it already |
167 | * exists, otherwise it will be created. Defaults to true, so no appending |
168 | * except on restart. If set to false and {@link #setAppendAllowed(boolean) |
169 | * appendAllowed} is also false then there will be an exception when the |
170 | * stream is opened to prevent existing data being potentially corrupted. |
171 | * |
172 | * @param shouldDeleteIfExists the flag value to set |
173 | */ |
174 | public void setShouldDeleteIfExists(boolean shouldDeleteIfExists) { |
175 | this.shouldDeleteIfExists = shouldDeleteIfExists; |
176 | } |
177 | |
178 | /** |
179 | * Flag to indicate that the target file should be appended if it already |
180 | * exists. If this flag is set then the flag |
181 | * {@link #setShouldDeleteIfExists(boolean) shouldDeleteIfExists} is |
182 | * automatically set to false, so that flag should not be set explicitly. |
183 | * Defaults value is false. |
184 | * |
185 | * @param append the flag value to set |
186 | */ |
187 | public void setAppendAllowed(boolean append) { |
188 | this.append = append; |
189 | this.shouldDeleteIfExists = false; |
190 | } |
191 | |
192 | /** |
193 | * Flag to indicate that the target file should be deleted if no lines have |
194 | * been written (other than header and footer) on close. Defaults to false. |
195 | * |
196 | * @param shouldDeleteIfEmpty the flag value to set |
197 | */ |
198 | public void setShouldDeleteIfEmpty(boolean shouldDeleteIfEmpty) { |
199 | this.shouldDeleteIfEmpty = shouldDeleteIfEmpty; |
200 | } |
201 | |
202 | /** |
203 | * Set the flag indicating whether or not state should be saved in the |
204 | * provided {@link ExecutionContext} during the {@link ItemStream} call to |
205 | * update. Setting this to false means that it will always start at the |
206 | * beginning on a restart. |
207 | * |
208 | * @param saveState |
209 | */ |
210 | public void setSaveState(boolean saveState) { |
211 | this.saveState = saveState; |
212 | } |
213 | |
214 | /** |
215 | * headerCallback will be called before writing the first item to file. |
216 | * Newline will be automatically appended after the header is written. |
217 | */ |
218 | public void setHeaderCallback(FlatFileHeaderCallback headerCallback) { |
219 | this.headerCallback = headerCallback; |
220 | } |
221 | |
222 | /** |
223 | * footerCallback will be called after writing the last item to file, but |
224 | * before the file is closed. |
225 | */ |
226 | public void setFooterCallback(FlatFileFooterCallback footerCallback) { |
227 | this.footerCallback = footerCallback; |
228 | } |
229 | |
230 | /** |
231 | * Flag to indicate that writing to the buffer should be delayed if a |
232 | * transaction is active. Defaults to true. |
233 | */ |
234 | public void setTransactional(boolean transactional) { |
235 | this.transactional = transactional; |
236 | } |
237 | |
238 | /** |
239 | * Writes out a string followed by a "new line", where the format of the new |
240 | * line separator is determined by the underlying operating system. If the |
241 | * input is not a String and a converter is available the converter will be |
242 | * applied and then this method recursively called with the result. If the |
243 | * input is an array or collection each value will be written to a separate |
244 | * line (recursively calling this method for each value). If no converter is |
245 | * supplied the input object's toString method will be used.<br/> |
246 | * |
247 | * @param items list of items to be written to output stream |
248 | * @throws Exception if the transformer or file output fail, |
249 | * WriterNotOpenException if the writer has not been initialized. |
250 | */ |
251 | @Override |
252 | public void write(List<? extends T> items) throws Exception { |
253 | |
254 | if (!getOutputState().isInitialized()) { |
255 | throw new WriterNotOpenException("Writer must be open before it can be written to"); |
256 | } |
257 | |
258 | if (logger.isDebugEnabled()) { |
259 | logger.debug("Writing to flat file with " + items.size() + " items."); |
260 | } |
261 | |
262 | OutputState state = getOutputState(); |
263 | |
264 | StringBuilder lines = new StringBuilder(); |
265 | int lineCount = 0; |
266 | for (T item : items) { |
267 | lines.append(lineAggregator.aggregate(item) + lineSeparator); |
268 | lineCount++; |
269 | } |
270 | try { |
271 | state.write(lines.toString()); |
272 | } |
273 | catch (IOException e) { |
274 | throw new WriteFailedException("Could not write data. The file may be corrupt.", e); |
275 | } |
276 | state.linesWritten += lineCount; |
277 | } |
278 | |
279 | /** |
280 | * @see ItemStream#close() |
281 | */ |
282 | @Override |
283 | public void close() { |
284 | super.close(); |
285 | if (state != null) { |
286 | try { |
287 | if (footerCallback != null && state.outputBufferedWriter != null) { |
288 | footerCallback.writeFooter(state.outputBufferedWriter); |
289 | state.outputBufferedWriter.flush(); |
290 | } |
291 | } |
292 | catch (IOException e) { |
293 | throw new ItemStreamException("Failed to write footer before closing", e); |
294 | } |
295 | finally { |
296 | state.close(); |
297 | if (state.linesWritten == 0 && shouldDeleteIfEmpty) { |
298 | try { |
299 | resource.getFile().delete(); |
300 | } |
301 | catch (IOException e) { |
302 | throw new ItemStreamException("Failed to delete empty file on close", e); |
303 | } |
304 | } |
305 | state = null; |
306 | } |
307 | } |
308 | } |
309 | |
310 | /** |
311 | * Initialize the reader. This method may be called multiple times before |
312 | * close is called. |
313 | * |
314 | * @see ItemStream#open(ExecutionContext) |
315 | */ |
316 | @Override |
317 | public void open(ExecutionContext executionContext) throws ItemStreamException { |
318 | super.open(executionContext); |
319 | |
320 | Assert.notNull(resource, "The resource must be set"); |
321 | |
322 | if (!getOutputState().isInitialized()) { |
323 | doOpen(executionContext); |
324 | } |
325 | } |
326 | |
327 | private void doOpen(ExecutionContext executionContext) throws ItemStreamException { |
328 | OutputState outputState = getOutputState(); |
329 | if (executionContext.containsKey(getExecutionContextKey(RESTART_DATA_NAME))) { |
330 | outputState.restoreFrom(executionContext); |
331 | } |
332 | try { |
333 | outputState.initializeBufferedWriter(); |
334 | } |
335 | catch (IOException ioe) { |
336 | throw new ItemStreamException("Failed to initialize writer", ioe); |
337 | } |
338 | if (outputState.lastMarkedByteOffsetPosition == 0 && !outputState.appending) { |
339 | if (headerCallback != null) { |
340 | try { |
341 | headerCallback.writeHeader(outputState.outputBufferedWriter); |
342 | outputState.write(lineSeparator); |
343 | } |
344 | catch (IOException e) { |
345 | throw new ItemStreamException("Could not write headers. The file may be corrupt.", e); |
346 | } |
347 | } |
348 | } |
349 | } |
350 | |
351 | /** |
352 | * @see ItemStream#update(ExecutionContext) |
353 | */ |
354 | @Override |
355 | public void update(ExecutionContext executionContext) { |
356 | super.update(executionContext); |
357 | if (state == null) { |
358 | throw new ItemStreamException("ItemStream not open or already closed."); |
359 | } |
360 | |
361 | Assert.notNull(executionContext, "ExecutionContext must not be null"); |
362 | |
363 | if (saveState) { |
364 | |
365 | try { |
366 | executionContext.putLong(getExecutionContextKey(RESTART_DATA_NAME), state.position()); |
367 | } |
368 | catch (IOException e) { |
369 | throw new ItemStreamException("ItemStream does not return current position properly", e); |
370 | } |
371 | |
372 | executionContext.putLong(getExecutionContextKey(WRITTEN_STATISTICS_NAME), state.linesWritten); |
373 | } |
374 | } |
375 | |
376 | // Returns object representing state. |
377 | private OutputState getOutputState() { |
378 | if (state == null) { |
379 | File file; |
380 | try { |
381 | file = resource.getFile(); |
382 | } |
383 | catch (IOException e) { |
384 | throw new ItemStreamException("Could not convert resource to file: [" + resource + "]", e); |
385 | } |
386 | Assert.state(!file.exists() || file.canWrite(), "Resource is not writable: [" + resource + "]"); |
387 | state = new OutputState(); |
388 | state.setDeleteIfExists(shouldDeleteIfExists); |
389 | state.setAppendAllowed(append); |
390 | state.setEncoding(encoding); |
391 | } |
392 | return state; |
393 | } |
394 | |
395 | /** |
396 | * Encapsulates the runtime state of the writer. All state changing |
397 | * operations on the writer go through this class. |
398 | */ |
399 | private class OutputState { |
400 | // default encoding for writing to output files - set to UTF-8. |
401 | private static final String DEFAULT_CHARSET = "UTF-8"; |
402 | |
403 | private FileOutputStream os; |
404 | |
405 | // The bufferedWriter over the file channel that is actually written |
406 | Writer outputBufferedWriter; |
407 | |
408 | FileChannel fileChannel; |
409 | |
410 | // this represents the charset encoding (if any is needed) for the |
411 | // output file |
412 | String encoding = DEFAULT_CHARSET; |
413 | |
414 | boolean restarted = false; |
415 | |
416 | long lastMarkedByteOffsetPosition = 0; |
417 | |
418 | long linesWritten = 0; |
419 | |
420 | boolean shouldDeleteIfExists = true; |
421 | |
422 | boolean initialized = false; |
423 | |
424 | private boolean append = false; |
425 | |
426 | private boolean appending = false; |
427 | |
428 | /** |
429 | * Return the byte offset position of the cursor in the output file as a |
430 | * long integer. |
431 | */ |
432 | public long position() throws IOException { |
433 | long pos = 0; |
434 | |
435 | if (fileChannel == null) { |
436 | return 0; |
437 | } |
438 | |
439 | outputBufferedWriter.flush(); |
440 | pos = fileChannel.position(); |
441 | if (transactional) { |
442 | pos += ((TransactionAwareBufferedWriter) outputBufferedWriter).getBufferSize(); |
443 | } |
444 | |
445 | return pos; |
446 | |
447 | } |
448 | |
449 | /** |
450 | * @param append |
451 | */ |
452 | public void setAppendAllowed(boolean append) { |
453 | this.append = append; |
454 | } |
455 | |
456 | /** |
457 | * @param executionContext |
458 | */ |
459 | public void restoreFrom(ExecutionContext executionContext) { |
460 | lastMarkedByteOffsetPosition = executionContext.getLong(getExecutionContextKey(RESTART_DATA_NAME)); |
461 | linesWritten = executionContext.getLong(getExecutionContextKey(WRITTEN_STATISTICS_NAME)); |
462 | if (shouldDeleteIfEmpty && linesWritten == 0) { |
463 | // previous execution deleted the output file because no items were written |
464 | restarted = false; |
465 | lastMarkedByteOffsetPosition = 0; |
466 | } else { |
467 | restarted = true; |
468 | } |
469 | } |
470 | |
471 | /** |
472 | * @param shouldDeleteIfExists |
473 | */ |
474 | public void setDeleteIfExists(boolean shouldDeleteIfExists) { |
475 | this.shouldDeleteIfExists = shouldDeleteIfExists; |
476 | } |
477 | |
478 | /** |
479 | * @param encoding |
480 | */ |
481 | public void setEncoding(String encoding) { |
482 | this.encoding = encoding; |
483 | } |
484 | |
485 | /** |
486 | * Close the open resource and reset counters. |
487 | */ |
488 | public void close() { |
489 | |
490 | initialized = false; |
491 | restarted = false; |
492 | try { |
493 | if (outputBufferedWriter != null) { |
494 | outputBufferedWriter.close(); |
495 | } |
496 | } |
497 | catch (IOException ioe) { |
498 | throw new ItemStreamException("Unable to close the the ItemWriter", ioe); |
499 | } |
500 | finally { |
501 | if (!transactional) { |
502 | closeStream(); |
503 | } |
504 | } |
505 | } |
506 | |
507 | private void closeStream() { |
508 | try { |
509 | if (fileChannel != null) { |
510 | fileChannel.close(); |
511 | } |
512 | } |
513 | catch (IOException ioe) { |
514 | throw new ItemStreamException("Unable to close the the ItemWriter", ioe); |
515 | } |
516 | finally { |
517 | try { |
518 | if (os != null) { |
519 | os.close(); |
520 | } |
521 | } |
522 | catch (IOException ioe) { |
523 | throw new ItemStreamException("Unable to close the the ItemWriter", ioe); |
524 | } |
525 | } |
526 | } |
527 | |
528 | /** |
529 | * @param line |
530 | * @throws IOException |
531 | */ |
532 | public void write(String line) throws IOException { |
533 | if (!initialized) { |
534 | initializeBufferedWriter(); |
535 | } |
536 | |
537 | outputBufferedWriter.write(line); |
538 | outputBufferedWriter.flush(); |
539 | } |
540 | |
541 | /** |
542 | * Truncate the output at the last known good point. |
543 | * |
544 | * @throws IOException |
545 | */ |
546 | public void truncate() throws IOException { |
547 | fileChannel.truncate(lastMarkedByteOffsetPosition); |
548 | fileChannel.position(lastMarkedByteOffsetPosition); |
549 | } |
550 | |
551 | /** |
552 | * Creates the buffered writer for the output file channel based on |
553 | * configuration information. |
554 | * @throws IOException |
555 | */ |
556 | private void initializeBufferedWriter() throws IOException { |
557 | |
558 | File file = resource.getFile(); |
559 | FileUtils.setUpOutputFile(file, restarted, append, shouldDeleteIfExists); |
560 | |
561 | os = new FileOutputStream(file.getAbsolutePath(), true); |
562 | fileChannel = os.getChannel(); |
563 | |
564 | outputBufferedWriter = getBufferedWriter(fileChannel, encoding); |
565 | outputBufferedWriter.flush(); |
566 | |
567 | if (append) { |
568 | // Bug in IO library? This doesn't work... |
569 | // lastMarkedByteOffsetPosition = fileChannel.position(); |
570 | if (file.length() > 0) { |
571 | appending = true; |
572 | // Don't write the headers again |
573 | } |
574 | } |
575 | |
576 | Assert.state(outputBufferedWriter != null); |
577 | // in case of restarting reset position to last committed point |
578 | if (restarted) { |
579 | checkFileSize(); |
580 | truncate(); |
581 | } |
582 | |
583 | initialized = true; |
584 | } |
585 | |
586 | public boolean isInitialized() { |
587 | return initialized; |
588 | } |
589 | |
590 | /** |
591 | * Returns the buffered writer opened to the beginning of the file |
592 | * specified by the absolute path name contained in absoluteFileName. |
593 | */ |
594 | private Writer getBufferedWriter(FileChannel fileChannel, String encoding) { |
595 | try { |
596 | final FileChannel channel = fileChannel; |
597 | if (transactional) { |
598 | TransactionAwareBufferedWriter writer = new TransactionAwareBufferedWriter(channel, new Runnable() { |
599 | @Override |
600 | public void run() { |
601 | closeStream(); |
602 | } |
603 | }); |
604 | |
605 | writer.setEncoding(encoding); |
606 | writer.setForceSync(forceSync); |
607 | return writer; |
608 | } |
609 | else { |
610 | Writer writer = new BufferedWriter(Channels.newWriter(fileChannel, encoding)) { |
611 | @Override |
612 | public void flush() throws IOException { |
613 | super.flush(); |
614 | if (forceSync) { |
615 | channel.force(false); |
616 | } |
617 | } |
618 | }; |
619 | |
620 | return writer; |
621 | } |
622 | } |
623 | catch (UnsupportedCharsetException ucse) { |
624 | throw new ItemStreamException("Bad encoding configuration for output file " + fileChannel, ucse); |
625 | } |
626 | } |
627 | |
628 | /** |
629 | * Checks (on setState) to make sure that the current output file's size |
630 | * is not smaller than the last saved commit point. If it is, then the |
631 | * file has been damaged in some way and whole task must be started over |
632 | * again from the beginning. |
633 | * @throws IOException if there is an IO problem |
634 | */ |
635 | private void checkFileSize() throws IOException { |
636 | long size = -1; |
637 | |
638 | outputBufferedWriter.flush(); |
639 | size = fileChannel.size(); |
640 | |
641 | if (size < lastMarkedByteOffsetPosition) { |
642 | throw new ItemStreamException("Current file size is smaller than size at last commit"); |
643 | } |
644 | } |
645 | |
646 | } |
647 | |
648 | } |