Programming Cookbook Blog

In this post of Spring Batch, I will explain how to read from multiple files.

We take the help of org.springframework.batch.item.file.MultiResourceItemReader, it reads from multiple file.

The MultiResourceItemReader takes two arguments
1) list of resources or pattern followed by the multiple source files, from which the data has to be read
2) Reference to reader bean which will do the actual reading of files

Below is the xml configuration

1   <bean id="employee" class="package8.Employee" scope="prototype"/>
2   
3   <bean id="reader" class="org.springframework.batch.item.file.FlatFileItemReader">
4       <property name="lineMapper">
5           <bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
6               <property name="lineTokenizer">
7                   <bean class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
8                       <property name="names" value="id,name,status,salary"/>
9                   </bean>
10              </property>
11              <property name="fieldSetMapper">
12                  <bean class="org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper">
13                      <property name="prototypeBeanName" value="employee"/>
14                  </bean>
15              </property>
16          </bean>
17      </property> 
18  </bean>
19  
20  <bean id="multiFileReader" class="org.springframework.batch.item.file.MultiResourceItemReader">
21      <property name="resources" value="file:MultiFileDemoData*.txt"/>
22      <property name="delegate" ref="reader"/>
23  </bean>

At line 20 to 23, we create a bean of type MultiResourceItemReader and provide a reference to reader and file name pattern “MultiFileDemoData*.txt”.

Files matching the pattern “MultiFileDemoData*.txt” (like MultiFileDemoData1.txt and MultiFileDemoData2.txt etc) are read one after another in sequence by reader bean.

The reader bean is defined from line 3 to 18.

Now we integrate into the job as shown in the below xml configuration

1   <batch:job id="importProductsJob">
2       <batch:step id="readWriteProducts">
3           <batch:tasklet>
4               <batch:chunk reader="multiFileReader" writer="writer" commit-interval="50"/>
5           </batch:tasklet>
6       </batch:step>
7   </batch:job>

At line 4, we set the reader property to multiFileReader bean.

Below is the complete xml configuration

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns:batch="http://www.springframework.org/schema/batch"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
                        http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch.xsd">
    
    <bean id="employee" class="package8.Employee" scope="prototype"/>
    
    <bean id="reader" class="org.springframework.batch.item.file.FlatFileItemReader">
        <property name="lineMapper">
            <bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
                <property name="lineTokenizer">
                    <bean class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
                        <property name="names" value="id,name,status,salary"/>
                    </bean>
                </property>
                <property name="fieldSetMapper">
                    <bean class="org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper">
                        <property name="prototypeBeanName" value="employee"/>
                    </bean>
                </property>
            </bean>
        </property> 
    </bean>
    
    <bean id="multiFileReader" class="org.springframework.batch.item.file.MultiResourceItemReader">
        <property name="resources" value="file:MultiFileDemoData*.txt"/>
        <property name="delegate" ref="reader"/>
    </bean>
    
    <bean id="writer" class="org.springframework.batch.item.file.FlatFileItemWriter">
        <property name="resource" value="file:DemoData1.txt"/>
        <property name="lineAggregator">
            <bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator">
                <property name="fieldExtractor">
                    <bean class="org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor">
                        <property name="names" value="id,name,status,salary"/>
                    </bean>
                </property>
            </bean>
        </property>
    </bean>
    
    <batch:job id="importProductsJob">
        <batch:step id="readWriteProducts">
            <batch:tasklet>
                <batch:chunk reader="multiFileReader" writer="writer" commit-interval="50"/>
            </batch:tasklet>
        </batch:step>
    </batch:job>
    
    <bean id="transactionManager" class="org.springframework.batch.support.transaction.ResourcelessTransactionManager" />
    
    <bean id="jobRepository" class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
        <property name="transactionManager" ref="transactionManager"/>
    </bean>
    
    <bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
        <property name="jobRepository" ref="jobRepository"/>
    </bean>
</beans>