From c54c5ac847c61ecfd5a62bd9afa7a585a8412505 Mon Sep 17 00:00:00 2001 From: MrBird <852252810@qq.com> Date: Thu, 12 Mar 2020 11:24:11 +0800 Subject: [PATCH] =?UTF-8?q?Spring=20Batch=E8=BE=93=E5=87=BA=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 69.spring-batch-itemwriter/pom.xml | 56 +++++++++++++ .../SpringBatchItemwriterApplication.java | 15 ++++ .../java/cc/mrbird/batch/entity/TestData.java | 54 +++++++++++++ .../batch/job/DatabaseItemWriterDemo.java | 64 +++++++++++++++ .../mrbird/batch/job/FileItemWriterDemo.java | 75 ++++++++++++++++++ .../batch/job/JsonFileItemWriterDemo.java | 63 +++++++++++++++ .../batch/job/MultiFileItemWriteDemo.java | 79 +++++++++++++++++++ .../batch/job/XmlFileItemWriterDemo.java | 74 +++++++++++++++++ .../batch/reader/ItemReaderConfigure.java | 34 ++++++++ .../batch/writer/ItemWriterConfigure.java | 78 ++++++++++++++++++ .../src/main/resources/application.yml | 6 ++ 11 files changed, 598 insertions(+) create mode 100644 69.spring-batch-itemwriter/pom.xml create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/SpringBatchItemwriterApplication.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/entity/TestData.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/DatabaseItemWriterDemo.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/FileItemWriterDemo.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/JsonFileItemWriterDemo.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/MultiFileItemWriteDemo.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/XmlFileItemWriterDemo.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/reader/ItemReaderConfigure.java create mode 100644 69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/writer/ItemWriterConfigure.java create mode 100644 69.spring-batch-itemwriter/src/main/resources/application.yml diff --git a/69.spring-batch-itemwriter/pom.xml b/69.spring-batch-itemwriter/pom.xml new file mode 100644 index 0000000..620723c --- /dev/null +++ b/69.spring-batch-itemwriter/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.2.5.RELEASE + + + cc.mrbird + spring-batch-itemwriter + 0.0.1-SNAPSHOT + spring-batch-itemwriter + Demo project for Spring Boot + + + 1.8 + + + + + org.springframework.boot + spring-boot-starter-batch + + + + mysql + mysql-connector-java + + + org.springframework.boot + spring-boot-starter-jdbc + + + + org.springframework + spring-oxm + + + com.thoughtworks.xstream + xstream + 1.4.11.1 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/SpringBatchItemwriterApplication.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/SpringBatchItemwriterApplication.java new file mode 100644 index 0000000..950938b --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/SpringBatchItemwriterApplication.java @@ -0,0 +1,15 @@ +package cc.mrbird.batch; + +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +@EnableBatchProcessing +public class SpringBatchItemwriterApplication { + + public static void main(String[] args) { + SpringApplication.run(SpringBatchItemwriterApplication.class, args); + } + +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/entity/TestData.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/entity/TestData.java new file mode 100644 index 0000000..1e21271 --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/entity/TestData.java @@ -0,0 +1,54 @@ +package cc.mrbird.batch.entity; + +/** + * @author MrBird + */ +public class TestData { + + private int id; + private String field1; + private String field2; + private String field3; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getField1() { + return field1; + } + + public void setField1(String field1) { + this.field1 = field1; + } + + public String getField2() { + return field2; + } + + public void setField2(String field2) { + this.field2 = field2; + } + + public String getField3() { + return field3; + } + + public void setField3(String field3) { + this.field3 = field3; + } + + @Override + public String toString() { + return "TestData{" + + "id=" + id + + ", field1='" + field1 + '\'' + + ", field2='" + field2 + '\'' + + ", field3='" + field3 + '\'' + + '}'; + } +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/DatabaseItemWriterDemo.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/DatabaseItemWriterDemo.java new file mode 100644 index 0000000..999d0bc --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/DatabaseItemWriterDemo.java @@ -0,0 +1,64 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider; +import org.springframework.batch.item.database.JdbcBatchItemWriter; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import javax.sql.DataSource; + +/** + * @author MrBird + */ +@Component +public class DatabaseItemWriterDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + @Autowired + private ListItemReader simpleReader; + @Autowired + private DataSource dataSource; + + @Bean + public Job datasourceItemWriterJob() { + return jobBuilderFactory.get("datasourceItemWriterJob") + .start(step()) + .build(); + } + + private Step step() { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(simpleReader) + .writer(dataSourceItemWriter()) + .build(); + } + + private ItemWriter dataSourceItemWriter() { + // ItemWriter的实现类之一，mysql数据库数据写入使用JdbcBatchItemWriter， + // 其他实现：MongoItemWriter,Neo4jItemWriter等 + JdbcBatchItemWriter writer = new JdbcBatchItemWriter<>(); + writer.setDataSource(dataSource); // 设置数据源 + + String sql = "insert into TEST(id,field1,field2,field3) values (:id,:field1,:field2,:field3)"; + writer.setSql(sql); // 设置插入sql脚本 + + // 映射TestData对象属性到占位符中的属性 + BeanPropertyItemSqlParameterSourceProvider provider = new BeanPropertyItemSqlParameterSourceProvider<>(); + writer.setItemSqlParameterSourceProvider(provider); + + writer.afterPropertiesSet(); // 设置一些额外属性 + return writer; + } +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/FileItemWriterDemo.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/FileItemWriterDemo.java new file mode 100644 index 0000000..27bba07 --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/FileItemWriterDemo.java @@ -0,0 +1,75 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.file.FlatFileItemWriter; +import org.springframework.batch.item.file.transform.LineAggregator; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.FileSystemResource; +import org.springframework.stereotype.Component; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * @author MrBird + */ +@Component +public class FileItemWriterDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + @Autowired + private ListItemReader simpleReader; + + @Bean + public Job fileItemWriterJob() throws Exception { + return jobBuilderFactory.get("fileItemWriterJob") + .start(step()) + .build(); + } + + private Step step() throws Exception { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(simpleReader) + .writer(fileItemWriter()) + .build(); + } + + private FlatFileItemWriter fileItemWriter() throws Exception { + FlatFileItemWriter writer = new FlatFileItemWriter<>(); + + FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file"); + Path path = Paths.get(file.getPath()); + if (!Files.exists(path)) { + Files.createFile(path); + } + writer.setResource(file); // 设置目标文件路径 + + // 把读到的每个TestData对象转换为JSON字符串 + LineAggregator aggregator = item -> { + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(item); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + return ""; + }; + + writer.setLineAggregator(aggregator); + writer.afterPropertiesSet(); + return writer; + } +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/JsonFileItemWriterDemo.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/JsonFileItemWriterDemo.java new file mode 100644 index 0000000..b37be2b --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/JsonFileItemWriterDemo.java @@ -0,0 +1,63 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.json.JacksonJsonObjectMarshaller; +import org.springframework.batch.item.json.JsonFileItemWriter; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.FileSystemResource; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * @author MrBird + */ +@Component +public class JsonFileItemWriterDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + @Autowired + private ListItemReader simpleReader; + + @Bean + public Job jsonFileItemWriterJob() throws Exception { + return jobBuilderFactory.get("jsonFileItemWriterJob") + .start(step()) + .build(); + } + + private Step step() throws Exception { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(simpleReader) + .writer(jsonFileItemWriter()) + .build(); + } + + private JsonFileItemWriter jsonFileItemWriter() throws IOException { + // 文件输出目标地址 + FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.json"); + Path path = Paths.get(file.getPath()); + if (!Files.exists(path)) { + Files.createFile(path); + } + // 将对象转换为json + JacksonJsonObjectMarshaller marshaller = new JacksonJsonObjectMarshaller<>(); + JsonFileItemWriter writer = new JsonFileItemWriter<>(file, marshaller); + // 设置别名 + writer.setName("testDatasonFileItemWriter"); + return writer; + } +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/MultiFileItemWriteDemo.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/MultiFileItemWriteDemo.java new file mode 100644 index 0000000..4d511d9 --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/MultiFileItemWriteDemo.java @@ -0,0 +1,79 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemStreamWriter; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.support.ClassifierCompositeItemWriter; +import org.springframework.batch.item.support.CompositeItemWriter; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.classify.Classifier; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import javax.sql.DataSource; +import java.util.Arrays; + +/** + * @author MrBird + */ +@Component +public class MultiFileItemWriteDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + @Autowired + private ListItemReader simpleReader; + @Autowired + private ItemStreamWriter fileItemWriter; + @Autowired + private ItemStreamWriter xmlFileItemWriter; + + @Bean + public Job multiFileItemWriterJob() { + return jobBuilderFactory.get("multiFileItemWriterJob6") + .start(step()) + .build(); + } + + private Step step() { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(simpleReader) + .writer(multiFileItemWriter()) + // .stream(fileItemWriter) + // .stream(xmlFileItemWriter) + .build(); + } + + // 输出数据到多个文件 + private CompositeItemWriter multiFileItemWriter() { + // 使用CompositeItemWriter代理 + CompositeItemWriter writer = new CompositeItemWriter<>(); + // 设置具体写代理 + writer.setDelegates(Arrays.asList(fileItemWriter, xmlFileItemWriter)); + return writer; + } + + // 将数据分类，然后分别输出到对应的文件(此时需要将writer注册到ioc容器，否则报 + // WriterNotOpenException: Writer must be open before it can be written to) + private ClassifierCompositeItemWriter classifierMultiFileItemWriter() { + ClassifierCompositeItemWriter writer = new ClassifierCompositeItemWriter<>(); + writer.setClassifier((Classifier>) testData -> { + try { + // id能被2整除则输出到普通文本，否则输出到xml文本 + return testData.getId() % 2 == 0 ? fileItemWriter : xmlFileItemWriter; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + }); + return writer; + } +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/XmlFileItemWriterDemo.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/XmlFileItemWriterDemo.java new file mode 100644 index 0000000..9648bf4 --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/XmlFileItemWriterDemo.java @@ -0,0 +1,74 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.batch.item.xml.StaxEventItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.FileSystemResource; +import org.springframework.oxm.xstream.XStreamMarshaller; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +/** + * @author MrBird + */ +@Component +public class XmlFileItemWriterDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + @Autowired + private ListItemReader simpleReader; + + @Bean + public Job xmlFileItemWriterJob() throws Exception { + return jobBuilderFactory.get("xmlFileItemWriterJob") + .start(step()) + .build(); + } + + private Step step() throws Exception { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(simpleReader) + .writer(xmlFileItemWriter()) + .build(); + } + + private StaxEventItemWriter xmlFileItemWriter() throws IOException { + StaxEventItemWriter writer = new StaxEventItemWriter<>(); + + // 通过XStreamMarshaller将TestData转换为xml + XStreamMarshaller marshaller = new XStreamMarshaller(); + + Map> map = new HashMap<>(1); + map.put("test", TestData.class); + + marshaller.setAliases(map); // 设置xml标签 + + writer.setRootTagName("tests"); // 设置根标签 + writer.setMarshaller(marshaller); + + FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.xml"); + Path path = Paths.get(file.getPath()); + if (!Files.exists(path)) { + Files.createFile(path); + } + + writer.setResource(file); // 设置目标文件路径 + return writer; + } +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/reader/ItemReaderConfigure.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/reader/ItemReaderConfigure.java new file mode 100644 index 0000000..b8adb0e --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/reader/ItemReaderConfigure.java @@ -0,0 +1,34 @@ +package cc.mrbird.batch.reader; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author MrBird + */ +@Configuration +public class ItemReaderConfigure { + + @Bean + public ListItemReader simpleReader() { + List data = new ArrayList<>(); + TestData testData1 = new TestData(); + testData1.setId(1); + testData1.setField1("11"); + testData1.setField2("12"); + testData1.setField3("13"); + data.add(testData1); + TestData testData2 = new TestData(); + testData2.setId(2); + testData2.setField1("21"); + testData2.setField2("22"); + testData2.setField3("23"); + data.add(testData2); + return new ListItemReader<>(data); + } +} diff --git a/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/writer/ItemWriterConfigure.java b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/writer/ItemWriterConfigure.java new file mode 100644 index 0000000..956249f --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/writer/ItemWriterConfigure.java @@ -0,0 +1,78 @@ +package cc.mrbird.batch.writer; + +import cc.mrbird.batch.entity.TestData; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.batch.item.file.FlatFileItemWriter; +import org.springframework.batch.item.file.transform.LineAggregator; +import org.springframework.batch.item.xml.StaxEventItemWriter; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.FileSystemResource; +import org.springframework.oxm.xstream.XStreamMarshaller; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +/** + * @author MrBird + */ +@Configuration +public class ItemWriterConfigure { + + @Bean + public FlatFileItemWriter fileItemWriter() throws Exception { + FlatFileItemWriter writer = new FlatFileItemWriter<>(); + + FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file"); + Path path = Paths.get(file.getPath()); + if (!Files.exists(path)) { + Files.createFile(path); + } + + writer.setResource(file); // 设置目标文件路径 + + // 把读到的每个TestData对象转换为字符串 + LineAggregator aggregator = item -> { + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(item); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + return ""; + }; + + writer.setLineAggregator(aggregator); + writer.afterPropertiesSet(); + return writer; + } + + @Bean + public StaxEventItemWriter xmlFileItemWriter() throws Exception { + StaxEventItemWriter writer = new StaxEventItemWriter<>(); + + // 通过XStreamMarshaller将TestData转换为xml + XStreamMarshaller marshaller = new XStreamMarshaller(); + + Map> map = new HashMap<>(1); + map.put("test", TestData.class); + + marshaller.setAliases(map); // 设置xml标签 + + writer.setRootTagName("tests"); // 设置根标签 + writer.setMarshaller(marshaller); + + FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.xml"); + Path path = Paths.get(file.getPath()); + if (!Files.exists(path)) { + Files.createFile(path); + } + + writer.setResource(file); // 设置目标文件路径 + return writer; + } +} diff --git a/69.spring-batch-itemwriter/src/main/resources/application.yml b/69.spring-batch-itemwriter/src/main/resources/application.yml new file mode 100644 index 0000000..4c962f7 --- /dev/null +++ b/69.spring-batch-itemwriter/src/main/resources/application.yml @@ -0,0 +1,6 @@ +spring: + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/springbatch + username: root + password: 123456 \ No newline at end of file