learning
/
SpringAll
mirror of https://github.com/wuyouzhuguli/SpringAll.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Spring Batch输出数据

This commit is contained in:
MrBird 2020-03-12 11:24:11 +08:00
parent 9e41813382
commit c54c5ac847
11 changed files with 598 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
69.spring-batch-itemwriter/pom.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.5.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>cc.mrbird</groupId>
<artifactId>spring-batch-itemwriter</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>spring-batch-itemwriter</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-batch</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-oxm</artifactId>
</dependency>
<dependency>
<groupId>com.thoughtworks.xstream</groupId>
<artifactId>xstream</artifactId>
<version>1.4.11.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

15
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/SpringBatchItemwriterApplication.java Normal file
View File

@ -0,0 +1,15 @@
package cc.mrbird.batch;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
@EnableBatchProcessing
public class SpringBatchItemwriterApplication {
public static void main(String[] args) {
SpringApplication.run(SpringBatchItemwriterApplication.class, args);
}
}

54
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/entity/TestData.java Normal file
View File

@ -0,0 +1,54 @@
package cc.mrbird.batch.entity;
/**
* @author MrBird
*/
public class TestData {
private int id;
private String field1;
private String field2;
private String field3;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getField1() {
return field1;
}
public void setField1(String field1) {
this.field1 = field1;
}
public String getField2() {
return field2;
}
public void setField2(String field2) {
this.field2 = field2;
}
public String getField3() {
return field3;
}
public void setField3(String field3) {
this.field3 = field3;
}
@Override
public String toString() {
return "TestData{" +
"id=" + id +
", field1='" + field1 + '\'' +
", field2='" + field2 + '\'' +
", field3='" + field3 + '\'' +
'}';
}
}

64
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/DatabaseItemWriterDemo.java Normal file
View File

@ -0,0 +1,64 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import javax.sql.DataSource;
/**
* @author MrBird
*/
@Component
public class DatabaseItemWriterDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Autowired
private DataSource dataSource;
@Bean
public Job datasourceItemWriterJob() {
return jobBuilderFactory.get("datasourceItemWriterJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.writer(dataSourceItemWriter())
.build();
}
private ItemWriter<TestData> dataSourceItemWriter() {
// ItemWriter的实现类之一，mysql数据库数据写入使用JdbcBatchItemWriter，
// 其他实现：MongoItemWriter,Neo4jItemWriter等
JdbcBatchItemWriter<TestData> writer = new JdbcBatchItemWriter<>();
writer.setDataSource(dataSource); // 设置数据源
String sql = "insert into TEST(id,field1,field2,field3) values (:id,:field1,:field2,:field3)";
writer.setSql(sql); // 设置插入sql脚本
// 映射TestData对象属性到占位符中的属性
BeanPropertyItemSqlParameterSourceProvider<TestData> provider = new BeanPropertyItemSqlParameterSourceProvider<>();
writer.setItemSqlParameterSourceProvider(provider);
writer.afterPropertiesSet(); // 设置一些额外属性
return writer;
}
}

75
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/FileItemWriterDemo.java Normal file
View File

@ -0,0 +1,75 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.transform.LineAggregator;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.FileSystemResource;
import org.springframework.stereotype.Component;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* @author MrBird
*/
@Component
public class FileItemWriterDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Bean
public Job fileItemWriterJob() throws Exception {
return jobBuilderFactory.get("fileItemWriterJob")
.start(step())
.build();
}
private Step step() throws Exception {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.writer(fileItemWriter())
.build();
}
private FlatFileItemWriter<TestData> fileItemWriter() throws Exception {
FlatFileItemWriter<TestData> writer = new FlatFileItemWriter<>();
FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file");
Path path = Paths.get(file.getPath());
if (!Files.exists(path)) {
Files.createFile(path);
}
writer.setResource(file); // 设置目标文件路径
// 把读到的每个TestData对象转换为JSON字符串
LineAggregator<TestData> aggregator = item -> {
try {
ObjectMapper mapper = new ObjectMapper();
return mapper.writeValueAsString(item);
} catch (JsonProcessingException e) {
e.printStackTrace();
}
return "";
};
writer.setLineAggregator(aggregator);
writer.afterPropertiesSet();
return writer;
}
}

63
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/JsonFileItemWriterDemo.java Normal file
View File

@ -0,0 +1,63 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.json.JacksonJsonObjectMarshaller;
import org.springframework.batch.item.json.JsonFileItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.FileSystemResource;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* @author MrBird
*/
@Component
public class JsonFileItemWriterDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Bean
public Job jsonFileItemWriterJob() throws Exception {
return jobBuilderFactory.get("jsonFileItemWriterJob")
.start(step())
.build();
}
private Step step() throws Exception {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.writer(jsonFileItemWriter())
.build();
}
private JsonFileItemWriter<TestData> jsonFileItemWriter() throws IOException {
// 文件输出目标地址
FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.json");
Path path = Paths.get(file.getPath());
if (!Files.exists(path)) {
Files.createFile(path);
}
// 将对象转换为json
JacksonJsonObjectMarshaller<TestData> marshaller = new JacksonJsonObjectMarshaller<>();
JsonFileItemWriter<TestData> writer = new JsonFileItemWriter<>(file, marshaller);
// 设置别名
writer.setName("testDatasonFileItemWriter");
return writer;
}
}

79
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/MultiFileItemWriteDemo.java Normal file
View File

@ -0,0 +1,79 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemStreamWriter;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.support.ClassifierCompositeItemWriter;
import org.springframework.batch.item.support.CompositeItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.classify.Classifier;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import javax.sql.DataSource;
import java.util.Arrays;
/**
* @author MrBird
*/
@Component
public class MultiFileItemWriteDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Autowired
private ItemStreamWriter<TestData> fileItemWriter;
@Autowired
private ItemStreamWriter<TestData> xmlFileItemWriter;
@Bean
public Job multiFileItemWriterJob() {
return jobBuilderFactory.get("multiFileItemWriterJob6")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.writer(multiFileItemWriter())
// .stream(fileItemWriter)
// .stream(xmlFileItemWriter)
.build();
}
// 输出数据到多个文件
private CompositeItemWriter<TestData> multiFileItemWriter() {
// 使用CompositeItemWriter代理
CompositeItemWriter<TestData> writer = new CompositeItemWriter<>();
// 设置具体写代理
writer.setDelegates(Arrays.asList(fileItemWriter, xmlFileItemWriter));
return writer;
}
// 将数据分类，然后分别输出到对应的文件(此时需要将writer注册到ioc容器，否则报
// WriterNotOpenException: Writer must be open before it can be written to)
private ClassifierCompositeItemWriter<TestData> classifierMultiFileItemWriter() {
ClassifierCompositeItemWriter<TestData> writer = new ClassifierCompositeItemWriter<>();
writer.setClassifier((Classifier<TestData, ItemWriter<? super TestData>>) testData -> {
try {
// id能被2整除则输出到普通文本，否则输出到xml文本
return testData.getId() % 2 == 0 ? fileItemWriter : xmlFileItemWriter;
} catch (Exception e) {
e.printStackTrace();
}
return null;
});
return writer;
}
}

74
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/XmlFileItemWriterDemo.java Normal file
View File

@ -0,0 +1,74 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.batch.item.xml.StaxEventItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.FileSystemResource;
import org.springframework.oxm.xstream.XStreamMarshaller;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
/**
* @author MrBird
*/
@Component
public class XmlFileItemWriterDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Bean
public Job xmlFileItemWriterJob() throws Exception {
return jobBuilderFactory.get("xmlFileItemWriterJob")
.start(step())
.build();
}
private Step step() throws Exception {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.writer(xmlFileItemWriter())
.build();
}
private StaxEventItemWriter<TestData> xmlFileItemWriter() throws IOException {
StaxEventItemWriter<TestData> writer = new StaxEventItemWriter<>();
// 通过XStreamMarshaller将TestData转换为xml
XStreamMarshaller marshaller = new XStreamMarshaller();
Map<String,Class<TestData>> map = new HashMap<>(1);
map.put("test", TestData.class);
marshaller.setAliases(map); // 设置xml标签
writer.setRootTagName("tests"); // 设置根标签
writer.setMarshaller(marshaller);
FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.xml");
Path path = Paths.get(file.getPath());
if (!Files.exists(path)) {
Files.createFile(path);
}
writer.setResource(file); // 设置目标文件路径
return writer;
}
}

34
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/reader/ItemReaderConfigure.java Normal file
View File

@ -0,0 +1,34 @@
package cc.mrbird.batch.reader;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.ArrayList;
import java.util.List;
/**
* @author MrBird
*/
@Configuration
public class ItemReaderConfigure {
@Bean
public ListItemReader<TestData> simpleReader() {
List<TestData> data = new ArrayList<>();
TestData testData1 = new TestData();
testData1.setId(1);
testData1.setField1("11");
testData1.setField2("12");
testData1.setField3("13");
data.add(testData1);
TestData testData2 = new TestData();
testData2.setId(2);
testData2.setField1("21");
testData2.setField2("22");
testData2.setField3("23");
data.add(testData2);
return new ListItemReader<>(data);
}
}

78
69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/writer/ItemWriterConfigure.java Normal file
View File

@ -0,0 +1,78 @@
package cc.mrbird.batch.writer;
import cc.mrbird.batch.entity.TestData;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.transform.LineAggregator;
import org.springframework.batch.item.xml.StaxEventItemWriter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.FileSystemResource;
import org.springframework.oxm.xstream.XStreamMarshaller;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
/**
* @author MrBird
*/
@Configuration
public class ItemWriterConfigure {
@Bean
public FlatFileItemWriter<TestData> fileItemWriter() throws Exception {
FlatFileItemWriter<TestData> writer = new FlatFileItemWriter<>();
FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file");
Path path = Paths.get(file.getPath());
if (!Files.exists(path)) {
Files.createFile(path);
}
writer.setResource(file); // 设置目标文件路径
// 把读到的每个TestData对象转换为字符串
LineAggregator<TestData> aggregator = item -> {
try {
ObjectMapper mapper = new ObjectMapper();
return mapper.writeValueAsString(item);
} catch (JsonProcessingException e) {
e.printStackTrace();
}
return "";
};
writer.setLineAggregator(aggregator);
writer.afterPropertiesSet();
return writer;
}
@Bean
public StaxEventItemWriter<TestData> xmlFileItemWriter() throws Exception {
StaxEventItemWriter<TestData> writer = new StaxEventItemWriter<>();
// 通过XStreamMarshaller将TestData转换为xml
XStreamMarshaller marshaller = new XStreamMarshaller();
Map<String, Class<TestData>> map = new HashMap<>(1);
map.put("test", TestData.class);
marshaller.setAliases(map); // 设置xml标签
writer.setRootTagName("tests"); // 设置根标签
writer.setMarshaller(marshaller);
FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.xml");
Path path = Paths.get(file.getPath());
if (!Files.exists(path)) {
Files.createFile(path);
}
writer.setResource(file); // 设置目标文件路径
return writer;
}
}

6
69.spring-batch-itemwriter/src/main/resources/application.yml Normal file
View File

@ -0,0 +1,6 @@
spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/springbatch
username: root
password: 123456