learning
/
SpringAll
mirror of https://github.com/wuyouzhuguli/SpringAll.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Spring Batch处理数据

This commit is contained in:
MrBird 2020-03-12 11:24:36 +08:00
parent c54c5ac847
commit 93e149d5bd
12 changed files with 458 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
70.spring-batch-itemprocessor/pom.xml Normal file
View File

@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.5.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>cc.mrbird</groupId>
<artifactId>spring-batch-itemprocessor</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>spring-batch-itemprocessor</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-batch</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

15
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/SpringBatchItemprocessorApplication.java Normal file
View File

@ -0,0 +1,15 @@
package cc.mrbird.batch;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
@EnableBatchProcessing
public class SpringBatchItemprocessorApplication {
public static void main(String[] args) {
SpringApplication.run(SpringBatchItemprocessorApplication.class, args);
}
}

57
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/TestData.java Normal file
View File

@ -0,0 +1,57 @@
package cc.mrbird.batch.entity;
import javax.validation.constraints.NotBlank;
/**
* @author MrBird
*/
public class TestData {
private int id;
private String field1;
private String field2;
@NotBlank
private String field3;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getField1() {
return field1;
}
public void setField1(String field1) {
this.field1 = field1;
}
public String getField2() {
return field2;
}
public void setField2(String field2) {
this.field2 = field2;
}
public String getField3() {
return field3;
}
public void setField3(String field3) {
this.field3 = field3;
}
@Override
public String toString() {
return "TestData{" +
"id=" + id +
", field1='" + field1 + '\'' +
", field2='" + field2 + '\'' +
", field3='" + field3 + '\'' +
'}';
}
}

50
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/BeanValidatingItemProcessorDemo.java Normal file
View File

@ -0,0 +1,50 @@
package cc.mrbird.batch.entity.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.batch.item.validator.BeanValidatingItemProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class BeanValidatingItemProcessorDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Bean
public Job beanValidatingItemProcessorJob() throws Exception {
return jobBuilderFactory.get("beanValidatingItemProcessorJob")
.start(step())
.build();
}
private Step step() throws Exception {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.processor(beanValidatingItemProcessor())
.writer(list -> list.forEach(System.out::println))
.build();
}
private BeanValidatingItemProcessor<TestData> beanValidatingItemProcessor() throws Exception {
BeanValidatingItemProcessor<TestData> beanValidatingItemProcessor = new BeanValidatingItemProcessor<>();
// 开启过滤，不符合规则的数据被过滤掉；
beanValidatingItemProcessor.setFilter(true);
beanValidatingItemProcessor.afterPropertiesSet();
return beanValidatingItemProcessor;
}
}

61
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/CompositeItemProcessorDemo.java Normal file
View File

@ -0,0 +1,61 @@
package cc.mrbird.batch.entity.job;
import cc.mrbird.batch.entity.TestData;
import cc.mrbird.batch.processor.TestDataFilterItemProcessor;
import cc.mrbird.batch.processor.TestDataTransformItemPorcessor;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.support.CompositeItemProcessor;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.List;
/**
* @author MrBird
*/
@Component
public class CompositeItemProcessorDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Autowired
private TestDataFilterItemProcessor testDataFilterItemProcessor;
@Autowired
private TestDataTransformItemPorcessor testDataTransformItemPorcessor;
@Bean
public Job compositeItemProcessorJob() {
return jobBuilderFactory.get("compositeItemProcessorJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.processor(compositeItemProcessor())
.writer(list -> list.forEach(System.out::println))
.build();
}
// CompositeItemProcessor组合多种中间处理器
private CompositeItemProcessor<TestData, TestData> compositeItemProcessor() {
CompositeItemProcessor<TestData, TestData> processor = new CompositeItemProcessor<>();
List<ItemProcessor<TestData, TestData>> processors = Arrays.asList(testDataFilterItemProcessor, testDataTransformItemPorcessor);
// 代理两个processor
processor.setDelegates(processors);
return processor;
}
}

44
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/TestDataFilterItemProcessorDemo.java Normal file
View File

@ -0,0 +1,44 @@
package cc.mrbird.batch.entity.job;
import cc.mrbird.batch.entity.TestData;
import cc.mrbird.batch.processor.TestDataFilterItemProcessor;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class TestDataFilterItemProcessorDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Autowired
private TestDataFilterItemProcessor testDataFilterItemProcessor;
@Bean
public Job testDataFilterItemProcessorJob() {
return jobBuilderFactory.get("testDataFilterItemProcessorJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.processor(testDataFilterItemProcessor)
.writer(list -> list.forEach(System.out::println))
.build();
}
}

45
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/TestDataTransformItemPorcessorDemo.java Normal file
View File

@ -0,0 +1,45 @@
package cc.mrbird.batch.entity.job;
import cc.mrbird.batch.entity.TestData;
import cc.mrbird.batch.processor.TestDataFilterItemProcessor;
import cc.mrbird.batch.processor.TestDataTransformItemPorcessor;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class TestDataTransformItemPorcessorDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Autowired
private TestDataTransformItemPorcessor testDataTransformItemPorcessor;
@Bean
public Job testDataTransformItemPorcessorJob() {
return jobBuilderFactory.get("testDataTransformItemPorcessorJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.processor(testDataTransformItemPorcessor)
.writer(list -> list.forEach(System.out::println))
.build();
}
}

55
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/ValidatingItemProcessorDemo.java Normal file
View File

@ -0,0 +1,55 @@
package cc.mrbird.batch.entity.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.batch.item.validator.ValidatingItemProcessor;
import org.springframework.batch.item.validator.ValidationException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class ValidatingItemProcessorDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
private ListItemReader<TestData> simpleReader;
@Bean
public Job validatingItemProcessorJob() {
return jobBuilderFactory.get("validatingItemProcessorJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(simpleReader)
.processor(validatingItemProcessor())
.writer(list -> list.forEach(System.out::println))
.build();
}
private ValidatingItemProcessor<TestData> validatingItemProcessor() {
ValidatingItemProcessor<TestData> processor = new ValidatingItemProcessor<>();
processor.setValidator(value -> {
// 对每一条数据进行校验
if ("".equals(value.getField3())) {
// 如果field3的值为空串，则抛异常
throw new ValidationException("field3的值不合法");
}
});
return processor;
}
}

40
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/reader/ItemReaderConfigure.java Normal file
View File

@ -0,0 +1,40 @@
package cc.mrbird.batch.entity.reader;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.ArrayList;
import java.util.List;
/**
* @author MrBird
*/
@Configuration
public class ItemReaderConfigure {
@Bean
public ListItemReader<TestData> simpleReader() {
List<TestData> data = new ArrayList<>();
TestData testData1 = new TestData();
testData1.setId(1);
testData1.setField1("11");
testData1.setField2("12");
testData1.setField3("13");
data.add(testData1);
TestData testData2 = new TestData();
testData2.setId(2);
testData2.setField1("21");
testData2.setField2("22");
testData2.setField3("23");
data.add(testData2);
TestData testData3 = new TestData();
testData3.setId(3);
testData3.setField1("31");
testData3.setField2("32");
testData3.setField3("");
data.add(testData3);
return new ListItemReader<>(data);
}
}

17
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/processor/TestDataFilterItemProcessor.java Normal file
View File

@ -0,0 +1,17 @@
package cc.mrbird.batch.processor;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class TestDataFilterItemProcessor implements ItemProcessor<TestData, TestData> {
@Override
public TestData process(TestData item) {
// 返回null，会过滤掉这条数据
return "".equals(item.getField3()) ? null : item;
}
}

19
70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/processor/TestDataTransformItemPorcessor.java Normal file
View File

@ -0,0 +1,19 @@
package cc.mrbird.batch.processor;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class TestDataTransformItemPorcessor implements ItemProcessor<TestData, TestData> {
@Override
public TestData process(TestData item) {
// field1值拼接 hello
item.setField1(item.getField1() + " hello");
return item;
}
}

6
70.spring-batch-itemprocessor/src/main/resources/application.yml Normal file
View File

@ -0,0 +1,6 @@
spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/springbatch
username: root
password: 123456