Spring Batch处理数据

70.spring-batch-itemprocessor/pom.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.2.5.RELEASE</version>
+        <relativePath/> <!-- lookup parent from repository -->
+    </parent>
+    <groupId>cc.mrbird</groupId>
+    <artifactId>spring-batch-itemprocessor</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>spring-batch-itemprocessor</name>
+    <description>Demo project for Spring Boot</description>
+
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-batch</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-validation</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/SpringBatchItemprocessorApplication.java

@@ -0,0 +1,15 @@
+package cc.mrbird.batch;
+
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+@EnableBatchProcessing
+public class SpringBatchItemprocessorApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(SpringBatchItemprocessorApplication.class, args);
+    }
+
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/TestData.java

@@ -0,0 +1,57 @@
+package cc.mrbird.batch.entity;
+
+import javax.validation.constraints.NotBlank;
+
+/**
+ * @author MrBird
+ */
+public class TestData {
+
+    private int id;
+    private String field1;
+    private String field2;
+    @NotBlank
+    private String field3;
+
+    public int getId() {
+        return id;
+    }
+
+    public void setId(int id) {
+        this.id = id;
+    }
+
+    public String getField1() {
+        return field1;
+    }
+
+    public void setField1(String field1) {
+        this.field1 = field1;
+    }
+
+    public String getField2() {
+        return field2;
+    }
+
+    public void setField2(String field2) {
+        this.field2 = field2;
+    }
+
+    public String getField3() {
+        return field3;
+    }
+
+    public void setField3(String field3) {
+        this.field3 = field3;
+    }
+
+    @Override
+    public String toString() {
+        return "TestData{" +
+                "id=" + id +
+                ", field1='" + field1 + '\'' +
+                ", field2='" + field2 + '\'' +
+                ", field3='" + field3 + '\'' +
+                '}';
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/BeanValidatingItemProcessorDemo.java

@@ -0,0 +1,50 @@
+package cc.mrbird.batch.entity.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.batch.item.validator.BeanValidatingItemProcessor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class BeanValidatingItemProcessorDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+
+    @Bean
+    public Job beanValidatingItemProcessorJob() throws Exception {
+        return jobBuilderFactory.get("beanValidatingItemProcessorJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() throws Exception {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .processor(beanValidatingItemProcessor())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    private BeanValidatingItemProcessor<TestData> beanValidatingItemProcessor() throws Exception {
+        BeanValidatingItemProcessor<TestData> beanValidatingItemProcessor = new BeanValidatingItemProcessor<>();
+        // 开启过滤，不符合规则的数据被过滤掉；
+        beanValidatingItemProcessor.setFilter(true);
+        beanValidatingItemProcessor.afterPropertiesSet();
+        return beanValidatingItemProcessor;
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/CompositeItemProcessorDemo.java

@@ -0,0 +1,61 @@
+package cc.mrbird.batch.entity.job;
+
+import cc.mrbird.batch.entity.TestData;
+import cc.mrbird.batch.processor.TestDataFilterItemProcessor;
+import cc.mrbird.batch.processor.TestDataTransformItemPorcessor;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.batch.item.support.CompositeItemProcessor;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class CompositeItemProcessorDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+    @Autowired
+    private TestDataFilterItemProcessor testDataFilterItemProcessor;
+    @Autowired
+    private TestDataTransformItemPorcessor testDataTransformItemPorcessor;
+
+    @Bean
+    public Job compositeItemProcessorJob() {
+        return jobBuilderFactory.get("compositeItemProcessorJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .processor(compositeItemProcessor())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    // CompositeItemProcessor组合多种中间处理器
+    private CompositeItemProcessor<TestData, TestData> compositeItemProcessor() {
+        CompositeItemProcessor<TestData, TestData> processor = new CompositeItemProcessor<>();
+        List<ItemProcessor<TestData, TestData>> processors = Arrays.asList(testDataFilterItemProcessor, testDataTransformItemPorcessor);
+        // 代理两个processor
+        processor.setDelegates(processors);
+        return processor;
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/TestDataFilterItemProcessorDemo.java

@@ -0,0 +1,44 @@
+package cc.mrbird.batch.entity.job;
+
+import cc.mrbird.batch.entity.TestData;
+import cc.mrbird.batch.processor.TestDataFilterItemProcessor;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class TestDataFilterItemProcessorDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+    @Autowired
+    private TestDataFilterItemProcessor testDataFilterItemProcessor;
+
+    @Bean
+    public Job testDataFilterItemProcessorJob() {
+        return jobBuilderFactory.get("testDataFilterItemProcessorJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .processor(testDataFilterItemProcessor)
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/TestDataTransformItemPorcessorDemo.java

@@ -0,0 +1,45 @@
+package cc.mrbird.batch.entity.job;
+
+import cc.mrbird.batch.entity.TestData;
+import cc.mrbird.batch.processor.TestDataFilterItemProcessor;
+import cc.mrbird.batch.processor.TestDataTransformItemPorcessor;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class TestDataTransformItemPorcessorDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+    @Autowired
+    private TestDataTransformItemPorcessor testDataTransformItemPorcessor;
+
+    @Bean
+    public Job testDataTransformItemPorcessorJob() {
+        return jobBuilderFactory.get("testDataTransformItemPorcessorJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .processor(testDataTransformItemPorcessor)
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/job/ValidatingItemProcessorDemo.java

@@ -0,0 +1,55 @@
+package cc.mrbird.batch.entity.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.batch.item.validator.ValidatingItemProcessor;
+import org.springframework.batch.item.validator.ValidationException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class ValidatingItemProcessorDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+
+    @Bean
+    public Job validatingItemProcessorJob() {
+        return jobBuilderFactory.get("validatingItemProcessorJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .processor(validatingItemProcessor())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    private ValidatingItemProcessor<TestData> validatingItemProcessor() {
+        ValidatingItemProcessor<TestData> processor = new ValidatingItemProcessor<>();
+        processor.setValidator(value -> {
+            // 对每一条数据进行校验
+            if ("".equals(value.getField3())) {
+                // 如果field3的值为空串，则抛异常
+                throw new ValidationException("field3的值不合法");
+            }
+        });
+        return processor;
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/entity/reader/ItemReaderConfigure.java

@@ -0,0 +1,40 @@
+package cc.mrbird.batch.entity.reader;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author MrBird
+ */
+@Configuration
+public class ItemReaderConfigure {
+
+    @Bean
+    public ListItemReader<TestData> simpleReader() {
+        List<TestData> data = new ArrayList<>();
+        TestData testData1 = new TestData();
+        testData1.setId(1);
+        testData1.setField1("11");
+        testData1.setField2("12");
+        testData1.setField3("13");
+        data.add(testData1);
+        TestData testData2 = new TestData();
+        testData2.setId(2);
+        testData2.setField1("21");
+        testData2.setField2("22");
+        testData2.setField3("23");
+        data.add(testData2);
+        TestData testData3 = new TestData();
+        testData3.setId(3);
+        testData3.setField1("31");
+        testData3.setField2("32");
+        testData3.setField3("");
+        data.add(testData3);
+        return new ListItemReader<>(data);
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/processor/TestDataFilterItemProcessor.java

@@ -0,0 +1,17 @@
+package cc.mrbird.batch.processor;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class TestDataFilterItemProcessor implements ItemProcessor<TestData, TestData> {
+    @Override
+    public TestData process(TestData item) {
+        // 返回null，会过滤掉这条数据
+        return "".equals(item.getField3()) ? null : item;
+    }
+}

70.spring-batch-itemprocessor/src/main/java/cc/mrbird/batch/processor/TestDataTransformItemPorcessor.java

@@ -0,0 +1,19 @@
+package cc.mrbird.batch.processor;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.beans.BeanUtils;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class TestDataTransformItemPorcessor implements ItemProcessor<TestData, TestData> {
+    @Override
+    public TestData process(TestData item) {
+        // field1值拼接 hello
+        item.setField1(item.getField1() + " hello");
+        return item;
+    }
+}

70.spring-batch-itemprocessor/src/main/resources/application.yml

@@ -0,0 +1,6 @@
+spring:
+  datasource:
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/springbatch
+    username: root
+    password: 123456