Spring Batch输出数据

69.spring-batch-itemwriter/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.2.5.RELEASE</version>
+        <relativePath/> <!-- lookup parent from repository -->
+    </parent>
+    <groupId>cc.mrbird</groupId>
+    <artifactId>spring-batch-itemwriter</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>spring-batch-itemwriter</name>
+    <description>Demo project for Spring Boot</description>
+
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-batch</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-jdbc</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-oxm</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.thoughtworks.xstream</groupId>
+            <artifactId>xstream</artifactId>
+            <version>1.4.11.1</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/SpringBatchItemwriterApplication.java

@@ -0,0 +1,15 @@
+package cc.mrbird.batch;
+
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+@EnableBatchProcessing
+public class SpringBatchItemwriterApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(SpringBatchItemwriterApplication.class, args);
+    }
+
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/entity/TestData.java

@@ -0,0 +1,54 @@
+package cc.mrbird.batch.entity;
+
+/**
+ * @author MrBird
+ */
+public class TestData {
+
+    private int id;
+    private String field1;
+    private String field2;
+    private String field3;
+
+    public int getId() {
+        return id;
+    }
+
+    public void setId(int id) {
+        this.id = id;
+    }
+
+    public String getField1() {
+        return field1;
+    }
+
+    public void setField1(String field1) {
+        this.field1 = field1;
+    }
+
+    public String getField2() {
+        return field2;
+    }
+
+    public void setField2(String field2) {
+        this.field2 = field2;
+    }
+
+    public String getField3() {
+        return field3;
+    }
+
+    public void setField3(String field3) {
+        this.field3 = field3;
+    }
+
+    @Override
+    public String toString() {
+        return "TestData{" +
+                "id=" + id +
+                ", field1='" + field1 + '\'' +
+                ", field2='" + field2 + '\'' +
+                ", field3='" + field3 + '\'' +
+                '}';
+    }
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/DatabaseItemWriterDemo.java

@@ -0,0 +1,64 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemWriter;
+import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
+import org.springframework.batch.item.database.JdbcBatchItemWriter;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import javax.sql.DataSource;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class DatabaseItemWriterDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+    @Autowired
+    private DataSource dataSource;
+
+    @Bean
+    public Job datasourceItemWriterJob() {
+        return jobBuilderFactory.get("datasourceItemWriterJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .writer(dataSourceItemWriter())
+                .build();
+    }
+
+    private ItemWriter<TestData> dataSourceItemWriter() {
+        // ItemWriter的实现类之一，mysql数据库数据写入使用JdbcBatchItemWriter，
+        // 其他实现：MongoItemWriter,Neo4jItemWriter等
+        JdbcBatchItemWriter<TestData> writer = new JdbcBatchItemWriter<>();
+        writer.setDataSource(dataSource); // 设置数据源
+
+        String sql = "insert into TEST(id,field1,field2,field3) values (:id,:field1,:field2,:field3)";
+        writer.setSql(sql); // 设置插入sql脚本
+
+        // 映射TestData对象属性到占位符中的属性
+        BeanPropertyItemSqlParameterSourceProvider<TestData> provider = new BeanPropertyItemSqlParameterSourceProvider<>();
+        writer.setItemSqlParameterSourceProvider(provider);
+
+        writer.afterPropertiesSet(); // 设置一些额外属性
+        return writer;
+    }
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/FileItemWriterDemo.java

@@ -0,0 +1,75 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.file.FlatFileItemWriter;
+import org.springframework.batch.item.file.transform.LineAggregator;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.stereotype.Component;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class FileItemWriterDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+
+    @Bean
+    public Job fileItemWriterJob() throws Exception {
+        return jobBuilderFactory.get("fileItemWriterJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() throws Exception {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .writer(fileItemWriter())
+                .build();
+    }
+
+    private FlatFileItemWriter<TestData> fileItemWriter() throws Exception {
+        FlatFileItemWriter<TestData> writer = new FlatFileItemWriter<>();
+
+        FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file");
+        Path path = Paths.get(file.getPath());
+        if (!Files.exists(path)) {
+            Files.createFile(path);
+        }
+        writer.setResource(file); // 设置目标文件路径
+
+        // 把读到的每个TestData对象转换为JSON字符串
+        LineAggregator<TestData> aggregator = item -> {
+            try {
+                ObjectMapper mapper = new ObjectMapper();
+                return mapper.writeValueAsString(item);
+            } catch (JsonProcessingException e) {
+                e.printStackTrace();
+            }
+            return "";
+        };
+
+        writer.setLineAggregator(aggregator);
+        writer.afterPropertiesSet();
+        return writer;
+    }
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/JsonFileItemWriterDemo.java

@@ -0,0 +1,63 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.json.JacksonJsonObjectMarshaller;
+import org.springframework.batch.item.json.JsonFileItemWriter;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class JsonFileItemWriterDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+
+    @Bean
+    public Job jsonFileItemWriterJob() throws Exception {
+        return jobBuilderFactory.get("jsonFileItemWriterJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() throws Exception {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .writer(jsonFileItemWriter())
+                .build();
+    }
+
+    private JsonFileItemWriter<TestData> jsonFileItemWriter() throws IOException {
+        // 文件输出目标地址
+        FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.json");
+        Path path = Paths.get(file.getPath());
+        if (!Files.exists(path)) {
+            Files.createFile(path);
+        }
+        // 将对象转换为json
+        JacksonJsonObjectMarshaller<TestData> marshaller = new JacksonJsonObjectMarshaller<>();
+        JsonFileItemWriter<TestData> writer = new JsonFileItemWriter<>(file, marshaller);
+        // 设置别名
+        writer.setName("testDatasonFileItemWriter");
+        return writer;
+    }
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/MultiFileItemWriteDemo.java

@@ -0,0 +1,79 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemStreamWriter;
+import org.springframework.batch.item.ItemWriter;
+import org.springframework.batch.item.support.ClassifierCompositeItemWriter;
+import org.springframework.batch.item.support.CompositeItemWriter;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.classify.Classifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import javax.sql.DataSource;
+import java.util.Arrays;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class MultiFileItemWriteDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+    @Autowired
+    private ItemStreamWriter<TestData> fileItemWriter;
+    @Autowired
+    private ItemStreamWriter<TestData> xmlFileItemWriter;
+
+    @Bean
+    public Job multiFileItemWriterJob() {
+        return jobBuilderFactory.get("multiFileItemWriterJob6")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .writer(multiFileItemWriter())
+                // .stream(fileItemWriter)
+                // .stream(xmlFileItemWriter)
+                .build();
+    }
+
+    // 输出数据到多个文件
+    private CompositeItemWriter<TestData> multiFileItemWriter() {
+        // 使用CompositeItemWriter代理
+        CompositeItemWriter<TestData> writer = new CompositeItemWriter<>();
+        // 设置具体写代理
+        writer.setDelegates(Arrays.asList(fileItemWriter, xmlFileItemWriter));
+        return writer;
+    }
+
+    // 将数据分类，然后分别输出到对应的文件(此时需要将writer注册到ioc容器，否则报
+    // WriterNotOpenException: Writer must be open before it can be written to)
+    private ClassifierCompositeItemWriter<TestData> classifierMultiFileItemWriter() {
+        ClassifierCompositeItemWriter<TestData> writer = new ClassifierCompositeItemWriter<>();
+        writer.setClassifier((Classifier<TestData, ItemWriter<? super TestData>>) testData -> {
+            try {
+                // id能被2整除则输出到普通文本，否则输出到xml文本
+                return testData.getId() % 2 == 0 ? fileItemWriter : xmlFileItemWriter;
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            return null;
+        });
+        return writer;
+    }
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/job/XmlFileItemWriterDemo.java

@@ -0,0 +1,74 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.batch.item.xml.StaxEventItemWriter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.oxm.xstream.XStreamMarshaller;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class XmlFileItemWriterDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    @Autowired
+    private ListItemReader<TestData> simpleReader;
+
+    @Bean
+    public Job xmlFileItemWriterJob() throws Exception {
+        return jobBuilderFactory.get("xmlFileItemWriterJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() throws Exception {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(simpleReader)
+                .writer(xmlFileItemWriter())
+                .build();
+    }
+
+    private StaxEventItemWriter<TestData> xmlFileItemWriter() throws IOException {
+        StaxEventItemWriter<TestData> writer = new StaxEventItemWriter<>();
+
+        // 通过XStreamMarshaller将TestData转换为xml
+        XStreamMarshaller marshaller = new XStreamMarshaller();
+
+        Map<String,Class<TestData>> map = new HashMap<>(1);
+        map.put("test", TestData.class);
+
+        marshaller.setAliases(map); // 设置xml标签
+
+        writer.setRootTagName("tests"); // 设置根标签
+        writer.setMarshaller(marshaller);
+
+        FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.xml");
+        Path path = Paths.get(file.getPath());
+        if (!Files.exists(path)) {
+            Files.createFile(path);
+        }
+
+        writer.setResource(file); // 设置目标文件路径
+        return writer;
+    }
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/reader/ItemReaderConfigure.java

@@ -0,0 +1,34 @@
+package cc.mrbird.batch.reader;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.item.support.ListItemReader;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author MrBird
+ */
+@Configuration
+public class ItemReaderConfigure {
+
+    @Bean
+    public ListItemReader<TestData> simpleReader() {
+        List<TestData> data = new ArrayList<>();
+        TestData testData1 = new TestData();
+        testData1.setId(1);
+        testData1.setField1("11");
+        testData1.setField2("12");
+        testData1.setField3("13");
+        data.add(testData1);
+        TestData testData2 = new TestData();
+        testData2.setId(2);
+        testData2.setField1("21");
+        testData2.setField2("22");
+        testData2.setField3("23");
+        data.add(testData2);
+        return new ListItemReader<>(data);
+    }
+}

69.spring-batch-itemwriter/src/main/java/cc/mrbird/batch/writer/ItemWriterConfigure.java

@@ -0,0 +1,78 @@
+package cc.mrbird.batch.writer;
+
+import cc.mrbird.batch.entity.TestData;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.batch.item.file.FlatFileItemWriter;
+import org.springframework.batch.item.file.transform.LineAggregator;
+import org.springframework.batch.item.xml.StaxEventItemWriter;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.oxm.xstream.XStreamMarshaller;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author MrBird
+ */
+@Configuration
+public class ItemWriterConfigure {
+
+    @Bean
+    public FlatFileItemWriter<TestData> fileItemWriter() throws Exception {
+        FlatFileItemWriter<TestData> writer = new FlatFileItemWriter<>();
+
+        FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file");
+        Path path = Paths.get(file.getPath());
+        if (!Files.exists(path)) {
+            Files.createFile(path);
+        }
+
+        writer.setResource(file); // 设置目标文件路径
+
+        // 把读到的每个TestData对象转换为字符串
+        LineAggregator<TestData> aggregator = item -> {
+            try {
+                ObjectMapper mapper = new ObjectMapper();
+                return mapper.writeValueAsString(item);
+            } catch (JsonProcessingException e) {
+                e.printStackTrace();
+            }
+            return "";
+        };
+
+        writer.setLineAggregator(aggregator);
+        writer.afterPropertiesSet();
+        return writer;
+    }
+
+    @Bean
+    public StaxEventItemWriter<TestData> xmlFileItemWriter() throws Exception {
+        StaxEventItemWriter<TestData> writer = new StaxEventItemWriter<>();
+
+        // 通过XStreamMarshaller将TestData转换为xml
+        XStreamMarshaller marshaller = new XStreamMarshaller();
+
+        Map<String, Class<TestData>> map = new HashMap<>(1);
+        map.put("test", TestData.class);
+
+        marshaller.setAliases(map); // 设置xml标签
+
+        writer.setRootTagName("tests"); // 设置根标签
+        writer.setMarshaller(marshaller);
+
+        FileSystemResource file = new FileSystemResource("/Users/mrbird/Desktop/file.xml");
+        Path path = Paths.get(file.getPath());
+        if (!Files.exists(path)) {
+            Files.createFile(path);
+        }
+
+        writer.setResource(file); // 设置目标文件路径
+        return writer;
+    }
+}

69.spring-batch-itemwriter/src/main/resources/application.yml

@@ -0,0 +1,6 @@
+spring:
+  datasource:
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/springbatch
+    username: root
+    password: 123456