Spring Batch读取数据

68.spring-batch-itemreader/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.2.5.RELEASE</version>
+        <relativePath/> <!-- lookup parent from repository -->
+    </parent>
+    <groupId>cc.mrbird</groupId>
+    <artifactId>spring-batch-itemreader</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>spring-batch-itemreader</name>
+    <description>Demo project for Spring Boot</description>
+
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-batch</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-jdbc</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-oxm</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.thoughtworks.xstream</groupId>
+            <artifactId>xstream</artifactId>
+            <version>1.4.11.1</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/SpringBatchItemreaderApplication.java

@@ -0,0 +1,15 @@
+package cc.mrbird.batch;
+
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+@EnableBatchProcessing
+public class SpringBatchItemreaderApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(SpringBatchItemreaderApplication.class, args);
+    }
+
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/entity/TestData.java

@@ -0,0 +1,53 @@
+package cc.mrbird.batch.entity;
+
+/**
+ * @author MrBird
+ */
+public class TestData {
+    private int id;
+    private String field1;
+    private String field2;
+    private String field3;
+
+    public int getId() {
+        return id;
+    }
+
+    public void setId(int id) {
+        this.id = id;
+    }
+
+    public String getField1() {
+        return field1;
+    }
+
+    public void setField1(String field1) {
+        this.field1 = field1;
+    }
+
+    public String getField2() {
+        return field2;
+    }
+
+    public void setField2(String field2) {
+        this.field2 = field2;
+    }
+
+    public String getField3() {
+        return field3;
+    }
+
+    public void setField3(String field3) {
+        this.field3 = field3;
+    }
+
+    @Override
+    public String toString() {
+        return "TestData{" +
+                "id=" + id +
+                ", field1='" + field1 + '\'' +
+                ", field2='" + field2 + '\'' +
+                ", field3='" + field3 + '\'' +
+                '}';
+    }
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/DataSourceItemReaderDemo.java

@@ -0,0 +1,80 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.database.JdbcPagingItemReader;
+import org.springframework.batch.item.database.Order;
+import org.springframework.batch.item.database.support.MySqlPagingQueryProvider;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import javax.sql.DataSource;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class DataSourceItemReaderDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+    // 注入数据源
+    @Autowired
+    private DataSource dataSource;
+
+    @Bean
+    public Job dataSourceItemReaderJob() throws Exception {
+        return jobBuilderFactory.get("dataSourceItemReaderJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() throws Exception {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(dataSourceItemReader())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    private ItemReader<TestData> dataSourceItemReader() throws Exception {
+        JdbcPagingItemReader<TestData> reader = new JdbcPagingItemReader<>();
+        reader.setDataSource(dataSource); // 设置数据源
+        reader.setFetchSize(5); // 每次取多少条记录
+        reader.setPageSize(5); // 设置每页数据量
+
+        // 指定sql查询语句 select id,field1,field2,field3 from TEST
+        MySqlPagingQueryProvider provider = new MySqlPagingQueryProvider();
+        provider.setSelectClause("id,field1,field2,field3"); //设置查询字段
+        provider.setFromClause("from TEST"); // 设置从哪张表查询
+
+        // 将读取到的数据转换为TestData对象
+        reader.setRowMapper((resultSet, rowNum) -> {
+            TestData data = new TestData();
+            data.setId(resultSet.getInt(1));
+            data.setField1(resultSet.getString(2)); // 读取第一个字段，类型为String
+            data.setField2(resultSet.getString(3));
+            data.setField3(resultSet.getString(4));
+            return data;
+        });
+
+        Map<String, Order> sort = new HashMap<>(1);
+        sort.put("id", Order.ASCENDING);
+        provider.setSortKeys(sort); // 设置排序,通过id 升序
+
+        reader.setQueryProvider(provider);
+
+        // 设置namedParameterJdbcTemplate等属性
+        reader.afterPropertiesSet();
+        return reader;
+    }
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/FileItemReaderDemo.java

@@ -0,0 +1,71 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.file.FlatFileItemReader;
+import org.springframework.batch.item.file.mapping.DefaultLineMapper;
+import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class FileItemReaderDemo {
+    // 任务创建工厂
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    // 步骤创建工厂
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+
+    @Bean
+    public Job fileItemReaderJob() {
+        return jobBuilderFactory.get("fileItemReaderJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(fileItemReader())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    private ItemReader<TestData> fileItemReader() {
+        FlatFileItemReader<TestData> reader = new FlatFileItemReader<>();
+        reader.setResource(new ClassPathResource("file")); // 设置文件资源地址
+        reader.setLinesToSkip(1); // 忽略第一行
+
+        // AbstractLineTokenizer的三个实现类之一，以固定分隔符处理行数据读取,
+        // 使用默认构造器的时候，使用逗号作为分隔符，也可以通过有参构造器来指定分隔符
+        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
+
+        // 设置属性名，类似于表头
+        tokenizer.setNames("id", "field1", "field2", "field3");
+        // 将每行数据转换为TestData对象
+        DefaultLineMapper<TestData> mapper = new DefaultLineMapper<>();
+        mapper.setLineTokenizer(tokenizer);
+        // 设置映射方式
+        mapper.setFieldSetMapper(fieldSet -> {
+            TestData data = new TestData();
+            data.setId(fieldSet.readInt("id"));
+            data.setField1(fieldSet.readString("field1"));
+            data.setField2(fieldSet.readString("field2"));
+            data.setField3(fieldSet.readString("field3"));
+            return data;
+        });
+
+        reader.setLineMapper(mapper);
+        return reader;
+    }
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/JSONFileItemReaderDemo.java

@@ -0,0 +1,52 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.json.JacksonJsonObjectReader;
+import org.springframework.batch.item.json.JsonItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class JSONFileItemReaderDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+
+    @Bean
+    public Job jsonFileItemReaderJob() {
+        return jobBuilderFactory.get("jsonFileItemReaderJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(jsonItemReader())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    private ItemReader<TestData> jsonItemReader() {
+        // 设置json文件地址
+        ClassPathResource resource = new ClassPathResource("file.json");
+        // 设置json文件转换的目标对象类型
+        JacksonJsonObjectReader<TestData> jacksonJsonObjectReader = new JacksonJsonObjectReader<>(TestData.class);
+        JsonItemReader<TestData> reader = new JsonItemReader<>(resource, jacksonJsonObjectReader);
+        // 给reader设置一个别名
+        reader.setName("testDataJsonItemReader");
+        return reader;
+    }
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MultiFileIteamReaderDemo.java

@@ -0,0 +1,86 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.file.FlatFileItemReader;
+import org.springframework.batch.item.file.MultiResourceItemReader;
+import org.springframework.batch.item.file.mapping.DefaultLineMapper;
+import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author MrBird
+ *
+ * 5. 演示多文件读取
+ */
+@Component
+public class MultiFileIteamReaderDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+
+
+    @Bean
+    public Job multiFileItemReaderJob() {
+        return jobBuilderFactory.get("multiFileItemReaderJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(multiFileItemReader())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    private ItemReader<TestData> multiFileItemReader() {
+        MultiResourceItemReader<TestData> reader = new MultiResourceItemReader<>();
+        reader.setDelegate(fileItemReader()); // 设置文件读取代理，方法可以使用前面文件读取中的例子
+
+        Resource[] resources = new Resource[]{
+                new ClassPathResource("file1"),
+                new ClassPathResource("file2")
+        };
+
+        reader.setResources(resources); // 设置多文件源
+        return reader;
+    }
+
+    private FlatFileItemReader<TestData> fileItemReader() {
+        FlatFileItemReader<TestData> reader = new FlatFileItemReader<>();
+        reader.setLinesToSkip(1); // 忽略第一行
+
+        // AbstractLineTokenizer的三个实现类之一，以固定分隔符处理行数据读取,
+        // 使用默认构造器的时候，使用逗号作为分隔符，也可以通过有参构造器来指定分隔符
+        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
+        // 设置属姓名，类似于表头
+        tokenizer.setNames("id", "field1", "field2", "field3");
+        // 将每行数据转换为TestData对象
+        DefaultLineMapper<TestData> mapper = new DefaultLineMapper<>();
+        mapper.setLineTokenizer(tokenizer);
+        // 设置映射方式
+        mapper.setFieldSetMapper(fieldSet -> {
+            TestData data = new TestData();
+            data.setId(fieldSet.readInt("id"));
+            data.setField1(fieldSet.readString("field1"));
+            data.setField2(fieldSet.readString("field2"));
+            data.setField3(fieldSet.readString("field3"));
+            return data;
+        });
+
+        reader.setLineMapper(mapper);
+        return reader;
+    }
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MySimpleItemReaderDemo.java

@@ -0,0 +1,46 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.reader.MySimpleIteamReader;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class MySimpleItemReaderDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+
+    @Bean
+    public Job mySimpleItemReaderJob() {
+        return jobBuilderFactory.get("mySimpleItemReaderJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<String, String>chunk(2)
+                .reader(mySimpleItemReader())
+                .writer(list -> list.forEach(System.out::println))  // 简单输出，后面再详细介绍writer
+                .build();
+    }
+
+    private ItemReader<String> mySimpleItemReader() {
+        List<String> data = Arrays.asList("java", "c++", "javascript", "python");
+        return new MySimpleIteamReader(data);
+    }
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/XmlFileItemReaderDemo.java

@@ -0,0 +1,59 @@
+package cc.mrbird.batch.job;
+
+import cc.mrbird.batch.entity.TestData;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.xml.StaxEventItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.oxm.xstream.XStreamMarshaller;
+import org.springframework.stereotype.Component;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author MrBird
+ */
+@Component
+public class XmlFileItemReaderDemo {
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+
+    @Bean
+    public Job xmlFileItemReaderJob() {
+        return jobBuilderFactory.get("xmlFileItemReaderJob")
+                .start(step())
+                .build();
+    }
+
+    private Step step() {
+        return stepBuilderFactory.get("step")
+                .<TestData, TestData>chunk(2)
+                .reader(xmlFileItemReader())
+                .writer(list -> list.forEach(System.out::println))
+                .build();
+    }
+
+    private ItemReader<TestData> xmlFileItemReader() {
+        StaxEventItemReader<TestData> reader = new StaxEventItemReader<>();
+        reader.setResource(new ClassPathResource("file.xml")); // 设置xml文件源
+        reader.setFragmentRootElementName("test"); // 指定xml文件的根标签
+        // 将xml数据转换为TestData对象
+        XStreamMarshaller marshaller = new XStreamMarshaller();
+        // 指定需要转换的目标数据类型
+        Map<String, Class<TestData>> map = new HashMap<>(1);
+        map.put("test", TestData.class);
+        marshaller.setAliases(map);
+
+        reader.setUnmarshaller(marshaller);
+        return reader;
+    }
+}

68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/reader/MySimpleIteamReader.java

@@ -0,0 +1,24 @@
+package cc.mrbird.batch.reader;
+
+import org.springframework.batch.item.ItemReader;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author MrBird
+ */
+public class MySimpleIteamReader implements ItemReader<String> {
+
+    private Iterator<String> iterator;
+
+    public MySimpleIteamReader(List<String> data) {
+        this.iterator = data.iterator();
+    }
+
+    @Override
+    public String read() {
+        // 数据一个接着一个读取
+        return iterator.hasNext() ? iterator.next() : null;
+    }
+}

68.spring-batch-itemreader/src/main/resources/TEST.sql

@@ -0,0 +1,44 @@
+/*
+ Navicat Premium Data Transfer
+
+ Source Server         : localhost_mysql
+ Source Server Type    : MySQL
+ Source Server Version : 50724
+ Source Host           : localhost:3306
+ Source Schema         : springbatch
+
+ Target Server Type    : MySQL
+ Target Server Version : 50724
+ File Encoding         : 65001
+
+ Date: 07/03/2020 15:50:05
+*/
+
+SET NAMES utf8mb4;
+SET FOREIGN_KEY_CHECKS = 0;
+
+-- ----------------------------
+-- Table structure for TEST
+-- ----------------------------
+DROP TABLE IF EXISTS `TEST`;
+CREATE TABLE `TEST` (
+  `id` bigint(10) NOT NULL COMMENT 'ID',
+  `field1` varchar(10) NOT NULL COMMENT '字段一',
+  `field2` varchar(10) NOT NULL COMMENT '字段二',
+  `field3` varchar(10) NOT NULL COMMENT '字段三',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Records of TEST
+-- ----------------------------
+BEGIN;
+INSERT INTO `TEST` VALUES (1, '11', '12', '13');
+INSERT INTO `TEST` VALUES (2, '21', '22', '23');
+INSERT INTO `TEST` VALUES (3, '31', '32', '33');
+INSERT INTO `TEST` VALUES (4, '41', '42', '43');
+INSERT INTO `TEST` VALUES (5, '51', '52', '53');
+INSERT INTO `TEST` VALUES (6, '61', '62', '63');
+COMMIT;
+
+SET FOREIGN_KEY_CHECKS = 1;

68.spring-batch-itemreader/src/main/resources/application.yml

@@ -0,0 +1,6 @@
+spring:
+  datasource:
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/springbatch
+    username: root
+    password: 123456

68.spring-batch-itemreader/src/main/resources/file

@@ -0,0 +1,7 @@
+// 演示文件数据读取
+1,11,12,13
+2,21,22,23
+3,31,32,33
+4,41,42,43
+5,51,52,53
+6,61,62,63

68.spring-batch-itemreader/src/main/resources/file.json

@@ -0,0 +1,20 @@
+[
+  {
+    "id": 1,
+    "field1": "11",
+    "field2": "12",
+    "field3": "13"
+  },
+  {
+    "id": 2,
+    "field1": "21",
+    "field2": "22",
+    "field3": "23"
+  },
+  {
+    "id": 3,
+    "field1": "31",
+    "field2": "32",
+    "field3": "33"
+  }
+]

68.spring-batch-itemreader/src/main/resources/file.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<tests>
+    <test>
+        <id>1</id>
+        <field1>11</field1>
+        <field2>12</field2>
+        <field3>13</field3>
+    </test>
+    <test>
+        <id>2</id>
+        <field1>21</field1>
+        <field2>22</field2>
+        <field3>23</field3>
+    </test>
+    <test>
+        <id>3</id>
+        <field1>31</field1>
+        <field2>32</field2>
+        <field3>33</field3>
+    </test>
+    <test>
+        <id>4</id>
+        <field1>41</field1>
+        <field2>42</field2>
+        <field3>43</field3>
+    </test>
+    <test>
+        <id>5</id>
+        <field1>51</field1>
+        <field2>52</field2>
+        <field3>53</field3>
+    </test>
+    <test>
+        <id>6</id>
+        <field1>61</field1>
+        <field2>62</field2>
+        <field3>63</field3>
+    </test>
+</tests>

68.spring-batch-itemreader/src/main/resources/file1

@@ -0,0 +1,7 @@
+// 演示文件数据读取
+1,11,12,13
+2,21,22,23
+3,31,32,33
+4,41,42,43
+5,51,52,53
+6,61,62,63

68.spring-batch-itemreader/src/main/resources/file2

@@ -0,0 +1,3 @@
+// 演示文件数据读取
+7,71,72,73
+8,81,82,83