learning
/
SpringAll
mirror of https://github.com/wuyouzhuguli/SpringAll.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Spring Batch读取数据

This commit is contained in:
MrBird 2020-03-12 11:23:47 +08:00
parent 82c9fc3832
commit 9e41813382
17 changed files with 668 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
68.spring-batch-itemreader/pom.xml Normal file
View File

@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.5.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>cc.mrbird</groupId>
<artifactId>spring-batch-itemreader</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>spring-batch-itemreader</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-batch</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-oxm</artifactId>
</dependency>
<dependency>
<groupId>com.thoughtworks.xstream</groupId>
<artifactId>xstream</artifactId>
<version>1.4.11.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

15
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/SpringBatchItemreaderApplication.java Normal file
View File

@ -0,0 +1,15 @@
package cc.mrbird.batch;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
@EnableBatchProcessing
public class SpringBatchItemreaderApplication {
public static void main(String[] args) {
SpringApplication.run(SpringBatchItemreaderApplication.class, args);
}
}

53
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/entity/TestData.java Normal file
View File

@ -0,0 +1,53 @@
package cc.mrbird.batch.entity;
/**
* @author MrBird
*/
public class TestData {
private int id;
private String field1;
private String field2;
private String field3;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getField1() {
return field1;
}
public void setField1(String field1) {
this.field1 = field1;
}
public String getField2() {
return field2;
}
public void setField2(String field2) {
this.field2 = field2;
}
public String getField3() {
return field3;
}
public void setField3(String field3) {
this.field3 = field3;
}
@Override
public String toString() {
return "TestData{" +
"id=" + id +
", field1='" + field1 + '\'' +
", field2='" + field2 + '\'' +
", field3='" + field3 + '\'' +
'}';
}
}

80
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/DataSourceItemReaderDemo.java Normal file
View File

@ -0,0 +1,80 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.database.JdbcPagingItemReader;
import org.springframework.batch.item.database.Order;
import org.springframework.batch.item.database.support.MySqlPagingQueryProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import javax.sql.DataSource;
import java.util.HashMap;
import java.util.Map;
/**
* @author MrBird
*/
@Component
public class DataSourceItemReaderDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
// 注入数据源
@Autowired
private DataSource dataSource;
@Bean
public Job dataSourceItemReaderJob() throws Exception {
return jobBuilderFactory.get("dataSourceItemReaderJob")
.start(step())
.build();
}
private Step step() throws Exception {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(dataSourceItemReader())
.writer(list -> list.forEach(System.out::println))
.build();
}
private ItemReader<TestData> dataSourceItemReader() throws Exception {
JdbcPagingItemReader<TestData> reader = new JdbcPagingItemReader<>();
reader.setDataSource(dataSource); // 设置数据源
reader.setFetchSize(5); // 每次取多少条记录
reader.setPageSize(5); // 设置每页数据量
// 指定sql查询语句 select id,field1,field2,field3 from TEST
MySqlPagingQueryProvider provider = new MySqlPagingQueryProvider();
provider.setSelectClause("id,field1,field2,field3"); //设置查询字段
provider.setFromClause("from TEST"); // 设置从哪张表查询
// 将读取到的数据转换为TestData对象
reader.setRowMapper((resultSet, rowNum) -> {
TestData data = new TestData();
data.setId(resultSet.getInt(1));
data.setField1(resultSet.getString(2)); // 读取第一个字段，类型为String
data.setField2(resultSet.getString(3));
data.setField3(resultSet.getString(4));
return data;
});
Map<String, Order> sort = new HashMap<>(1);
sort.put("id", Order.ASCENDING);
provider.setSortKeys(sort); // 设置排序,通过id 升序
reader.setQueryProvider(provider);
// 设置namedParameterJdbcTemplate等属性
reader.afterPropertiesSet();
return reader;
}
}

71
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/FileItemReaderDemo.java Normal file
View File

@ -0,0 +1,71 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class FileItemReaderDemo {
// 任务创建工厂
@Autowired
private JobBuilderFactory jobBuilderFactory;
// 步骤创建工厂
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Bean
public Job fileItemReaderJob() {
return jobBuilderFactory.get("fileItemReaderJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(fileItemReader())
.writer(list -> list.forEach(System.out::println))
.build();
}
private ItemReader<TestData> fileItemReader() {
FlatFileItemReader<TestData> reader = new FlatFileItemReader<>();
reader.setResource(new ClassPathResource("file")); // 设置文件资源地址
reader.setLinesToSkip(1); // 忽略第一行
// AbstractLineTokenizer的三个实现类之一，以固定分隔符处理行数据读取,
// 使用默认构造器的时候，使用逗号作为分隔符，也可以通过有参构造器来指定分隔符
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
// 设置属性名，类似于表头
tokenizer.setNames("id", "field1", "field2", "field3");
// 将每行数据转换为TestData对象
DefaultLineMapper<TestData> mapper = new DefaultLineMapper<>();
mapper.setLineTokenizer(tokenizer);
// 设置映射方式
mapper.setFieldSetMapper(fieldSet -> {
TestData data = new TestData();
data.setId(fieldSet.readInt("id"));
data.setField1(fieldSet.readString("field1"));
data.setField2(fieldSet.readString("field2"));
data.setField3(fieldSet.readString("field3"));
return data;
});
reader.setLineMapper(mapper);
return reader;
}
}

52
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/JSONFileItemReaderDemo.java Normal file
View File

@ -0,0 +1,52 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.json.JacksonJsonObjectReader;
import org.springframework.batch.item.json.JsonItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*/
@Component
public class JSONFileItemReaderDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Bean
public Job jsonFileItemReaderJob() {
return jobBuilderFactory.get("jsonFileItemReaderJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(jsonItemReader())
.writer(list -> list.forEach(System.out::println))
.build();
}
private ItemReader<TestData> jsonItemReader() {
// 设置json文件地址
ClassPathResource resource = new ClassPathResource("file.json");
// 设置json文件转换的目标对象类型
JacksonJsonObjectReader<TestData> jacksonJsonObjectReader = new JacksonJsonObjectReader<>(TestData.class);
JsonItemReader<TestData> reader = new JsonItemReader<>(resource, jacksonJsonObjectReader);
// 给reader设置一个别名
reader.setName("testDataJsonItemReader");
return reader;
}
}

86
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MultiFileIteamReaderDemo.java Normal file
View File

@ -0,0 +1,86 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.MultiResourceItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Component;
/**
* @author MrBird
*
* 5. 演示多文件读取
*/
@Component
public class MultiFileIteamReaderDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Bean
public Job multiFileItemReaderJob() {
return jobBuilderFactory.get("multiFileItemReaderJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(multiFileItemReader())
.writer(list -> list.forEach(System.out::println))
.build();
}
private ItemReader<TestData> multiFileItemReader() {
MultiResourceItemReader<TestData> reader = new MultiResourceItemReader<>();
reader.setDelegate(fileItemReader()); // 设置文件读取代理，方法可以使用前面文件读取中的例子
Resource[] resources = new Resource[]{
new ClassPathResource("file1"),
new ClassPathResource("file2")
};
reader.setResources(resources); // 设置多文件源
return reader;
}
private FlatFileItemReader<TestData> fileItemReader() {
FlatFileItemReader<TestData> reader = new FlatFileItemReader<>();
reader.setLinesToSkip(1); // 忽略第一行
// AbstractLineTokenizer的三个实现类之一，以固定分隔符处理行数据读取,
// 使用默认构造器的时候，使用逗号作为分隔符，也可以通过有参构造器来指定分隔符
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
// 设置属姓名，类似于表头
tokenizer.setNames("id", "field1", "field2", "field3");
// 将每行数据转换为TestData对象
DefaultLineMapper<TestData> mapper = new DefaultLineMapper<>();
mapper.setLineTokenizer(tokenizer);
// 设置映射方式
mapper.setFieldSetMapper(fieldSet -> {
TestData data = new TestData();
data.setId(fieldSet.readInt("id"));
data.setField1(fieldSet.readString("field1"));
data.setField2(fieldSet.readString("field2"));
data.setField3(fieldSet.readString("field3"));
return data;
});
reader.setLineMapper(mapper);
return reader;
}
}

46
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MySimpleItemReaderDemo.java Normal file
View File

@ -0,0 +1,46 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.reader.MySimpleIteamReader;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.List;
/**
* @author MrBird
*/
@Component
public class MySimpleItemReaderDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Bean
public Job mySimpleItemReaderJob() {
return jobBuilderFactory.get("mySimpleItemReaderJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<String, String>chunk(2)
.reader(mySimpleItemReader())
.writer(list -> list.forEach(System.out::println)) // 简单输出，后面再详细介绍writer
.build();
}
private ItemReader<String> mySimpleItemReader() {
List<String> data = Arrays.asList("java", "c++", "javascript", "python");
return new MySimpleIteamReader(data);
}
}

59
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/XmlFileItemReaderDemo.java Normal file
View File

@ -0,0 +1,59 @@
package cc.mrbird.batch.job;
import cc.mrbird.batch.entity.TestData;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.xml.StaxEventItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.oxm.xstream.XStreamMarshaller;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.Map;
/**
* @author MrBird
*/
@Component
public class XmlFileItemReaderDemo {
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Bean
public Job xmlFileItemReaderJob() {
return jobBuilderFactory.get("xmlFileItemReaderJob")
.start(step())
.build();
}
private Step step() {
return stepBuilderFactory.get("step")
.<TestData, TestData>chunk(2)
.reader(xmlFileItemReader())
.writer(list -> list.forEach(System.out::println))
.build();
}
private ItemReader<TestData> xmlFileItemReader() {
StaxEventItemReader<TestData> reader = new StaxEventItemReader<>();
reader.setResource(new ClassPathResource("file.xml")); // 设置xml文件源
reader.setFragmentRootElementName("test"); // 指定xml文件的根标签
// 将xml数据转换为TestData对象
XStreamMarshaller marshaller = new XStreamMarshaller();
// 指定需要转换的目标数据类型
Map<String, Class<TestData>> map = new HashMap<>(1);
map.put("test", TestData.class);
marshaller.setAliases(map);
reader.setUnmarshaller(marshaller);
return reader;
}
}

24
68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/reader/MySimpleIteamReader.java Normal file
View File

@ -0,0 +1,24 @@
package cc.mrbird.batch.reader;
import org.springframework.batch.item.ItemReader;
import java.util.Iterator;
import java.util.List;
/**
* @author MrBird
*/
public class MySimpleIteamReader implements ItemReader<String> {
private Iterator<String> iterator;
public MySimpleIteamReader(List<String> data) {
this.iterator = data.iterator();
}
@Override
public String read() {
// 数据一个接着一个读取
return iterator.hasNext() ? iterator.next() : null;
}
}

44
68.spring-batch-itemreader/src/main/resources/TEST.sql Normal file
View File

@ -0,0 +1,44 @@
/*
Navicat Premium Data Transfer
Source Server : localhost_mysql
Source Server Type : MySQL
Source Server Version : 50724
Source Host : localhost:3306
Source Schema : springbatch
Target Server Type : MySQL
Target Server Version : 50724
File Encoding : 65001
Date: 07/03/2020 15:50:05
*/
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for TEST
-- ----------------------------
DROP TABLE IF EXISTS `TEST`;
CREATE TABLE `TEST` (
`id` bigint(10) NOT NULL COMMENT 'ID',
`field1` varchar(10) NOT NULL COMMENT '字段一',
`field2` varchar(10) NOT NULL COMMENT '字段二',
`field3` varchar(10) NOT NULL COMMENT '字段三',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- ----------------------------
-- Records of TEST
-- ----------------------------
BEGIN;
INSERT INTO `TEST` VALUES (1, '11', '12', '13');
INSERT INTO `TEST` VALUES (2, '21', '22', '23');
INSERT INTO `TEST` VALUES (3, '31', '32', '33');
INSERT INTO `TEST` VALUES (4, '41', '42', '43');
INSERT INTO `TEST` VALUES (5, '51', '52', '53');
INSERT INTO `TEST` VALUES (6, '61', '62', '63');
COMMIT;
SET FOREIGN_KEY_CHECKS = 1;

6
68.spring-batch-itemreader/src/main/resources/application.yml Normal file
View File

@ -0,0 +1,6 @@
spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/springbatch
username: root
password: 123456

7
68.spring-batch-itemreader/src/main/resources/file Normal file
View File

@ -0,0 +1,7 @@
// 演示文件数据读取
1,11,12,13
2,21,22,23
3,31,32,33
4,41,42,43
5,51,52,53
6,61,62,63

20
68.spring-batch-itemreader/src/main/resources/file.json Normal file
View File

@ -0,0 +1,20 @@
[
{
"id": 1,
"field1": "11",
"field2": "12",
"field3": "13"
},
{
"id": 2,
"field1": "21",
"field2": "22",
"field3": "23"
},
{
"id": 3,
"field1": "31",
"field2": "32",
"field3": "33"
}
]

39
68.spring-batch-itemreader/src/main/resources/file.xml Normal file
View File

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="utf-8" ?>
<tests>
<test>
<id>1</id>
<field1>11</field1>
<field2>12</field2>
<field3>13</field3>
</test>
<test>
<id>2</id>
<field1>21</field1>
<field2>22</field2>
<field3>23</field3>
</test>
<test>
<id>3</id>
<field1>31</field1>
<field2>32</field2>
<field3>33</field3>
</test>
<test>
<id>4</id>
<field1>41</field1>
<field2>42</field2>
<field3>43</field3>
</test>
<test>
<id>5</id>
<field1>51</field1>
<field2>52</field2>
<field3>53</field3>
</test>
<test>
<id>6</id>
<field1>61</field1>
<field2>62</field2>
<field3>63</field3>
</test>
</tests>

7
68.spring-batch-itemreader/src/main/resources/file1 Normal file
View File

@ -0,0 +1,7 @@
// 演示文件数据读取
1,11,12,13
2,21,22,23
3,31,32,33
4,41,42,43
5,51,52,53
6,61,62,63

3
68.spring-batch-itemreader/src/main/resources/file2 Normal file
View File

@ -0,0 +1,3 @@
// 演示文件数据读取
7,71,72,73
8,81,82,83