diff --git a/68.spring-batch-itemreader/pom.xml b/68.spring-batch-itemreader/pom.xml new file mode 100644 index 0000000..ce15ccd --- /dev/null +++ b/68.spring-batch-itemreader/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.2.5.RELEASE + + + cc.mrbird + spring-batch-itemreader + 0.0.1-SNAPSHOT + spring-batch-itemreader + Demo project for Spring Boot + + + 1.8 + + + + + org.springframework.boot + spring-boot-starter-batch + + + + mysql + mysql-connector-java + + + org.springframework.boot + spring-boot-starter-jdbc + + + + org.springframework + spring-oxm + + + com.thoughtworks.xstream + xstream + 1.4.11.1 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/SpringBatchItemreaderApplication.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/SpringBatchItemreaderApplication.java new file mode 100644 index 0000000..242ed00 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/SpringBatchItemreaderApplication.java @@ -0,0 +1,15 @@ +package cc.mrbird.batch; + +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +@EnableBatchProcessing +public class SpringBatchItemreaderApplication { + + public static void main(String[] args) { + SpringApplication.run(SpringBatchItemreaderApplication.class, args); + } + +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/entity/TestData.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/entity/TestData.java new file mode 100644 index 0000000..e7119c0 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/entity/TestData.java @@ -0,0 +1,53 @@ +package cc.mrbird.batch.entity; + +/** + * @author MrBird + */ +public class TestData { + private int id; + private String field1; + private String field2; + private String field3; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getField1() { + return field1; + } + + public void setField1(String field1) { + this.field1 = field1; + } + + public String getField2() { + return field2; + } + + public void setField2(String field2) { + this.field2 = field2; + } + + public String getField3() { + return field3; + } + + public void setField3(String field3) { + this.field3 = field3; + } + + @Override + public String toString() { + return "TestData{" + + "id=" + id + + ", field1='" + field1 + '\'' + + ", field2='" + field2 + '\'' + + ", field3='" + field3 + '\'' + + '}'; + } +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/DataSourceItemReaderDemo.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/DataSourceItemReaderDemo.java new file mode 100644 index 0000000..edc148e --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/DataSourceItemReaderDemo.java @@ -0,0 +1,80 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.database.JdbcPagingItemReader; +import org.springframework.batch.item.database.Order; +import org.springframework.batch.item.database.support.MySqlPagingQueryProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import javax.sql.DataSource; +import java.util.HashMap; +import java.util.Map; + +/** + * @author MrBird + */ +@Component +public class DataSourceItemReaderDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + // 注入数据源 + @Autowired + private DataSource dataSource; + + @Bean + public Job dataSourceItemReaderJob() throws Exception { + return jobBuilderFactory.get("dataSourceItemReaderJob") + .start(step()) + .build(); + } + + private Step step() throws Exception { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(dataSourceItemReader()) + .writer(list -> list.forEach(System.out::println)) + .build(); + } + + private ItemReader dataSourceItemReader() throws Exception { + JdbcPagingItemReader reader = new JdbcPagingItemReader<>(); + reader.setDataSource(dataSource); // 设置数据源 + reader.setFetchSize(5); // 每次取多少条记录 + reader.setPageSize(5); // 设置每页数据量 + + // 指定sql查询语句 select id,field1,field2,field3 from TEST + MySqlPagingQueryProvider provider = new MySqlPagingQueryProvider(); + provider.setSelectClause("id,field1,field2,field3"); //设置查询字段 + provider.setFromClause("from TEST"); // 设置从哪张表查询 + + // 将读取到的数据转换为TestData对象 + reader.setRowMapper((resultSet, rowNum) -> { + TestData data = new TestData(); + data.setId(resultSet.getInt(1)); + data.setField1(resultSet.getString(2)); // 读取第一个字段，类型为String + data.setField2(resultSet.getString(3)); + data.setField3(resultSet.getString(4)); + return data; + }); + + Map sort = new HashMap<>(1); + sort.put("id", Order.ASCENDING); + provider.setSortKeys(sort); // 设置排序,通过id 升序 + + reader.setQueryProvider(provider); + + // 设置namedParameterJdbcTemplate等属性 + reader.afterPropertiesSet(); + return reader; + } +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/FileItemReaderDemo.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/FileItemReaderDemo.java new file mode 100644 index 0000000..5824593 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/FileItemReaderDemo.java @@ -0,0 +1,71 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.mapping.DefaultLineMapper; +import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Component; + +/** + * @author MrBird + */ +@Component +public class FileItemReaderDemo { + // 任务创建工厂 + @Autowired + private JobBuilderFactory jobBuilderFactory; + // 步骤创建工厂 + @Autowired + private StepBuilderFactory stepBuilderFactory; + + @Bean + public Job fileItemReaderJob() { + return jobBuilderFactory.get("fileItemReaderJob") + .start(step()) + .build(); + } + + private Step step() { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(fileItemReader()) + .writer(list -> list.forEach(System.out::println)) + .build(); + } + + private ItemReader fileItemReader() { + FlatFileItemReader reader = new FlatFileItemReader<>(); + reader.setResource(new ClassPathResource("file")); // 设置文件资源地址 + reader.setLinesToSkip(1); // 忽略第一行 + + // AbstractLineTokenizer的三个实现类之一，以固定分隔符处理行数据读取, + // 使用默认构造器的时候，使用逗号作为分隔符，也可以通过有参构造器来指定分隔符 + DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(); + + // 设置属性名，类似于表头 + tokenizer.setNames("id", "field1", "field2", "field3"); + // 将每行数据转换为TestData对象 + DefaultLineMapper mapper = new DefaultLineMapper<>(); + mapper.setLineTokenizer(tokenizer); + // 设置映射方式 + mapper.setFieldSetMapper(fieldSet -> { + TestData data = new TestData(); + data.setId(fieldSet.readInt("id")); + data.setField1(fieldSet.readString("field1")); + data.setField2(fieldSet.readString("field2")); + data.setField3(fieldSet.readString("field3")); + return data; + }); + + reader.setLineMapper(mapper); + return reader; + } +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/JSONFileItemReaderDemo.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/JSONFileItemReaderDemo.java new file mode 100644 index 0000000..85a2662 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/JSONFileItemReaderDemo.java @@ -0,0 +1,52 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.json.JacksonJsonObjectReader; +import org.springframework.batch.item.json.JsonItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Component; + +/** + * @author MrBird + */ +@Component +public class JSONFileItemReaderDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + + @Bean + public Job jsonFileItemReaderJob() { + return jobBuilderFactory.get("jsonFileItemReaderJob") + .start(step()) + .build(); + } + + private Step step() { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(jsonItemReader()) + .writer(list -> list.forEach(System.out::println)) + .build(); + } + + private ItemReader jsonItemReader() { + // 设置json文件地址 + ClassPathResource resource = new ClassPathResource("file.json"); + // 设置json文件转换的目标对象类型 + JacksonJsonObjectReader jacksonJsonObjectReader = new JacksonJsonObjectReader<>(TestData.class); + JsonItemReader reader = new JsonItemReader<>(resource, jacksonJsonObjectReader); + // 给reader设置一个别名 + reader.setName("testDataJsonItemReader"); + return reader; + } +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MultiFileIteamReaderDemo.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MultiFileIteamReaderDemo.java new file mode 100644 index 0000000..19b8b8b --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MultiFileIteamReaderDemo.java @@ -0,0 +1,86 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.MultiResourceItemReader; +import org.springframework.batch.item.file.mapping.DefaultLineMapper; +import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Component; + +/** + * @author MrBird + * + * 5. 演示多文件读取 + */ +@Component +public class MultiFileIteamReaderDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + + + @Bean + public Job multiFileItemReaderJob() { + return jobBuilderFactory.get("multiFileItemReaderJob") + .start(step()) + .build(); + } + + private Step step() { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(multiFileItemReader()) + .writer(list -> list.forEach(System.out::println)) + .build(); + } + + private ItemReader multiFileItemReader() { + MultiResourceItemReader reader = new MultiResourceItemReader<>(); + reader.setDelegate(fileItemReader()); // 设置文件读取代理，方法可以使用前面文件读取中的例子 + + Resource[] resources = new Resource[]{ + new ClassPathResource("file1"), + new ClassPathResource("file2") + }; + + reader.setResources(resources); // 设置多文件源 + return reader; + } + + private FlatFileItemReader fileItemReader() { + FlatFileItemReader reader = new FlatFileItemReader<>(); + reader.setLinesToSkip(1); // 忽略第一行 + + // AbstractLineTokenizer的三个实现类之一，以固定分隔符处理行数据读取, + // 使用默认构造器的时候，使用逗号作为分隔符，也可以通过有参构造器来指定分隔符 + DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(); + // 设置属姓名，类似于表头 + tokenizer.setNames("id", "field1", "field2", "field3"); + // 将每行数据转换为TestData对象 + DefaultLineMapper mapper = new DefaultLineMapper<>(); + mapper.setLineTokenizer(tokenizer); + // 设置映射方式 + mapper.setFieldSetMapper(fieldSet -> { + TestData data = new TestData(); + data.setId(fieldSet.readInt("id")); + data.setField1(fieldSet.readString("field1")); + data.setField2(fieldSet.readString("field2")); + data.setField3(fieldSet.readString("field3")); + return data; + }); + + reader.setLineMapper(mapper); + return reader; + } +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MySimpleItemReaderDemo.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MySimpleItemReaderDemo.java new file mode 100644 index 0000000..f7bd94c --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/MySimpleItemReaderDemo.java @@ -0,0 +1,46 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.reader.MySimpleIteamReader; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import java.util.Arrays; +import java.util.List; + +/** + * @author MrBird + */ +@Component +public class MySimpleItemReaderDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + + @Bean + public Job mySimpleItemReaderJob() { + return jobBuilderFactory.get("mySimpleItemReaderJob") + .start(step()) + .build(); + } + + private Step step() { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(mySimpleItemReader()) + .writer(list -> list.forEach(System.out::println)) // 简单输出，后面再详细介绍writer + .build(); + } + + private ItemReader mySimpleItemReader() { + List data = Arrays.asList("java", "c++", "javascript", "python"); + return new MySimpleIteamReader(data); + } +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/XmlFileItemReaderDemo.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/XmlFileItemReaderDemo.java new file mode 100644 index 0000000..c8411b1 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/job/XmlFileItemReaderDemo.java @@ -0,0 +1,59 @@ +package cc.mrbird.batch.job; + +import cc.mrbird.batch.entity.TestData; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.xml.StaxEventItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.oxm.xstream.XStreamMarshaller; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author MrBird + */ +@Component +public class XmlFileItemReaderDemo { + + @Autowired + private JobBuilderFactory jobBuilderFactory; + @Autowired + private StepBuilderFactory stepBuilderFactory; + + @Bean + public Job xmlFileItemReaderJob() { + return jobBuilderFactory.get("xmlFileItemReaderJob") + .start(step()) + .build(); + } + + private Step step() { + return stepBuilderFactory.get("step") + .chunk(2) + .reader(xmlFileItemReader()) + .writer(list -> list.forEach(System.out::println)) + .build(); + } + + private ItemReader xmlFileItemReader() { + StaxEventItemReader reader = new StaxEventItemReader<>(); + reader.setResource(new ClassPathResource("file.xml")); // 设置xml文件源 + reader.setFragmentRootElementName("test"); // 指定xml文件的根标签 + // 将xml数据转换为TestData对象 + XStreamMarshaller marshaller = new XStreamMarshaller(); + // 指定需要转换的目标数据类型 + Map> map = new HashMap<>(1); + map.put("test", TestData.class); + marshaller.setAliases(map); + + reader.setUnmarshaller(marshaller); + return reader; + } +} diff --git a/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/reader/MySimpleIteamReader.java b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/reader/MySimpleIteamReader.java new file mode 100644 index 0000000..215126a --- /dev/null +++ b/68.spring-batch-itemreader/src/main/java/cc/mrbird/batch/reader/MySimpleIteamReader.java @@ -0,0 +1,24 @@ +package cc.mrbird.batch.reader; + +import org.springframework.batch.item.ItemReader; + +import java.util.Iterator; +import java.util.List; + +/** + * @author MrBird + */ +public class MySimpleIteamReader implements ItemReader { + + private Iterator iterator; + + public MySimpleIteamReader(List data) { + this.iterator = data.iterator(); + } + + @Override + public String read() { + // 数据一个接着一个读取 + return iterator.hasNext() ? iterator.next() : null; + } +} diff --git a/68.spring-batch-itemreader/src/main/resources/TEST.sql b/68.spring-batch-itemreader/src/main/resources/TEST.sql new file mode 100644 index 0000000..ca3ee5c --- /dev/null +++ b/68.spring-batch-itemreader/src/main/resources/TEST.sql @@ -0,0 +1,44 @@ +/* + Navicat Premium Data Transfer + + Source Server : localhost_mysql + Source Server Type : MySQL + Source Server Version : 50724 + Source Host : localhost:3306 + Source Schema : springbatch + + Target Server Type : MySQL + Target Server Version : 50724 + File Encoding : 65001 + + Date: 07/03/2020 15:50:05 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for TEST +-- ---------------------------- +DROP TABLE IF EXISTS `TEST`; +CREATE TABLE `TEST` ( + `id` bigint(10) NOT NULL COMMENT 'ID', + `field1` varchar(10) NOT NULL COMMENT '字段一', + `field2` varchar(10) NOT NULL COMMENT '字段二', + `field3` varchar(10) NOT NULL COMMENT '字段三', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Records of TEST +-- ---------------------------- +BEGIN; +INSERT INTO `TEST` VALUES (1, '11', '12', '13'); +INSERT INTO `TEST` VALUES (2, '21', '22', '23'); +INSERT INTO `TEST` VALUES (3, '31', '32', '33'); +INSERT INTO `TEST` VALUES (4, '41', '42', '43'); +INSERT INTO `TEST` VALUES (5, '51', '52', '53'); +INSERT INTO `TEST` VALUES (6, '61', '62', '63'); +COMMIT; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/68.spring-batch-itemreader/src/main/resources/application.yml b/68.spring-batch-itemreader/src/main/resources/application.yml new file mode 100644 index 0000000..4c962f7 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/resources/application.yml @@ -0,0 +1,6 @@ +spring: + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/springbatch + username: root + password: 123456 \ No newline at end of file diff --git a/68.spring-batch-itemreader/src/main/resources/file b/68.spring-batch-itemreader/src/main/resources/file new file mode 100644 index 0000000..062d76d --- /dev/null +++ b/68.spring-batch-itemreader/src/main/resources/file @@ -0,0 +1,7 @@ +// 演示文件数据读取 +1,11,12,13 +2,21,22,23 +3,31,32,33 +4,41,42,43 +5,51,52,53 +6,61,62,63 \ No newline at end of file diff --git a/68.spring-batch-itemreader/src/main/resources/file.json b/68.spring-batch-itemreader/src/main/resources/file.json new file mode 100644 index 0000000..40811d6 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/resources/file.json @@ -0,0 +1,20 @@ +[ + { + "id": 1, + "field1": "11", + "field2": "12", + "field3": "13" + }, + { + "id": 2, + "field1": "21", + "field2": "22", + "field3": "23" + }, + { + "id": 3, + "field1": "31", + "field2": "32", + "field3": "33" + } +] \ No newline at end of file diff --git a/68.spring-batch-itemreader/src/main/resources/file.xml b/68.spring-batch-itemreader/src/main/resources/file.xml new file mode 100644 index 0000000..f131908 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/resources/file.xml @@ -0,0 +1,39 @@ + + + + 1 + 11 + 12 + 13 + + + 2 + 21 + 22 + 23 + + + 3 + 31 + 32 + 33 + + + 4 + 41 + 42 + 43 + + + 5 + 51 + 52 + 53 + + + 6 + 61 + 62 + 63 + + \ No newline at end of file diff --git a/68.spring-batch-itemreader/src/main/resources/file1 b/68.spring-batch-itemreader/src/main/resources/file1 new file mode 100644 index 0000000..062d76d --- /dev/null +++ b/68.spring-batch-itemreader/src/main/resources/file1 @@ -0,0 +1,7 @@ +// 演示文件数据读取 +1,11,12,13 +2,21,22,23 +3,31,32,33 +4,41,42,43 +5,51,52,53 +6,61,62,63 \ No newline at end of file diff --git a/68.spring-batch-itemreader/src/main/resources/file2 b/68.spring-batch-itemreader/src/main/resources/file2 new file mode 100644 index 0000000..7382f16 --- /dev/null +++ b/68.spring-batch-itemreader/src/main/resources/file2 @@ -0,0 +1,3 @@ +// 演示文件数据读取 +7,71,72,73 +8,81,82,83 \ No newline at end of file