Compare commits
24 Commits
Author | SHA1 | Date |
---|---|---|
|
ef4398d834 | |
|
7cb9d73a53 | |
|
e72d3a9ed0 | |
|
0720800755 | |
|
144d5f7a99 | |
|
8188868e16 | |
|
1713fd8d96 | |
|
fa1155911d | |
|
637cbbe327 | |
|
7abca3647d | |
|
9371be3854 | |
|
07770923c5 | |
|
4e8138d3a1 | |
|
c65e98972d | |
|
8b9e7859db | |
|
925a0e5435 | |
|
dd7ac4ae0d | |
|
ce5011e666 | |
|
4527321855 | |
|
0b638f82be | |
|
0067c76bd5 | |
|
6d8c265167 | |
|
3d7d43827b | |
|
ff9f7addfc |
|
@ -11,4 +11,3 @@ target
|
|||
node_modules
|
||||
test/derby.log
|
||||
/console-fe
|
||||
.flattened-pom.xml
|
||||
|
|
41
README.md
41
README.md
|
@ -1,11 +1,9 @@
|
|||
# Nacos Sync
|
||||
|
||||
## [Example](https://github.com/paderlol/nacos-sync-example)
|
||||
|
||||
## Function
|
||||
|
||||
- Console: provide API and console for management
|
||||
- Worker: provide the service registration synchronization.
|
||||
- Worker: provider the service registration synchronization.
|
||||
|
||||
## Architecture
|
||||
|
||||
|
@ -44,9 +42,15 @@ Info | +------------+ ^
|
|||
- NacosCluster target will dedup the synchronization information from Nacos.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Quick Start:
|
||||
- Swagger API: http://127.0.0.1:8083/swagger-ui.html#/
|
||||
- Web Console: http://127.0.0.1:8083/
|
||||
- Swagger API: http://127.0.0.1:8081/swagger-ui.html#/
|
||||
- Web Console: http://127.0.0.1:8081/
|
||||
- Others: TBD
|
||||
|
||||
# NacosSync Migration User Guide
|
||||
|
@ -78,7 +82,7 @@ Before you begin, install the following:
|
|||
|
||||
- 64bit OS: Linux/Unix/Mac/Windows supported, Linux/Unix/Mac recommended.
|
||||
- 64bit JDK 1.8+: downloads, JAVA_HOME settings.
|
||||
- Maven 3.5.2+: [downloads](https://maven.apache.org/download.cgi), [settings](https://maven.apache.org/settings.html).
|
||||
- Maven 3.2.x+: downloads, settings.
|
||||
- MySql 5.6.+
|
||||
|
||||
## Download & Build From Release
|
||||
|
@ -90,7 +94,7 @@ There are two ways to get NacosSync.
|
|||
|
||||
``` xml
|
||||
|
||||
cd nacos-sync/
|
||||
cd nacosSync/
|
||||
mvn clean package -U
|
||||
|
||||
```
|
||||
|
@ -99,7 +103,7 @@ The path to the target file:
|
|||
|
||||
``` xml
|
||||
|
||||
nacos-sync/nacossync-distribution/target/nacos-sync-0.5.0.tar.gz
|
||||
nacos-sync/nacossync-distribution/target/nacosSync.0.3.8.zip
|
||||
|
||||
```
|
||||
|
||||
|
@ -107,7 +111,7 @@ After extracting the installation package, the directory structure:
|
|||
|
||||
``` xml
|
||||
|
||||
nacos-sync
|
||||
nacosSync
|
||||
├── LICENSE
|
||||
├── NOTICE
|
||||
├── bin
|
||||
|
@ -118,7 +122,7 @@ nacos-sync
|
|||
│ ├── application.properties
|
||||
│ └── logback-spring.xml
|
||||
├── logs
|
||||
└── nacos-sync-server.jar
|
||||
└── nacosSync-server.jar
|
||||
|
||||
```
|
||||
|
||||
|
@ -126,7 +130,7 @@ nacos-sync
|
|||
|
||||
The default is Mysql database, which can support other relational databases
|
||||
|
||||
- Build db schema, the default schema name nacos_sync.
|
||||
- Build db schema, the default schema name nacos_Sync.
|
||||
- Tables do not need to be created separately, which is conducive to hibernate's automatic table creation function.
|
||||
- If the automatic table creation fails, you can build the table nacosSync.sql, the table statement is in the bin folder.
|
||||
|
||||
|
@ -155,18 +159,15 @@ sh startup.sh start
|
|||
|
||||
``` xml
|
||||
|
||||
http://127.0.0.1:8083/#/serviceSync
|
||||
http://127.0.0.1:8081/#/serviceSync
|
||||
|
||||
```
|
||||
|
||||
## Advanced Configuration
|
||||
|
||||
### Full Synchronization from Zookeeper to Nacos (Dubbo)
|
||||
When “*” is entered in the “Service Name” field of this form, it will fully synchronize all services from Zookeeper to Nacos, but only when using Dubbo.
|
||||

|
||||
|
||||
### Full Synchronization from Nacos to Nacos
|
||||
When “All” is entered in the “Service Name” field of this form, it will automatically synchronize all registered services within the **default group** of the current cluster.
|
||||
|
||||
This description explains the functionality clearly for English-speaking users.
|
||||

|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
<parent>
|
||||
<artifactId>nacossync-parent</artifactId>
|
||||
<groupId>com.alibaba.nacossync</groupId>
|
||||
<version>${revision}</version>
|
||||
<version>0.4.5</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
@ -27,4 +27,5 @@
|
|||
<artifactId>junit</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
</project>
|
||||
|
|
1
nacossync-console/src/main/resources/static/console-fe/dist/css/main.9093077e.css
vendored
Normal file
1
nacossync-console/src/main/resources/static/console-fe/dist/css/main.9093077e.css
vendored
Normal file
File diff suppressed because one or more lines are too long
|
@ -7,8 +7,8 @@
|
|||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||||
<title>Nacos-Sync</title>
|
||||
<link rel="shortcut icon" href="//www.aliyun.com/favicon.ico" type="image/x-icon">
|
||||
<link href="./css/main.e2917886.css" rel="stylesheet"></head>
|
||||
<link href="./css/main.9093077e.css" rel="stylesheet"></head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="text/javascript" src="./js/main.b73436b5.js"></script></body>
|
||||
<script type="text/javascript" src="./js/main.cf091959.js"></script></body>
|
||||
</html>
|
||||
|
|
328
nacossync-console/src/main/resources/static/console-fe/dist/js/main.cf091959.js
vendored
Normal file
328
nacossync-console/src/main/resources/static/console-fe/dist/js/main.cf091959.js
vendored
Normal file
File diff suppressed because one or more lines are too long
|
@ -18,9 +18,6 @@ class AddConfigDialog extends React.Component {
|
|||
visible: false,
|
||||
clusterName: '',
|
||||
clusterType: '',
|
||||
namespace: '',
|
||||
password: '',
|
||||
userName: '',
|
||||
connectKeyList: [],
|
||||
};
|
||||
}
|
||||
|
@ -30,8 +27,8 @@ class AddConfigDialog extends React.Component {
|
|||
}
|
||||
|
||||
save() {
|
||||
const { clusterName, namespace, userName, password, clusterType, connectKeyList } = this.state;
|
||||
add({ clusterName, namespace, userName, password, clusterType, connectKeyList })
|
||||
const { clusterName, clusterType, connectKeyList } = this.state;
|
||||
add({ clusterName, clusterType, connectKeyList })
|
||||
.then(() => {
|
||||
this.props.turnPage(1);
|
||||
this.close();
|
||||
|
@ -40,10 +37,7 @@ class AddConfigDialog extends React.Component {
|
|||
}
|
||||
|
||||
close() {
|
||||
this.setState({
|
||||
visible: false,
|
||||
clusterType: '',
|
||||
});
|
||||
this.setState({ visible: false });
|
||||
}
|
||||
|
||||
open = () => this.setState({ visible: true })
|
||||
|
@ -77,35 +71,6 @@ class AddConfigDialog extends React.Component {
|
|||
}
|
||||
</Select>
|
||||
</FormItem>
|
||||
{
|
||||
this.state.clusterType === 'NACOS' && (
|
||||
<>
|
||||
<FormItem
|
||||
label={`${locale.namespace}:`}
|
||||
>
|
||||
<Input
|
||||
placeholder={locale.namespacePlaceholder}
|
||||
onChange={ns => this.setState({ namespace: ns })}
|
||||
/>
|
||||
</FormItem>
|
||||
<FormItem
|
||||
label={`${locale.username}:`}
|
||||
>
|
||||
<Input
|
||||
placeholder={locale.usernamePlaceholder}
|
||||
onChange={un => this.setState({ userName: un })}
|
||||
/>
|
||||
</FormItem>
|
||||
<FormItem
|
||||
label={`${locale.password}:`}
|
||||
>
|
||||
<Input.Password
|
||||
placeholder={locale.passwordPlaceholder}
|
||||
onChange={pw => this.setState({ password: pw })}
|
||||
/>
|
||||
</FormItem>
|
||||
</>)
|
||||
}
|
||||
<FormItem label={`${locale.connectKeyList}:`}>
|
||||
<Input.TextArea
|
||||
onChange={(connectKeyListStr) => {
|
||||
|
|
|
@ -98,7 +98,6 @@ class ClusterConfig extends React.Component {
|
|||
<Table.Column title={locale.clusterName} dataIndex="clusterName" />
|
||||
<Table.Column title={locale.clusterType} dataIndex="clusterType" />
|
||||
<Table.Column title={locale.connectKeyList} dataIndex="connectKeyList" />
|
||||
<Table.Column title={locale.namespace} dataIndex="namespace" />
|
||||
<Table.Column
|
||||
title={locale.operation}
|
||||
cell={(value, index, record) => (
|
||||
|
|
|
@ -13,7 +13,7 @@ class Layout extends React.Component {
|
|||
<Header />
|
||||
<Row className="layout">
|
||||
<Col fixedSpan="9" className="nav-bar">
|
||||
<h1 className="title">Nacos-Sync 0.4.8</h1>
|
||||
<h1 className="title">Nacos-Sync 3.0</h1>
|
||||
<Menu />
|
||||
</Col>
|
||||
<Col className="main-panel">{this.props.children}</Col>
|
||||
|
|
|
@ -18,6 +18,7 @@ class AddSyncDialog extends React.Component {
|
|||
this.state = {
|
||||
visible: false,
|
||||
destClusterId: '',
|
||||
nameSpace: '',
|
||||
groupName: '',
|
||||
serviceName: '',
|
||||
sourceClusterId: '',
|
||||
|
@ -31,8 +32,8 @@ class AddSyncDialog extends React.Component {
|
|||
}
|
||||
|
||||
save() {
|
||||
const { destClusterId, groupName, serviceName, sourceClusterId, version } = this.state;
|
||||
add({ destClusterId, groupName, serviceName, sourceClusterId, version })
|
||||
const { destClusterId, nameSpace, groupName, serviceName, sourceClusterId, version } = this.state;
|
||||
add({ destClusterId, nameSpace, groupName, serviceName, sourceClusterId, version })
|
||||
.then(() => {
|
||||
this.props.turnPage(1);
|
||||
this.close();
|
||||
|
@ -64,6 +65,12 @@ class AddSyncDialog extends React.Component {
|
|||
onClose={() => this.close()}
|
||||
>
|
||||
<Form>
|
||||
<FormItem label={`${locale.nameSpace}:`}>
|
||||
<Input
|
||||
placeholder={locale.nameSpacePlaceholder}
|
||||
onChange={nameSpace => this.setState({ nameSpace })}
|
||||
/>
|
||||
</FormItem>
|
||||
<FormItem label={`${locale.serviceName}:`}>
|
||||
<Input
|
||||
placeholder={locale.serviceNamePlaceholder}
|
||||
|
|
|
@ -118,6 +118,7 @@ class ServiceSync extends React.Component {
|
|||
<Table dataSource={taskModels} loading={loading}>
|
||||
<Table.Column title={locale.serviceName} dataIndex="serviceName" />
|
||||
<Table.Column title={locale.groupName} dataIndex="groupName" />
|
||||
<Table.Column title={locale.nameSpace} dataIndex="nameSpace" />
|
||||
<Table.Column
|
||||
title={locale.sourceCluster}
|
||||
dataIndex="sourceClusterId"
|
||||
|
|
|
@ -19,9 +19,6 @@ const I18N_CONF = {
|
|||
clusterNamePlaceholder: 'Please enter the cluster name',
|
||||
clusterType: 'Cluster Type',
|
||||
connectKeyList: 'Connect Key List',
|
||||
namespace: 'Namespace',
|
||||
password: 'Password',
|
||||
username: 'Username',
|
||||
operation: 'Operation',
|
||||
deleteBtn: 'Delete',
|
||||
confirm: 'Prompt',
|
||||
|
@ -32,8 +29,6 @@ const I18N_CONF = {
|
|||
title: 'New Cluster',
|
||||
clusterName: 'Cluster Name',
|
||||
clusterNamePlaceholder: 'Please enter the cluster name',
|
||||
namespace: 'Namespace',
|
||||
namespacePlaceholder: 'Please enter the namespace',
|
||||
clusterType: 'Cluster Type',
|
||||
connectKeyList: 'Connect IP',
|
||||
connectKeyListPlaceholder: 'Please enter the ip',
|
||||
|
@ -43,6 +38,7 @@ const I18N_CONF = {
|
|||
serviceNamePlaceholder: 'Please enter service name',
|
||||
search: 'Search',
|
||||
addSync: 'New Sync',
|
||||
nameSpace: 'Namespace',
|
||||
serviceName: 'Service Name',
|
||||
groupName: 'Group',
|
||||
sourceCluster: 'Source Cluster',
|
||||
|
@ -65,6 +61,8 @@ const I18N_CONF = {
|
|||
serviceNamePlaceholder: 'Please enter service name',
|
||||
groupName: 'Group Name',
|
||||
groupNamePlaceholder: 'Please enter group name',
|
||||
nameSpace: 'Namespace',
|
||||
nameSpacePlaceholder: 'Please enter namespace id',
|
||||
sourceCluster: 'Source Cluster',
|
||||
destCluster: 'Dest Cluster',
|
||||
version: 'Version',
|
||||
|
|
|
@ -19,7 +19,6 @@ const I18N_CONF = {
|
|||
clusterNamePlaceholder: '请输入集群名',
|
||||
clusterType: '集群类型',
|
||||
connectKeyList: '集群IP列表',
|
||||
namespace: '命名空间',
|
||||
operation: '操作',
|
||||
deleteBtn: '删除',
|
||||
confirm: '提示',
|
||||
|
@ -30,12 +29,6 @@ const I18N_CONF = {
|
|||
title: '新增集群',
|
||||
clusterName: '集群名',
|
||||
clusterNamePlaceholder: '请输入集群名',
|
||||
namespace: '命名空间',
|
||||
namespacePlaceholder: '请输入命名空间',
|
||||
password: '密码',
|
||||
passwordPlaceholder: '请输入密码',
|
||||
username: '用户名',
|
||||
usernamePlaceholder: '请输入用户名',
|
||||
clusterType: '集群类型',
|
||||
connectKeyList: '集群IP列表',
|
||||
connectKeyListPlaceholder: '请输入集群IP',
|
||||
|
@ -45,8 +38,10 @@ const I18N_CONF = {
|
|||
serviceNamePlaceholder: '请输入服务名',
|
||||
search: '搜索',
|
||||
addSync: '新增同步',
|
||||
nameSpace: '命名空间',
|
||||
serviceName: '服务名',
|
||||
groupName: '分组',
|
||||
nameSpace: '命名空间',
|
||||
sourceCluster: '源集群',
|
||||
destCluster: '目标集群',
|
||||
instancesCount: '实例数',
|
||||
|
@ -63,10 +58,14 @@ const I18N_CONF = {
|
|||
},
|
||||
AddSyncDialog: {
|
||||
title: '新增同步',
|
||||
nameSpace: '命名空间',
|
||||
nameSpacePlaceholder: '请输入命名空间ID',
|
||||
serviceName: '服务名',
|
||||
serviceNamePlaceholder: '请输入服务名',
|
||||
groupName: '分组名',
|
||||
groupNamePlaceholder: '请输入分组名',
|
||||
nameSpace: '命名空间',
|
||||
nameSpacePlaceholder: '请输入命名空间',
|
||||
sourceCluster: '源集群',
|
||||
destCluster: '目标集群',
|
||||
version: '版本',
|
||||
|
|
File diff suppressed because one or more lines are too long
|
@ -7,8 +7,8 @@
|
|||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||||
<title>Nacos-Sync</title>
|
||||
<link rel="shortcut icon" href="//www.aliyun.com/favicon.ico" type="image/x-icon">
|
||||
<link href="./css/main.e2917886.css" rel="stylesheet"></head>
|
||||
<link href="./css/main.9093077e.css" rel="stylesheet"></head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="text/javascript" src="./js/main.b73436b5.js"></script></body>
|
||||
<script type="text/javascript" src="./js/main.cf091959.js"></script></body>
|
||||
</html>
|
||||
|
|
File diff suppressed because one or more lines are too long
|
@ -8,10 +8,6 @@ CREATE TABLE `cluster` (
|
|||
`cluster_name` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`cluster_type` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`connect_key_list` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`user_name` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`password` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`namespace` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`cluster_level` int default 0,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
/******************************************/
|
||||
|
@ -41,6 +37,5 @@ CREATE TABLE `task` (
|
|||
`task_status` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`version` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`worker_ip` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,
|
||||
`status` int default null ,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
|
|
|
@ -13,14 +13,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
pid=`ps ax | grep -i 'nacos-sync' |grep java | grep -v grep | awk '{print $1}'`
|
||||
pid=`ps ax | grep -i 'nacosSync' |grep java | grep -v grep | awk '{print $1}'`
|
||||
if [ -z "$pid" ] ; then
|
||||
echo "no nacos-sync running."
|
||||
echo "No nacosSync running."
|
||||
exit -1;
|
||||
fi
|
||||
|
||||
echo "the nacos-sync(${pid}) is running..."
|
||||
echo "The nacosSync(${pid}) is running..."
|
||||
|
||||
kill ${pid}
|
||||
|
||||
echo "Send shutdown request to nacos-sync(${pid}) OK"
|
||||
echo "Send shutdown request to nacosSync(${pid}) OK"
|
|
@ -1 +0,0 @@
|
|||
java -jar ../nacos-sync-server.jar -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m --spring.config.location=../conf/application.properties
|
|
@ -47,24 +47,24 @@ export BASE_DIR=`cd $(dirname $0)/..; pwd`
|
|||
|
||||
|
||||
JAVA_OPT="${JAVA_OPT} -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m"
|
||||
JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/nacos-sync-java-heapdump.hprof"
|
||||
JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/nacossync_java_heapdump.hprof"
|
||||
JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages"
|
||||
JAVA_OPT="${JAVA_OPT} -Dspring.config.location=${BASE_DIR}/conf/application.properties"
|
||||
JAVA_OPT="${JAVA_OPT} -DnacosSync.home=${BASE_DIR}"
|
||||
|
||||
JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p')
|
||||
if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
|
||||
JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/nacos-sync-gc.log:time,tags:filecount=10,filesize=102400"
|
||||
JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/nacossync_gc.log:time,tags:filecount=10,filesize=102400"
|
||||
else
|
||||
JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/nacos-sync-gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
|
||||
JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/nacossync_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
|
||||
fi
|
||||
|
||||
JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/nacos-sync-server.jar"
|
||||
JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/nacosSync-server.jar"
|
||||
JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/logback-spring.xml"
|
||||
|
||||
echo "JAVA_HOME:"$JAVA_HOME
|
||||
echo "BASE_DIR:"$BASE_DIR
|
||||
echo "JAVA:"$JAVA
|
||||
echo "=============JAVA_HOME:"$JAVA_HOME
|
||||
echo "=============BASE_DIR:"$BASE_DIR
|
||||
echo "=============JAVA:"$JAVA
|
||||
|
||||
|
||||
if [ ! -d "${BASE_DIR}/logs" ]; then
|
||||
|
@ -78,9 +78,9 @@ usage(){
|
|||
|
||||
start(){
|
||||
|
||||
echo "$JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/nacos-sync-start.out 2>&1 &
|
||||
nohup $JAVA ${JAVA_OPT} >> ${BASE_DIR}/logs/nacos-sync-start.out 2>&1 &
|
||||
echo "nacos-sync is starting,you can check the ${BASE_DIR}/logs/nacos-sync-start.out"
|
||||
echo "$JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/nacossync_start.out 2>&1 &
|
||||
nohup $JAVA ${JAVA_OPT} >> ${BASE_DIR}/logs/nacossync_start.out 2>&1 &
|
||||
echo "nacossync is starting,you can check the ${BASE_DIR}/logs/nacossync_start.out"
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -5,14 +5,14 @@
|
|||
<parent>
|
||||
<artifactId>nacossync-parent</artifactId>
|
||||
<groupId>com.alibaba.nacossync</groupId>
|
||||
<version>${revision}</version>
|
||||
<version>0.4.5</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<packaging>pom</packaging>
|
||||
<artifactId>nacossync-distribution</artifactId>
|
||||
|
||||
<build>
|
||||
<finalName>nacos-sync-${project.version}</finalName>
|
||||
<finalName>nacosSync.${parent.version}</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
|
@ -35,4 +35,4 @@
|
|||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
</project>
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
<id>bin</id>
|
||||
<includeBaseDirectory>true</includeBaseDirectory>
|
||||
<!-- file name after unzip -->
|
||||
<baseDirectory>nacos-sync</baseDirectory>
|
||||
<baseDirectory>nacosSync</baseDirectory>
|
||||
<formats>
|
||||
<format>dir</format>
|
||||
<format>tar.gz</format>
|
||||
|
@ -60,9 +60,9 @@
|
|||
<destName>NOTICE</destName>
|
||||
</file>
|
||||
<file>
|
||||
<source>../nacossync-worker/target/nacos-sync-server-${project.version}.jar</source>
|
||||
<source>../nacossync-worker/target/nacosSync-server.${parent.version}.jar</source>
|
||||
<outputDirectory></outputDirectory>
|
||||
<destName>nacos-sync-server.jar</destName>
|
||||
<destName>nacosSync-server.jar</destName>
|
||||
</file>
|
||||
</files>
|
||||
</assembly>
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
<parent>
|
||||
<artifactId>nacossync-parent</artifactId>
|
||||
<groupId>com.alibaba.nacossync</groupId>
|
||||
<version>${revision}</version>
|
||||
<version>0.4.5</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
@ -25,19 +25,27 @@
|
|||
|
||||
<artifactId>nacossync-test</artifactId>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
<artifactId>reactor-core</artifactId>
|
||||
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<groupId>com.alibaba.nacossync</groupId>
|
||||
<artifactId>nacossync-worker</artifactId>
|
||||
<version>0.4.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-test</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
|
@ -49,8 +57,10 @@
|
|||
<dependency>
|
||||
<groupId>com.ning</groupId>
|
||||
<artifactId>async-http-client</artifactId>
|
||||
<version>1.7.17</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
@ -58,6 +68,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.9</version>
|
||||
<configuration>
|
||||
<parallel>methods</parallel>
|
||||
<useUnlimitedThreads>true</useUnlimitedThreads>
|
||||
|
@ -65,4 +76,6 @@
|
|||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
|
||||
</project>
|
||||
|
|
|
@ -16,11 +16,29 @@
|
|||
<parent>
|
||||
<artifactId>nacossync-parent</artifactId>
|
||||
<groupId>com.alibaba.nacossync</groupId>
|
||||
<version>${revision}</version>
|
||||
<version>0.4.5</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>nacossync-worker</artifactId>
|
||||
|
||||
<version>0.4.5</version>
|
||||
<properties>
|
||||
<zookeeper.version>3.4.9</zookeeper.version>
|
||||
<curator.version>4.1.0</curator.version>
|
||||
<cloud.version>Finchley.SR2</cloud.version>
|
||||
<mockito.version>1.10.19</mockito.version>
|
||||
<nacos.client.verison>1.3.1</nacos.client.verison>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dependencies</artifactId>
|
||||
<version>${cloud.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
@ -34,6 +52,10 @@
|
|||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
|
@ -42,12 +64,12 @@
|
|||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-all</artifactId>
|
||||
<version>${mockito.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<!-- 默认使用HikariCP连接池 -->
|
||||
<dependency>
|
||||
|
@ -60,36 +82,43 @@
|
|||
</dependency>
|
||||
<!-- mysql -->
|
||||
<dependency>
|
||||
<groupId>com.mysql</groupId>
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springdoc</groupId>
|
||||
<artifactId>springdoc-openapi-ui</artifactId>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
<version>1.2.47</version>
|
||||
</dependency>
|
||||
<!-- swagger2 -->
|
||||
<dependency>
|
||||
<groupId>io.springfox</groupId>
|
||||
<artifactId>springfox-swagger2</artifactId>
|
||||
<version>2.6.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<groupId>io.springfox</groupId>
|
||||
<artifactId>springfox-swagger-ui</artifactId>
|
||||
<version>2.6.1</version>
|
||||
</dependency>
|
||||
<!--nacos-->
|
||||
<dependency>
|
||||
<groupId>com.alibaba.nacos</groupId>
|
||||
<artifactId>nacos-client</artifactId>
|
||||
<version>${nacos.client.verison}</version>
|
||||
</dependency>
|
||||
|
||||
<!--zookeeper-->
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<version>${zookeeper.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-recipes</artifactId>
|
||||
<version>${curator.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
|
@ -97,42 +126,22 @@
|
|||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-client</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- eureka -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- consul -->
|
||||
<dependency>
|
||||
<groupId>com.ecwid.consul</groupId>
|
||||
<artifactId>consul-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</dependencies>
|
||||
<build>
|
||||
<finalName>nacos-sync-server-${project.version}</finalName>
|
||||
<finalName>nacosSync-server.${parent.version}</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
@ -161,22 +170,6 @@
|
|||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>17</source>
|
||||
<target>17</target>
|
||||
<annotationProcessorPaths>
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.18.34</version>
|
||||
</path>
|
||||
</annotationProcessorPaths>
|
||||
</configuration>
|
||||
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
|
|
@ -14,32 +14,24 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync;
|
||||
|
||||
import com.alibaba.nacossync.util.BatchTaskExecutor;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
import org.springframework.cloud.netflix.eureka.EurekaClientAutoConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import springfox.documentation.swagger2.annotations.EnableSwagger2;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: SkyWalkerMain.java, v 0.1 2018-09-24 PM12:42 NacosSync Exp $$
|
||||
*/
|
||||
@EnableSwagger2
|
||||
@SpringBootApplication(exclude = EurekaClientAutoConfiguration.class)
|
||||
public class NacosSyncMain {
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
ConfigurableApplicationContext context = SpringApplication.run(NacosSyncMain.class, args);
|
||||
|
||||
// Register shutdown callback using Spring Boot's context lifecycle
|
||||
context.registerShutdownHook();
|
||||
context.addApplicationListener(event -> {
|
||||
if (event instanceof org.springframework.context.event.ContextClosedEvent) {
|
||||
BatchTaskExecutor.shutdown();
|
||||
}
|
||||
});
|
||||
|
||||
SpringApplication.run(NacosSyncMain.class, args);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,7 +14,6 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.api;
|
||||
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
|
@ -45,54 +44,57 @@ import org.springframework.web.bind.annotation.RestController;
|
|||
@Slf4j
|
||||
@RestController
|
||||
public class ClusterApi {
|
||||
|
||||
|
||||
private final ClusterAddProcessor clusterAddProcessor;
|
||||
|
||||
|
||||
private final ClusterDeleteProcessor clusterDeleteProcessor;
|
||||
|
||||
|
||||
private final ClusterDetailQueryProcessor clusterDetailQueryProcessor;
|
||||
|
||||
|
||||
private final ClusterListQueryProcessor clusterListQueryProcessor;
|
||||
|
||||
public ClusterApi(ClusterAddProcessor clusterAddProcessor, ClusterDeleteProcessor clusterDeleteProcessor,
|
||||
ClusterDetailQueryProcessor clusterDetailQueryProcessor,
|
||||
ClusterListQueryProcessor clusterListQueryProcessor) {
|
||||
|
||||
public ClusterApi(
|
||||
ClusterAddProcessor clusterAddProcessor, ClusterDeleteProcessor clusterDeleteProcessor,
|
||||
ClusterDetailQueryProcessor clusterDetailQueryProcessor, ClusterListQueryProcessor clusterListQueryProcessor) {
|
||||
this.clusterAddProcessor = clusterAddProcessor;
|
||||
this.clusterDeleteProcessor = clusterDeleteProcessor;
|
||||
this.clusterDetailQueryProcessor = clusterDetailQueryProcessor;
|
||||
this.clusterListQueryProcessor = clusterListQueryProcessor;
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/cluster/list", method = RequestMethod.GET)
|
||||
public ClusterListQueryResult clusters(ClusterListQueryRequest clusterListQueryRequest) {
|
||||
|
||||
return SkyWalkerTemplate.run(clusterListQueryProcessor, clusterListQueryRequest, new ClusterListQueryResult());
|
||||
|
||||
return SkyWalkerTemplate.run(clusterListQueryProcessor, clusterListQueryRequest,
|
||||
new ClusterListQueryResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/cluster/detail", method = RequestMethod.GET)
|
||||
public ClusterDetailQueryResult getByTaskId(ClusterDetailQueryRequest clusterDetailQueryRequest) {
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(clusterDetailQueryProcessor, clusterDetailQueryRequest,
|
||||
new ClusterDetailQueryResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/cluster/delete", method = RequestMethod.DELETE)
|
||||
public ClusterDeleteResult deleteCluster(ClusterDeleteRequest clusterDeleteRequest) {
|
||||
|
||||
return SkyWalkerTemplate.run(clusterDeleteProcessor, clusterDeleteRequest, new ClusterDeleteResult());
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(clusterDeleteProcessor, clusterDeleteRequest,
|
||||
new ClusterDeleteResult());
|
||||
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/cluster/add", method = RequestMethod.POST)
|
||||
public ClusterAddResult clusterAdd(@RequestBody ClusterAddRequest clusterAddRequest) {
|
||||
|
||||
return SkyWalkerTemplate.run(clusterAddProcessor, clusterAddRequest, new ClusterAddResult());
|
||||
|
||||
return SkyWalkerTemplate
|
||||
.run(clusterAddProcessor, clusterAddRequest, new ClusterAddResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/cluster/types", method = RequestMethod.GET)
|
||||
public ClusterTypeResult getClusterType() {
|
||||
|
||||
|
||||
return new ClusterTypeResult(ClusterTypeEnum.getClusterTypeCodes());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -14,7 +14,6 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.api;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
@ -43,36 +42,34 @@ import com.alibaba.nacossync.template.processor.ConfigQueryProcessor;
|
|||
@Slf4j
|
||||
@RestController
|
||||
public class SystemConfigApi {
|
||||
|
||||
private final ConfigQueryProcessor configQueryProcessor;
|
||||
|
||||
private final ConfigDeleteProcessor configDeleteProcessor;
|
||||
|
||||
private final ConfigAddProcessor configAddProcessor;
|
||||
|
||||
public SystemConfigApi(ConfigQueryProcessor configQueryProcessor, ConfigDeleteProcessor configDeleteProcessor,
|
||||
ConfigAddProcessor configAddProcessor) {
|
||||
this.configQueryProcessor = configQueryProcessor;
|
||||
this.configDeleteProcessor = configDeleteProcessor;
|
||||
this.configAddProcessor = configAddProcessor;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private ConfigQueryProcessor configQueryProcessor;
|
||||
|
||||
@Autowired
|
||||
private ConfigDeleteProcessor configDeleteProcessor;
|
||||
|
||||
@Autowired
|
||||
private ConfigAddProcessor configAddProcessor;
|
||||
|
||||
@RequestMapping(path = "/v1/systemconfig/list", method = RequestMethod.GET)
|
||||
public ConfigQueryResult tasks(ConfigQueryRequest configQueryRequest) {
|
||||
|
||||
return SkyWalkerTemplate.run(configQueryProcessor, configQueryRequest, new ConfigQueryResult());
|
||||
|
||||
return SkyWalkerTemplate.run(configQueryProcessor, configQueryRequest,
|
||||
new ConfigQueryResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/systemconfig/delete", method = RequestMethod.DELETE)
|
||||
public ConfigDeleteResult deleteTask(@RequestBody ConfigDeleteRequest configDeleteRequest) {
|
||||
|
||||
return SkyWalkerTemplate.run(configDeleteProcessor, configDeleteRequest, new ConfigDeleteResult());
|
||||
|
||||
return SkyWalkerTemplate.run(configDeleteProcessor, configDeleteRequest,
|
||||
new ConfigDeleteResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/systemconfig/add", method = RequestMethod.POST)
|
||||
public ConfigAddResult taskAdd(@RequestBody ConfigAddRequest configAddRequest) {
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(configAddProcessor, configAddRequest, new ConfigAddResult());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -10,10 +10,8 @@
|
|||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.api;
|
||||
|
||||
import com.alibaba.nacossync.pojo.request.TaskAddAllRequest;
|
||||
import com.alibaba.nacossync.pojo.request.TaskAddRequest;
|
||||
import com.alibaba.nacossync.pojo.request.TaskDeleteInBatchRequest;
|
||||
import com.alibaba.nacossync.pojo.request.TaskDeleteRequest;
|
||||
|
@ -25,7 +23,6 @@ import com.alibaba.nacossync.pojo.result.TaskAddResult;
|
|||
import com.alibaba.nacossync.pojo.result.TaskDetailQueryResult;
|
||||
import com.alibaba.nacossync.pojo.result.TaskListQueryResult;
|
||||
import com.alibaba.nacossync.template.SkyWalkerTemplate;
|
||||
import com.alibaba.nacossync.template.processor.TaskAddAllProcessor;
|
||||
import com.alibaba.nacossync.template.processor.TaskAddProcessor;
|
||||
import com.alibaba.nacossync.template.processor.TaskDeleteInBatchProcessor;
|
||||
import com.alibaba.nacossync.template.processor.TaskDeleteProcessor;
|
||||
|
@ -45,83 +42,67 @@ import org.springframework.web.bind.annotation.RestController;
|
|||
@Slf4j
|
||||
@RestController
|
||||
public class TaskApi {
|
||||
|
||||
|
||||
private final TaskUpdateProcessor taskUpdateProcessor;
|
||||
|
||||
|
||||
private final TaskAddProcessor taskAddProcessor;
|
||||
|
||||
private final TaskAddAllProcessor taskAddAllProcessor;
|
||||
|
||||
|
||||
private final TaskDeleteProcessor taskDeleteProcessor;
|
||||
|
||||
|
||||
private final TaskDeleteInBatchProcessor taskDeleteInBatchProcessor;
|
||||
|
||||
|
||||
private final TaskListQueryProcessor taskListQueryProcessor;
|
||||
|
||||
|
||||
private final TaskDetailProcessor taskDetailProcessor;
|
||||
|
||||
|
||||
public TaskApi(TaskUpdateProcessor taskUpdateProcessor, TaskAddProcessor taskAddProcessor,
|
||||
TaskAddAllProcessor taskAddAllProcessor, TaskDeleteProcessor taskDeleteProcessor,
|
||||
TaskDeleteInBatchProcessor taskDeleteInBatchProcessor, TaskListQueryProcessor taskListQueryProcessor,
|
||||
TaskDetailProcessor taskDetailProcessor) {
|
||||
TaskDeleteProcessor taskDeleteProcessor, TaskDeleteInBatchProcessor taskDeleteInBatchProcessor,
|
||||
TaskListQueryProcessor taskListQueryProcessor, TaskDetailProcessor taskDetailProcessor) {
|
||||
this.taskUpdateProcessor = taskUpdateProcessor;
|
||||
this.taskAddProcessor = taskAddProcessor;
|
||||
this.taskAddAllProcessor = taskAddAllProcessor;
|
||||
this.taskDeleteProcessor = taskDeleteProcessor;
|
||||
this.taskDeleteInBatchProcessor = taskDeleteInBatchProcessor;
|
||||
this.taskListQueryProcessor = taskListQueryProcessor;
|
||||
this.taskDetailProcessor = taskDetailProcessor;
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/task/list", method = RequestMethod.GET)
|
||||
public TaskListQueryResult tasks(TaskListQueryRequest taskListQueryRequest) {
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(taskListQueryProcessor, taskListQueryRequest, new TaskListQueryResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/task/detail", method = RequestMethod.GET)
|
||||
public TaskDetailQueryResult getByTaskId(TaskDetailQueryRequest taskDetailQueryRequest) {
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(taskDetailProcessor, taskDetailQueryRequest, new TaskDetailQueryResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/task/delete", method = RequestMethod.DELETE)
|
||||
public BaseResult deleteTask(TaskDeleteRequest taskDeleteRequest) {
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(taskDeleteProcessor, taskDeleteRequest, new BaseResult());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @author yongchao9
|
||||
* @param taskBatchDeleteRequest
|
||||
* @return
|
||||
* @author yongchao9
|
||||
*/
|
||||
@RequestMapping(path = "/v1/task/deleteInBatch", method = RequestMethod.DELETE)
|
||||
public BaseResult batchDeleteTask(TaskDeleteInBatchRequest taskBatchDeleteRequest) {
|
||||
return SkyWalkerTemplate.run(taskDeleteInBatchProcessor, taskBatchDeleteRequest, new BaseResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/task/add", method = RequestMethod.POST)
|
||||
public BaseResult taskAdd(@RequestBody TaskAddRequest addTaskRequest) {
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(taskAddProcessor, addTaskRequest, new TaskAddResult());
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO 目前仅支持 Nacos 为源的同步类型,待完善更多类型支持.
|
||||
* <p>
|
||||
* 支持从 sourceCluster 获取所有 service,然后生成同步到 destCluster 的任务。
|
||||
* </p>
|
||||
*/
|
||||
@RequestMapping(path = "/v1/task/addAll", method = RequestMethod.POST)
|
||||
public BaseResult taskAddAll(@RequestBody TaskAddAllRequest addAllRequest) {
|
||||
|
||||
return SkyWalkerTemplate.run(taskAddAllProcessor, addAllRequest, new TaskAddResult());
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(path = "/v1/task/update", method = RequestMethod.POST)
|
||||
public BaseResult updateTask(@RequestBody TaskUpdateRequest taskUpdateRequest) {
|
||||
|
||||
|
||||
return SkyWalkerTemplate.run(taskUpdateProcessor, taskUpdateRequest, new BaseResult());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -16,7 +16,8 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.cache;
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.alibaba.fastjson.TypeReference;
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.exception.SkyWalkerException;
|
||||
|
@ -24,16 +25,14 @@ import com.alibaba.nacossync.pojo.FinishedTask;
|
|||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.SneakyThrows;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import org.jboss.netty.util.internal.ThreadLocalRandom;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -41,29 +40,24 @@ import java.util.concurrent.ThreadLocalRandom;
|
|||
*/
|
||||
@Service
|
||||
public class SkyWalkerCacheServices {
|
||||
|
||||
private static final Map<String, FinishedTask> FINISHED_TASK_MAP = new ConcurrentHashMap<>();
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public SkyWalkerCacheServices(ClusterAccessService clusterAccessService, ObjectMapper objectMapper) {
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private ClusterAccessService clusterAccessService;
|
||||
|
||||
private static Map<String, FinishedTask> finishedTaskMap = new ConcurrentHashMap<>();
|
||||
|
||||
public String getClusterConnectKey(String clusterId) {
|
||||
List<String> allClusterConnectKey = getAllClusterConnectKey(clusterId);
|
||||
|
||||
return allClusterConnectKey.get(ThreadLocalRandom.current().nextInt(allClusterConnectKey.size()));
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public List<String> getAllClusterConnectKey(String clusterId) {
|
||||
ClusterDO clusterDO = clusterAccessService.findByClusterId(clusterId);
|
||||
|
||||
List<String> connectKeyList = objectMapper.readerForListOf(String.class)
|
||||
.readValue(clusterDO.getConnectKeyList());
|
||||
List<String> connectKeyList = JSONObject.parseObject(clusterDO.getConnectKeyList(),
|
||||
new TypeReference<List<String>>() {
|
||||
});
|
||||
|
||||
if (CollectionUtils.isEmpty(connectKeyList)) {
|
||||
throw new SkyWalkerException("getClusterConnectKey empty, clusterId:" + clusterId);
|
||||
|
@ -85,32 +79,23 @@ public class SkyWalkerCacheServices {
|
|||
FinishedTask finishedTask = new FinishedTask();
|
||||
finishedTask.setOperationId(operationId);
|
||||
|
||||
FINISHED_TASK_MAP.put(operationId, finishedTask);
|
||||
finishedTaskMap.put(operationId, finishedTask);
|
||||
}
|
||||
|
||||
public FinishedTask getFinishedTask(TaskDO taskDO) {
|
||||
|
||||
String operationId = SkyWalkerUtil.getOperationId(taskDO);
|
||||
|
||||
if (!StringUtils.hasLength(operationId)) {
|
||||
if (StringUtils.isEmpty(operationId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return FINISHED_TASK_MAP.get(operationId);
|
||||
}
|
||||
|
||||
|
||||
public void removeFinishedTask(String operationId) {
|
||||
if (!StringUtils.hasLength(operationId)) {
|
||||
return;
|
||||
}
|
||||
FINISHED_TASK_MAP.remove(operationId);
|
||||
return finishedTaskMap.get(operationId);
|
||||
}
|
||||
|
||||
public Map<String, FinishedTask> getFinishedTaskMap() {
|
||||
|
||||
return FINISHED_TASK_MAP;
|
||||
return finishedTaskMap;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -12,8 +12,6 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.constant;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
@ -21,7 +19,6 @@ import java.util.List;
|
|||
* @author NacosSync
|
||||
* @version $Id: ClusterTypeEnum.java, v 0.1 2018-09-25 下午4:38 NacosSync Exp $$
|
||||
*/
|
||||
@Getter
|
||||
public enum ClusterTypeEnum {
|
||||
|
||||
CS("CS", "configserver集群"),
|
||||
|
@ -35,16 +32,18 @@ public enum ClusterTypeEnum {
|
|||
ZK("ZK", "zookeeper集群");
|
||||
|
||||
|
||||
private final String code;
|
||||
private String code;
|
||||
|
||||
private String desc;
|
||||
|
||||
ClusterTypeEnum(String code, String desc) {
|
||||
this.code = code;
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
public static List<String> getClusterTypeCodes() {
|
||||
|
||||
List<String> list = new ArrayList<>();
|
||||
List<String> list = new ArrayList<String>();
|
||||
|
||||
for (ClusterTypeEnum clusterTypeEnum : ClusterTypeEnum.values()) {
|
||||
list.add(clusterTypeEnum.getCode());
|
||||
|
@ -52,7 +51,42 @@ public enum ClusterTypeEnum {
|
|||
return list;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Getter method for property <tt>code</tt>.
|
||||
*
|
||||
* @return property value of code
|
||||
*/
|
||||
public String getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setter method for property <tt>code </tt>.
|
||||
*
|
||||
* @param code value to be assigned to property code
|
||||
*/
|
||||
public void setCode(String code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
/**
|
||||
* Getter method for property <tt>desc</tt>.
|
||||
*
|
||||
* @return property value of desc
|
||||
*/
|
||||
public String getDesc() {
|
||||
return desc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setter method for property <tt>desc </tt>.
|
||||
*
|
||||
* @param desc value to be assigned to property desc
|
||||
*/
|
||||
public void setDesc(String desc) {
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
public static boolean contains(String clusterType) {
|
||||
|
||||
for (ClusterTypeEnum clusterTypeEnum : ClusterTypeEnum.values()) {
|
||||
|
|
|
@ -3,13 +3,10 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.constant;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: MetricsStatisticsType.java, v 0.1 2019年02月28日 下午2:17 NacosSync Exp $
|
||||
*/
|
||||
@Getter
|
||||
public enum MetricsStatisticsType {
|
||||
|
||||
CACHE_SIZE("nacosSync.finished.taskMap.cacheSize", "任务执行完成缓存列表数"),
|
||||
|
@ -31,10 +28,15 @@ public enum MetricsStatisticsType {
|
|||
/**
|
||||
* metricsName
|
||||
*/
|
||||
private final String metricsName;
|
||||
private String metricsName;
|
||||
private String desc;
|
||||
|
||||
MetricsStatisticsType(String code, String desc) {
|
||||
this.metricsName = code;
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
public String getMetricsName() {
|
||||
return metricsName;
|
||||
}
|
||||
|
||||
}
|
|
@ -16,23 +16,46 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.constant;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: ResultCodeEnum.java, v 0.1 2018-09-25 PM4:38 NacosSync Exp $$
|
||||
*/
|
||||
@Getter
|
||||
public enum ResultCodeEnum {
|
||||
|
||||
SUCCESS("SUCCESS", "请求成功", "请求成功"),
|
||||
SYSTEM_ERROR("SYSTEM_ERROR", "系统异常", "系统异常");
|
||||
|
||||
private final String code;
|
||||
|
||||
private String code;
|
||||
private String errorMessage;
|
||||
private String detail;
|
||||
|
||||
ResultCodeEnum(String code, String errorMessage, String detail) {
|
||||
this.code = code;
|
||||
this.errorMessage = errorMessage;
|
||||
this.detail = detail;
|
||||
}
|
||||
|
||||
public String getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public void setCode(String code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public String getErrorMessage() {
|
||||
return errorMessage;
|
||||
}
|
||||
|
||||
public void setErrorMessage(String errorMessage) {
|
||||
this.errorMessage = errorMessage;
|
||||
}
|
||||
|
||||
public String getDetail() {
|
||||
return detail;
|
||||
}
|
||||
|
||||
public void setDetail(String detail) {
|
||||
this.detail = detail;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
package com.alibaba.nacossync.constant;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/11/18.
|
||||
*/
|
||||
public enum ShardingLogTypeEnum {
|
||||
|
||||
ADD("add", "新增"),
|
||||
|
||||
DELETE("DELETE", "删除");
|
||||
|
||||
private String type;
|
||||
private String desc;
|
||||
|
||||
ShardingLogTypeEnum(String type, String desc) {
|
||||
this.type = type;
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public String getDesc() {
|
||||
return desc;
|
||||
}
|
||||
|
||||
public void setDesc(String desc) {
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
public static boolean contains(String type) {
|
||||
|
||||
for (ShardingLogTypeEnum shardingLogTypeEnum : ShardingLogTypeEnum.values()) {
|
||||
|
||||
if (shardingLogTypeEnum.getType().equals(type)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
|
@ -24,18 +24,11 @@ public class SkyWalkerConstants {
|
|||
|
||||
public static final String UNDERLINE = "_";
|
||||
|
||||
public static final String DEST_CLUSTER_ID_KEY = "destClusterId";
|
||||
public static final String DEST_CLUSTERID_KEY = "destClusterId";
|
||||
public static final String GROUP_NAME = "groupName";
|
||||
public static final String SYNC_SOURCE_KEY = "syncSource";
|
||||
public static final String SOURCE_CLUSTER_ID_KEY = "sourceClusterId";
|
||||
public static final String SOURCE_CLUSTERID_KEY = "sourceClusterId";
|
||||
public static final String MANAGEMENT_PORT_KEY="management.port";
|
||||
public static final String MANAGEMENT_CONTEXT_PATH_KEY="management.context-path";
|
||||
|
||||
public static final String SERVICE_NAME_PARAM="serviceNameParam";
|
||||
public static final String GROUP_NAME_PARAM="groupNameParam";
|
||||
public static final String PAGE_NO="pageNo";
|
||||
public static final String PAGE_SIZE="pageSize";
|
||||
public static final String SYNC_INSTANCE_TAG="sync.instance.tag";
|
||||
public static final String NACOS_ALL_SERVICE_NAME = "ALL";
|
||||
|
||||
}
|
||||
|
|
|
@ -16,8 +16,6 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.constant;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: TaskStatusEnum.java, v 0.1 2018-09-26 上午2:38 NacosSync Exp $$
|
||||
|
@ -32,18 +30,51 @@ public enum TaskStatusEnum {
|
|||
* delete the task
|
||||
*/
|
||||
DELETE("DELETE", "任务需要被删除");
|
||||
|
||||
|
||||
@Getter
|
||||
private final String code;
|
||||
private final String desc;
|
||||
private String code;
|
||||
private String desc;
|
||||
|
||||
TaskStatusEnum(String code, String desc) {
|
||||
this.code = code;
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Getter method for property <tt>code</tt>.
|
||||
*
|
||||
* @return property value of code
|
||||
*/
|
||||
public String getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setter method for property <tt>code </tt>.
|
||||
*
|
||||
* @param code value to be assigned to property code
|
||||
*/
|
||||
public void setCode(String code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
/**
|
||||
* Getter method for property <tt>desc</tt>.
|
||||
*
|
||||
* @return property value of desc
|
||||
*/
|
||||
public String getDesc() {
|
||||
return desc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setter method for property <tt>desc </tt>.
|
||||
*
|
||||
* @param desc value to be assigned to property desc
|
||||
*/
|
||||
public void setDesc(String desc) {
|
||||
this.desc = desc;
|
||||
}
|
||||
|
||||
public static boolean contains(String code) {
|
||||
|
||||
for (TaskStatusEnum taskStatusEnum : TaskStatusEnum.values()) {
|
||||
|
|
|
@ -16,10 +16,10 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.dao;
|
||||
|
||||
import com.alibaba.nacossync.dao.repository.ClusterRepository;
|
||||
import com.alibaba.nacossync.pojo.QueryCondition;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
|
@ -27,6 +27,9 @@ import org.springframework.data.domain.Sort;
|
|||
import org.springframework.data.jpa.domain.Specification;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.dao.repository.ClusterRepository;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
|
||||
import javax.persistence.criteria.CriteriaBuilder;
|
||||
import javax.persistence.criteria.Predicate;
|
||||
import javax.persistence.criteria.Root;
|
||||
|
@ -41,12 +44,9 @@ import java.util.List;
|
|||
@Slf4j
|
||||
public class ClusterAccessService implements PageQueryService<ClusterDO> {
|
||||
|
||||
private final ClusterRepository clusterRepository;
|
||||
|
||||
public ClusterAccessService(ClusterRepository clusterRepository) {
|
||||
this.clusterRepository = clusterRepository;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private ClusterRepository clusterRepository;
|
||||
|
||||
public ClusterDO insert(ClusterDO clusterDO) {
|
||||
|
||||
return clusterRepository.save(clusterDO);
|
||||
|
@ -103,12 +103,4 @@ public class ClusterAccessService implements PageQueryService<ClusterDO> {
|
|||
predicates.add(criteriaBuilder.like(root.get("clusterName"), "%" + queryCondition.getServiceName() + "%"));
|
||||
return predicates;
|
||||
}
|
||||
|
||||
public int findClusterLevel(String sourceClusterId){
|
||||
ClusterDO clusterDO = clusterRepository.findByClusterId(sourceClusterId);
|
||||
if (clusterDO != null) {
|
||||
return clusterDO.getClusterLevel();
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,13 +14,10 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.dao;
|
||||
|
||||
import com.alibaba.nacossync.constant.SkyWalkerConstants;
|
||||
import com.alibaba.nacossync.dao.repository.TaskRepository;
|
||||
import com.alibaba.nacossync.pojo.QueryCondition;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
|
@ -28,6 +25,9 @@ import org.springframework.data.domain.Sort;
|
|||
import org.springframework.data.jpa.domain.Specification;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.dao.repository.TaskRepository;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
|
||||
import javax.persistence.criteria.CriteriaBuilder;
|
||||
import javax.persistence.criteria.Predicate;
|
||||
import javax.persistence.criteria.Root;
|
||||
|
@ -40,94 +40,79 @@ import java.util.List;
|
|||
*/
|
||||
@Service
|
||||
public class TaskAccessService implements PageQueryService<TaskDO> {
|
||||
|
||||
private final TaskRepository taskRepository;
|
||||
|
||||
public TaskAccessService(TaskRepository taskRepository) {
|
||||
this.taskRepository = taskRepository;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private TaskRepository taskRepository;
|
||||
|
||||
public TaskDO findByTaskId(String taskId) {
|
||||
|
||||
|
||||
return taskRepository.findByTaskId(taskId);
|
||||
}
|
||||
|
||||
|
||||
public void deleteTaskById(String taskId) {
|
||||
taskRepository.deleteByTaskId(taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* batch delete tasks by taskIds
|
||||
*
|
||||
* @param taskIds
|
||||
* @author yongchao9
|
||||
* @param taskIds
|
||||
*/
|
||||
public void deleteTaskInBatch(List<String> taskIds) {
|
||||
List<TaskDO> tds = taskRepository.findAllByTaskIdIn(taskIds);
|
||||
taskRepository.deleteAllInBatch(tds);
|
||||
List<TaskDO> tds=taskRepository.findAllByTaskIdIn(taskIds);
|
||||
taskRepository.deleteInBatch(tds);
|
||||
}
|
||||
|
||||
|
||||
public Iterable<TaskDO> findAll() {
|
||||
|
||||
|
||||
return taskRepository.findAll();
|
||||
}
|
||||
|
||||
|
||||
public void addTask(TaskDO taskDO) {
|
||||
|
||||
|
||||
taskRepository.save(taskDO);
|
||||
|
||||
|
||||
}
|
||||
|
||||
public int countByDestClusterIdOrSourceClusterId(String destClusterId, String sourceClusterId) {
|
||||
return taskRepository.countByDestClusterIdOrSourceClusterId(destClusterId, sourceClusterId);
|
||||
}
|
||||
|
||||
|
||||
private Predicate getPredicate(CriteriaBuilder criteriaBuilder, List<Predicate> predicates) {
|
||||
Predicate[] p = new Predicate[predicates.size()];
|
||||
return criteriaBuilder.and(predicates.toArray(p));
|
||||
}
|
||||
|
||||
private List<Predicate> getPredicates(Root<TaskDO> root, CriteriaBuilder criteriaBuilder,
|
||||
QueryCondition queryCondition) {
|
||||
|
||||
|
||||
private List<Predicate> getPredicates(Root<TaskDO> root, CriteriaBuilder criteriaBuilder, QueryCondition queryCondition) {
|
||||
|
||||
List<Predicate> predicates = new ArrayList<>();
|
||||
predicates.add(criteriaBuilder.like(root.get("serviceName"), "%" + queryCondition.getServiceName() + "%"));
|
||||
|
||||
|
||||
return predicates;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Page<TaskDO> findPageNoCriteria(Integer pageNum, Integer size) {
|
||||
|
||||
|
||||
Pageable pageable = PageRequest.of(pageNum, size, Sort.Direction.DESC, "id");
|
||||
|
||||
|
||||
return taskRepository.findAll(pageable);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Page<TaskDO> findPageCriteria(Integer pageNum, Integer size, QueryCondition queryCondition) {
|
||||
|
||||
|
||||
Pageable pageable = PageRequest.of(pageNum, size, Sort.Direction.DESC, "id");
|
||||
|
||||
|
||||
return getTaskDOS(queryCondition, pageable);
|
||||
}
|
||||
|
||||
|
||||
private Page<TaskDO> getTaskDOS(QueryCondition queryCondition, Pageable pageable) {
|
||||
return taskRepository.findAll((Specification<TaskDO>) (root, criteriaQuery, criteriaBuilder) -> {
|
||||
|
||||
List<Predicate> predicates = getPredicates(root, criteriaBuilder, queryCondition);
|
||||
|
||||
return getPredicate(criteriaBuilder, predicates);
|
||||
|
||||
}, pageable);
|
||||
return taskRepository.findAll(
|
||||
(Specification<TaskDO>) (root, criteriaQuery, criteriaBuilder) -> {
|
||||
|
||||
List<Predicate> predicates = getPredicates(root,
|
||||
criteriaBuilder, queryCondition);
|
||||
|
||||
return getPredicate(criteriaBuilder, predicates);
|
||||
|
||||
}, pageable);
|
||||
}
|
||||
|
||||
public List<TaskDO> findAllByServiceNameEqualAll() {
|
||||
return taskRepository.findAllByServiceNameEqualsIgnoreCase(SkyWalkerConstants.NACOS_ALL_SERVICE_NAME);
|
||||
}
|
||||
|
||||
public List<TaskDO> findAllByServiceNameNotEqualAll() {
|
||||
return taskRepository.findAllByServiceNameNotIgnoreCase(SkyWalkerConstants.NACOS_ALL_SERVICE_NAME);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -16,13 +16,13 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.dao.repository;
|
||||
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
|
||||
import org.springframework.data.repository.CrudRepository;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -34,6 +34,6 @@ public interface ClusterRepository extends CrudRepository<ClusterDO, Integer>, J
|
|||
ClusterDO findByClusterId(String clusterId);
|
||||
|
||||
@Transactional
|
||||
void deleteByClusterId(String clusterId);
|
||||
int deleteByClusterId(String clusterId);
|
||||
|
||||
}
|
||||
|
|
|
@ -16,13 +16,15 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.dao.repository;
|
||||
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import java.util.List;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
|
||||
import org.springframework.data.repository.CrudRepository;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
import java.util.List;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -34,15 +36,10 @@ public interface TaskRepository extends CrudRepository<TaskDO, Integer>, JpaRepo
|
|||
TaskDO findByTaskId(String taskId);
|
||||
|
||||
@Transactional
|
||||
void deleteByTaskId(String taskId);
|
||||
int deleteByTaskId(String taskId);
|
||||
|
||||
List<TaskDO> findAllByTaskIdIn(List<String> taskIds);
|
||||
/**
|
||||
* query service is all,use ns leven sync data
|
||||
*/
|
||||
List<TaskDO> findAllByServiceNameEqualsIgnoreCase(String serviceName);
|
||||
List<TaskDO> findAllByServiceNameNotIgnoreCase(String serviceName);
|
||||
|
||||
int countByDestClusterIdOrSourceClusterId(String destClusterId,String sourceClusterId);
|
||||
List<TaskDO> getAllByWorkerIp(String workerIp);
|
||||
|
||||
}
|
||||
|
|
|
@ -1,13 +0,0 @@
|
|||
package com.alibaba.nacossync.event;
|
||||
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class DeleteAllSubTaskEvent {
|
||||
public DeleteAllSubTaskEvent(TaskDO taskDO) {
|
||||
this.taskDO = taskDO;
|
||||
}
|
||||
|
||||
private final TaskDO taskDO;
|
||||
}
|
|
@ -14,22 +14,23 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.event.listener;
|
||||
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import com.alibaba.nacossync.constant.MetricsStatisticsType;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import com.alibaba.nacossync.event.DeleteTaskEvent;
|
||||
import com.alibaba.nacossync.event.SyncTaskEvent;
|
||||
import com.alibaba.nacossync.extension.SyncManagerService;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import com.google.common.eventbus.Subscribe;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -38,59 +39,50 @@ import javax.annotation.PostConstruct;
|
|||
@Slf4j
|
||||
@Service
|
||||
public class EventListener {
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
|
||||
private final SyncManagerService syncManagerService;
|
||||
|
||||
private final EventBus eventBus;
|
||||
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
public EventListener(MetricsManager metricsManager, SyncManagerService syncManagerService, EventBus eventBus,
|
||||
SkyWalkerCacheServices skyWalkerCacheServices) {
|
||||
this.metricsManager = metricsManager;
|
||||
this.syncManagerService = syncManagerService;
|
||||
this.eventBus = eventBus;
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
@Autowired
|
||||
private SyncManagerService syncManagerService;
|
||||
|
||||
@Autowired
|
||||
private EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
private SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
@PostConstruct
|
||||
public void register() {
|
||||
eventBus.register(this);
|
||||
}
|
||||
|
||||
|
||||
@Subscribe
|
||||
public void sync(SyncTaskEvent syncTaskEvent) {
|
||||
|
||||
public void listenerSyncTaskEvent(SyncTaskEvent syncTaskEvent) {
|
||||
|
||||
try {
|
||||
|
||||
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||
if (syncManagerService.sync(syncTaskEvent.getTaskDO(), null)) {
|
||||
skyWalkerCacheServices.addFinishedTask(syncTaskEvent.getTaskDO());
|
||||
metricsManager.record(MetricsStatisticsType.SYNC_TASK_RT, stopwatch.elapsed().toMillis());
|
||||
} else {
|
||||
log.warn("syncTaskEvent process error");
|
||||
}
|
||||
long start = System.currentTimeMillis();
|
||||
syncManagerService.sync(syncTaskEvent.getTaskDO());
|
||||
skyWalkerCacheServices.addFinishedTask(syncTaskEvent.getTaskDO());
|
||||
metricsManager.record(MetricsStatisticsType.SYNC_TASK_RT, System.currentTimeMillis() - start);
|
||||
} catch (Exception e) {
|
||||
log.warn("syncTaskEvent process error", e);
|
||||
log.warn("listenerSyncTaskEvent process error", e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Subscribe
|
||||
public void delete(DeleteTaskEvent deleteTaskEvent) {
|
||||
|
||||
public void listenerDeleteTaskEvent(DeleteTaskEvent deleteTaskEvent) {
|
||||
|
||||
try {
|
||||
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||
if (syncManagerService.delete(deleteTaskEvent.getTaskDO())) {
|
||||
skyWalkerCacheServices.removeFinishedTask(deleteTaskEvent.getTaskDO().getOperationId());
|
||||
metricsManager.record(MetricsStatisticsType.DELETE_TASK_RT, stopwatch.elapsed().toMillis());
|
||||
} else {
|
||||
log.warn("deleteTaskEvent delete failure");
|
||||
}
|
||||
long start = System.currentTimeMillis();
|
||||
syncManagerService.delete(deleteTaskEvent.getTaskDO());
|
||||
skyWalkerCacheServices.addFinishedTask(deleteTaskEvent.getTaskDO());
|
||||
metricsManager.record(MetricsStatisticsType.DELETE_TASK_RT, System.currentTimeMillis() - start);
|
||||
} catch (Exception e) {
|
||||
log.warn("deleteTaskEvent delete failure.", e);
|
||||
log.warn("listenerDeleteTaskEvent process error", e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -17,7 +17,6 @@ import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
|||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.extension.annotation.NacosSyncService;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.StringUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
|
@ -39,34 +38,31 @@ public class SyncManagerService implements InitializingBean, ApplicationContextA
|
|||
|
||||
protected final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
private final ConcurrentHashMap<String, SyncService> syncServiceMap = new ConcurrentHashMap<String, SyncService>();
|
||||
private ConcurrentHashMap<String, SyncService> syncServiceMap = new ConcurrentHashMap<String, SyncService>();
|
||||
|
||||
private ApplicationContext applicationContext;
|
||||
|
||||
public SyncManagerService(
|
||||
SkyWalkerCacheServices skyWalkerCacheServices) {
|
||||
SkyWalkerCacheServices skyWalkerCacheServices) {
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
}
|
||||
|
||||
public boolean delete(TaskDO taskDO) throws NacosException {
|
||||
|
||||
return getSyncService(taskDO.getSourceClusterId(), taskDO.getDestClusterId()).delete(taskDO);
|
||||
|
||||
}
|
||||
|
||||
public boolean sync(TaskDO taskDO, Integer index) {
|
||||
|
||||
return getSyncService(taskDO.getSourceClusterId(), taskDO.getDestClusterId()).sync(taskDO, index);
|
||||
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
return getSyncService(taskDO.getSourceClusterId(), taskDO.getDestClusterId()).sync(taskDO);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() {
|
||||
this.applicationContext.getBeansWithAnnotation(NacosSyncService.class).forEach((key, value) -> {
|
||||
this.applicationContext.getBeansOfType(SyncService.class).forEach((key, value) -> {
|
||||
NacosSyncService nacosSyncService = value.getClass().getAnnotation(NacosSyncService.class);
|
||||
ClusterTypeEnum sourceCluster = nacosSyncService.sourceCluster();
|
||||
ClusterTypeEnum destinationCluster = nacosSyncService.destinationCluster();
|
||||
syncServiceMap.put(generateSyncKey(sourceCluster, destinationCluster), (SyncService) value);
|
||||
syncServiceMap.put(generateSyncKey(sourceCluster, destinationCluster), value);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -76,13 +72,10 @@ public class SyncManagerService implements InitializingBean, ApplicationContextA
|
|||
}
|
||||
|
||||
public SyncService getSyncService(String sourceClusterId, String destClusterId) {
|
||||
if (StringUtils.isEmpty(sourceClusterId) || StringUtils.isEmpty(destClusterId)) {
|
||||
throw new IllegalArgumentException("Source cluster id and destination cluster id must not be null or empty");
|
||||
}
|
||||
|
||||
ClusterTypeEnum sourceClusterType = this.skyWalkerCacheServices.getClusterType(sourceClusterId);
|
||||
ClusterTypeEnum destClusterType = this.skyWalkerCacheServices.getClusterType(destClusterId);
|
||||
|
||||
return syncServiceMap.get(generateSyncKey(sourceClusterType, destClusterType));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -14,9 +14,8 @@ package com.alibaba.nacossync.extension;
|
|||
|
||||
import com.alibaba.nacossync.constant.SkyWalkerConstants;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.Map;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -31,22 +30,20 @@ public interface SyncService {
|
|||
* @return
|
||||
*/
|
||||
boolean delete(TaskDO taskDO);
|
||||
|
||||
|
||||
/**
|
||||
* execute sync
|
||||
*
|
||||
* @param taskDO
|
||||
* @param index
|
||||
* @return
|
||||
*/
|
||||
boolean sync(TaskDO taskDO, Integer index);
|
||||
boolean sync(TaskDO taskDO);
|
||||
|
||||
/**
|
||||
* Determines that the current instance data is from another source cluster
|
||||
*/
|
||||
default boolean needSync(Map<String, String> sourceMetaData) {
|
||||
boolean syncTag = StringUtils.isBlank(sourceMetaData.get(SkyWalkerConstants.SYNC_INSTANCE_TAG));
|
||||
boolean blank = StringUtils.isBlank(sourceMetaData.get(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY));
|
||||
return syncTag && blank;
|
||||
return StringUtils.isBlank(sourceMetaData.get(SkyWalkerConstants.SOURCE_CLUSTERID_KEY));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -54,7 +51,7 @@ public interface SyncService {
|
|||
* cluster ID of the task
|
||||
*/
|
||||
default boolean needDelete(Map<String, String> destMetaData, TaskDO taskDO) {
|
||||
return StringUtils.equals(destMetaData.get(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY),
|
||||
return StringUtils.equals(destMetaData.get(SkyWalkerConstants.SOURCE_CLUSTERID_KEY),
|
||||
taskDO.getSourceClusterId());
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
package com.alibaba.nacossync.extension.client;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class InstanceQueryModel {
|
||||
|
||||
private String sourceClusterId;
|
||||
private String destClusterId;
|
||||
private String serviceName;
|
||||
private String groupName;
|
||||
private String version;
|
||||
private int pageNo;
|
||||
private int pageSize;
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package com.alibaba.nacossync.extension.client;
|
||||
|
||||
import com.alibaba.nacossync.pojo.view.TaskModel;
|
||||
import java.util.List;
|
||||
|
||||
public interface SyncQueryClient {
|
||||
|
||||
|
||||
List<TaskModel> getAllInstance(InstanceQueryModel instanceQueryModel);
|
||||
|
||||
}
|
|
@ -0,0 +1,54 @@
|
|||
package com.alibaba.nacossync.extension.client.impl;
|
||||
|
||||
import com.alibaba.nacos.api.exception.NacosException;
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.pojo.ListView;
|
||||
import com.alibaba.nacossync.extension.client.InstanceQueryModel;
|
||||
import com.alibaba.nacossync.extension.client.SyncQueryClient;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.pojo.view.TaskModel;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class NacosSyncQueryClientImpl implements SyncQueryClient {
|
||||
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
|
||||
public NacosSyncQueryClientImpl(
|
||||
NacosServerHolder nacosServerHolder) {
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TaskModel> getAllInstance(InstanceQueryModel instanceQueryModel) {
|
||||
NamingService namingService = nacosServerHolder
|
||||
.get(instanceQueryModel.getSourceClusterId(), instanceQueryModel.getGroupName());
|
||||
try {
|
||||
ListView<String> servicesOfServer = namingService
|
||||
.getServicesOfServer(instanceQueryModel.getPageNo(),
|
||||
instanceQueryModel.getPageSize());
|
||||
return servicesOfServer.getData().stream()
|
||||
.map(serviceName -> buildTaskModel(instanceQueryModel, serviceName))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
} catch (NacosException e) {
|
||||
log.error("When using nacos client failure query tasks", e);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
private TaskModel buildTaskModel(InstanceQueryModel instanceQueryModel, String serviceName) {
|
||||
TaskModel taskModel = new TaskModel();
|
||||
taskModel.setServiceName(serviceName);
|
||||
taskModel.setSourceClusterId(instanceQueryModel.getSourceClusterId());
|
||||
taskModel.setDestClusterId(instanceQueryModel.getDestClusterId());
|
||||
return taskModel;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -29,11 +29,11 @@ import java.util.concurrent.TimeUnit;
|
|||
*/
|
||||
@Slf4j
|
||||
public class EurekaBeatReactor {
|
||||
private final ScheduledExecutorService executorService;
|
||||
private ScheduledExecutorService executorService;
|
||||
|
||||
private static final long CLIENT_BEAT_INTERVAL = 5 * 1000;
|
||||
private volatile long clientBeatInterval = 5 * 1000;
|
||||
private final Map<String, InstanceInfo> eurekaBeat = new ConcurrentHashMap<>();
|
||||
private final EurekaHttpClient eurekaHttpClient;
|
||||
private EurekaHttpClient eurekaHttpClient;
|
||||
|
||||
public EurekaBeatReactor(EurekaHttpClient eurekaHttpClient) {
|
||||
this.eurekaHttpClient = eurekaHttpClient;
|
||||
|
@ -71,7 +71,7 @@ public class EurekaBeatReactor {
|
|||
} catch (Exception e) {
|
||||
log.error("[CLIENT-BEAT] Exception while scheduling beat.", e);
|
||||
} finally {
|
||||
executorService.schedule(this, CLIENT_BEAT_INTERVAL, TimeUnit.MILLISECONDS);
|
||||
executorService.schedule(this, clientBeatInterval, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -26,8 +26,8 @@ import java.util.Objects;
|
|||
* @date 2019-06-26
|
||||
*/
|
||||
public class EurekaNamingService {
|
||||
private final EurekaHttpClient eurekaHttpClient;
|
||||
private final EurekaBeatReactor beatReactor;
|
||||
private EurekaHttpClient eurekaHttpClient;
|
||||
private EurekaBeatReactor beatReactor;
|
||||
|
||||
|
||||
public EurekaNamingService(EurekaHttpClient eurekaHttpClient) {
|
||||
|
|
|
@ -25,7 +25,7 @@ import java.util.function.Consumer;
|
|||
*/
|
||||
@Service
|
||||
public class SpecialSyncEventBus {
|
||||
private final ConcurrentHashMap<String, SpecialSyncEvent> specialSyncEventRegistry = new ConcurrentHashMap<>();
|
||||
private ConcurrentHashMap<String, SpecialSyncEvent> specialSyncEventRegistry = new ConcurrentHashMap<>();
|
||||
|
||||
public void subscribe(TaskDO taskDO, Consumer<TaskDO> syncAction) {
|
||||
SpecialSyncEvent specialSyncEvent = new SpecialSyncEvent();
|
||||
|
|
|
@ -21,6 +21,7 @@ import com.alibaba.nacossync.pojo.model.TaskDO;
|
|||
import com.google.common.eventbus.EventBus;
|
||||
import com.google.common.eventbus.Subscribe;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
|
@ -33,12 +34,9 @@ import java.util.function.Consumer;
|
|||
@Service
|
||||
@Slf4j
|
||||
public class SpecialSyncEventListener {
|
||||
private final EventBus eventBus;
|
||||
|
||||
public SpecialSyncEventListener(EventBus eventBus) {
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private EventBus eventBus;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
eventBus.register(this);
|
||||
|
|
|
@ -13,10 +13,13 @@
|
|||
package com.alibaba.nacossync.extension.holder;
|
||||
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import com.google.common.base.Joiner;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
|
@ -25,36 +28,40 @@ import java.util.function.Supplier;
|
|||
* @date 2018-12-24 22:08
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class AbstractServerHolderImpl<T> implements Holder<T> {
|
||||
|
||||
private final Map<String, T> serviceMap = new ConcurrentHashMap<>();
|
||||
public abstract class AbstractServerHolderImpl<T> implements Holder {
|
||||
|
||||
|
||||
private final Map<String, T> serviceMap = new ConcurrentHashMap<>();
|
||||
@Autowired
|
||||
protected SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
@Override
|
||||
public T get(String clusterId) {
|
||||
public T get(String clusterId, String namespace) {
|
||||
final String finalNamespace = Optional.ofNullable(namespace).orElse(Strings.EMPTY);
|
||||
String key = Joiner.on("_").join(clusterId, finalNamespace);
|
||||
|
||||
return serviceMap.computeIfAbsent(clusterId, clusterKey -> {
|
||||
serviceMap.computeIfAbsent(key, clusterKey -> {
|
||||
try {
|
||||
log.info("Starting create cluster server, clusterId={}", clusterId);
|
||||
return createServer(clusterId, () -> skyWalkerCacheServices.getClusterConnectKey(clusterId));
|
||||
return createServer(clusterId, () -> skyWalkerCacheServices.getClusterConnectKey(clusterId),
|
||||
finalNamespace);
|
||||
} catch (Exception e) {
|
||||
log.error(String.format("clusterId=%s, start server failed", clusterId), e);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
return serviceMap.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create real cluster client instance
|
||||
*
|
||||
* @param clusterId cluster id
|
||||
* @param clusterId cluster id
|
||||
* @param serverAddressSupplier server address
|
||||
* @param namespace name space
|
||||
* @return cluster client instance
|
||||
*/
|
||||
abstract T createServer(String clusterId, Supplier<String> serverAddressSupplier)
|
||||
abstract T createServer(String clusterId, Supplier<String> serverAddressSupplier, String namespace)
|
||||
throws Exception;
|
||||
|
||||
}
|
||||
|
|
|
@ -30,7 +30,7 @@ public class ConsulServerHolder extends AbstractServerHolderImpl<ConsulClient> {
|
|||
public static final String HTTP = "http://";
|
||||
|
||||
@Override
|
||||
ConsulClient createServer(String clusterId, Supplier<String> serverAddressSupplier) throws Exception {
|
||||
ConsulClient createServer(String clusterId, Supplier<String> serverAddressSupplier, String namespace) throws Exception {
|
||||
String serverAddress = serverAddressSupplier.get();
|
||||
serverAddress = serverAddress.startsWith(HTTP) ? serverAddress : HTTP + serverAddress;
|
||||
URL url = new URL(serverAddress);
|
||||
|
|
|
@ -10,7 +10,6 @@
|
|||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.extension.holder;
|
||||
|
||||
import com.alibaba.nacossync.extension.eureka.EurekaNamingService;
|
||||
|
@ -31,27 +30,12 @@ import java.util.function.Supplier;
|
|||
@Service
|
||||
@Slf4j
|
||||
public class EurekaServerHolder extends AbstractServerHolderImpl<EurekaNamingService> {
|
||||
|
||||
private static final String HTTP_PREFIX = "http://";
|
||||
|
||||
private static final String HTTPS_PREFIX = "https://";
|
||||
|
||||
@Override
|
||||
EurekaNamingService createServer(String clusterId, Supplier<String> serverAddressSupplier) throws Exception {
|
||||
RestTemplateTransportClientFactory restTemplateTransportClientFactory = new RestTemplateTransportClientFactory();
|
||||
EurekaEndpoint eurekaEndpoint = new DefaultEndpoint(addHttpPrefix(serverAddressSupplier.get()));
|
||||
EurekaNamingService createServer(String clusterId, Supplier<String> serverAddressSupplier, String namespace) throws Exception {
|
||||
RestTemplateTransportClientFactory restTemplateTransportClientFactory =
|
||||
new RestTemplateTransportClientFactory();
|
||||
EurekaEndpoint eurekaEndpoint = new DefaultEndpoint(serverAddressSupplier.get());
|
||||
EurekaHttpClient eurekaHttpClient = restTemplateTransportClientFactory.newClient(eurekaEndpoint);
|
||||
return new EurekaNamingService(eurekaHttpClient);
|
||||
}
|
||||
|
||||
public String addHttpPrefix(String input) {
|
||||
if (input == null || input.isEmpty()) {
|
||||
return input;
|
||||
}
|
||||
if (!input.startsWith(HTTP_PREFIX) && !input.startsWith(HTTPS_PREFIX)) {
|
||||
input = HTTP_PREFIX + input;
|
||||
}
|
||||
|
||||
return input;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -22,8 +22,9 @@ public interface Holder<T> {
|
|||
/**
|
||||
* Through the cluster ID and namespace fetch cluster client service
|
||||
* @param clusterId cluster id
|
||||
* @param namespace name space
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
T get(String clusterId) throws Exception;
|
||||
T get(String clusterId, String namespace) throws Exception;
|
||||
}
|
||||
|
|
|
@ -18,14 +18,12 @@ import com.alibaba.nacos.api.naming.NamingService;
|
|||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import com.google.common.base.Joiner;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Properties;
|
||||
import java.util.function.Supplier;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author paderlol
|
||||
|
@ -36,13 +34,13 @@ import java.util.function.Supplier;
|
|||
public class NacosServerHolder extends AbstractServerHolderImpl<NamingService> {
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
|
||||
public NacosServerHolder(ClusterAccessService clusterAccessService) {
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
}
|
||||
|
||||
@Override
|
||||
NamingService createServer(String clusterId, Supplier<String> serverAddressSupplier)
|
||||
NamingService createServer(String clusterId, Supplier<String> serverAddressSupplier, String namespace)
|
||||
throws Exception {
|
||||
List<String> allClusterConnectKey = skyWalkerCacheServices
|
||||
.getAllClusterConnectKey(clusterId);
|
||||
|
@ -50,8 +48,7 @@ public class NacosServerHolder extends AbstractServerHolderImpl<NamingService> {
|
|||
String serverList = Joiner.on(",").join(allClusterConnectKey);
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(PropertyKeyConst.SERVER_ADDR, serverList);
|
||||
properties.setProperty(PropertyKeyConst.NAMESPACE, Optional.ofNullable(clusterDO.getNamespace()).orElse(
|
||||
Strings.EMPTY));
|
||||
properties.setProperty(PropertyKeyConst.NAMESPACE, namespace);
|
||||
Optional.ofNullable(clusterDO.getUserName()).ifPresent(value ->
|
||||
properties.setProperty(PropertyKeyConst.USERNAME, value)
|
||||
);
|
||||
|
@ -61,5 +58,4 @@ public class NacosServerHolder extends AbstractServerHolderImpl<NamingService> {
|
|||
);
|
||||
return NamingFactory.createNamingService(properties);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -32,7 +32,7 @@ public class ZookeeperServerHolder extends AbstractServerHolderImpl<CuratorFrame
|
|||
|
||||
|
||||
@Override
|
||||
CuratorFramework createServer(String clusterId, Supplier<String> serverAddressSupplier) {
|
||||
CuratorFramework createServer(String clusterId, Supplier<String> serverAddressSupplier, String namespace) {
|
||||
List<String> allClusterConnectKey = skyWalkerCacheServices
|
||||
.getAllClusterConnectKey(clusterId);
|
||||
String serverList = Joiner.on(",").join(allClusterConnectKey);
|
||||
|
|
|
@ -1,265 +0,0 @@
|
|||
package com.alibaba.nacossync.extension.impl;
|
||||
|
||||
import com.alibaba.nacos.api.exception.NacosException;
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.listener.EventListener;
|
||||
import com.alibaba.nacos.api.naming.listener.NamingEvent;
|
||||
import com.alibaba.nacos.api.naming.pojo.Instance;
|
||||
import com.alibaba.nacos.common.utils.CollectionUtils;
|
||||
import com.alibaba.nacossync.constant.MetricsStatisticsType;
|
||||
import com.alibaba.nacossync.extension.SyncService;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.BatchTaskExecutor;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.alibaba.nacossync.util.NacosUtils.getGroupNameOrDefault;
|
||||
|
||||
@Slf4j
|
||||
public abstract class AbstractNacosSync implements SyncService {
|
||||
|
||||
private final Map<String, EventListener> listenerMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<String, Set<String>> sourceInstanceSnapshot = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<String, Integer> syncTaskTap = new ConcurrentHashMap<>();
|
||||
|
||||
private final ConcurrentHashMap<String, TaskDO> allSyncTaskMap = new ConcurrentHashMap<>();
|
||||
private ScheduledExecutorService executorService;
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
@Getter
|
||||
@Autowired
|
||||
private NacosServerHolder nacosServerHolder;
|
||||
|
||||
|
||||
/**
|
||||
* Due to network issues or other reasons, the Nacos Sync synchronization tasks may fail,
|
||||
* resulting in the target cluster's registry missing synchronized instances.
|
||||
* To prevent the target cluster's registry from missing synchronized instances for an extended period,
|
||||
* a fallback worker thread is started every 5 minutes to execute all synchronization tasks.
|
||||
*/
|
||||
@PostConstruct
|
||||
public void afterPropertiesSet() {
|
||||
initializeExecutorService();
|
||||
scheduleSyncTasks();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
if (executorService != null && !executorService.isShutdown()) {
|
||||
executorService.shutdown();
|
||||
try {
|
||||
if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
|
||||
executorService.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
executorService.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void initializeExecutorService() {
|
||||
executorService = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r);
|
||||
t.setDaemon(true);
|
||||
t.setName("com.alibaba.nacossync.basic.synctask");
|
||||
return t;
|
||||
});
|
||||
}
|
||||
|
||||
private void scheduleSyncTasks() {
|
||||
executorService.scheduleWithFixedDelay(this::executeSyncTasks, 0, 300, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private void executeSyncTasks() {
|
||||
if (allSyncTaskMap.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Collection<TaskDO> taskCollections = allSyncTaskMap.values();
|
||||
List<TaskDO> taskDOList = new ArrayList<>(taskCollections);
|
||||
|
||||
if (CollectionUtils.isNotEmpty(taskDOList)) {
|
||||
BatchTaskExecutor.batchOperation(taskDOList, this::executeTask);
|
||||
}
|
||||
}
|
||||
|
||||
private void executeTask(TaskDO task) {
|
||||
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||
String taskId = task.getTaskId();
|
||||
try {
|
||||
NamingService sourceNamingService = nacosServerHolder.get(task.getSourceClusterId());
|
||||
doSync(taskId, task, sourceNamingService);
|
||||
} catch (NacosException e) {
|
||||
log.error("sync task from nacos to nacos failed, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
} catch (Exception e) {
|
||||
log.error("Unexpected error during sync task, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
} finally {
|
||||
stopwatch.stop();
|
||||
log.debug("Task execution time for taskId {}: {} ms", taskId, stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(TaskDO taskDO) {
|
||||
String taskId = taskDO.getTaskId();
|
||||
try {
|
||||
|
||||
NamingService sourceNamingService = nacosServerHolder.get(taskDO.getSourceClusterId());
|
||||
//移除订阅
|
||||
EventListener listener = listenerMap.remove(taskId);
|
||||
if (listener!= null) {
|
||||
sourceNamingService.unsubscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()), listener);
|
||||
}
|
||||
sourceNamingService.unsubscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
listenerMap.remove(taskId));
|
||||
sourceInstanceSnapshot.remove(taskId);
|
||||
allSyncTaskMap.remove(taskId);
|
||||
|
||||
// 删除目标集群中同步的实例列表
|
||||
deregisterInstance(taskDO);
|
||||
}catch (NacosException e) {
|
||||
log.error("Delete task from nacos to specify destination was failed, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
log.error("Unexpected error during sync task, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean sync(TaskDO taskDO, Integer index) {
|
||||
String taskId = taskDO.getTaskId();
|
||||
try {
|
||||
NamingService sourceNamingService = nacosServerHolder.get(taskDO.getSourceClusterId());
|
||||
allSyncTaskMap.put(taskId, taskDO);
|
||||
//防止暂停同步任务后,重新同步/或删除任务以后新建任务不会再接收到新的事件导致不能同步,所以每次订阅事件之前,先全量同步一次任务
|
||||
doSync(taskId, taskDO, sourceNamingService);
|
||||
this.listenerMap.putIfAbsent(taskId, event -> {
|
||||
if (event instanceof NamingEvent) {
|
||||
try {
|
||||
doSync(taskId, taskDO, sourceNamingService);
|
||||
} catch (Exception e) {
|
||||
log.error("event process fail, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
}
|
||||
}
|
||||
});
|
||||
sourceNamingService.subscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
listenerMap.get(taskId));
|
||||
}catch (NacosException e) {
|
||||
log.error("Nacos sync task process fail, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
log.error("Unexpected error during sync task, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void doSync(String taskId, TaskDO taskDO, NamingService sourceNamingService) throws Exception {
|
||||
if (syncTaskTap.putIfAbsent(taskId, 1) != null) {
|
||||
log.info("任务Id:{}上一个同步任务尚未结束", taskId);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// 直接从本地保存的serviceInfoMap中取订阅的服务实例
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), new ArrayList<>(), true);
|
||||
// 先删除不存在的
|
||||
this.removeInvalidInstance(taskDO, sourceInstances);
|
||||
|
||||
// 同步实例
|
||||
this.syncNewInstance(taskDO, sourceInstances);
|
||||
} finally {
|
||||
syncTaskTap.remove(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
private void syncNewInstance(TaskDO taskDO, List<Instance> sourceInstances) throws NacosException {
|
||||
Set<String> latestSyncInstance = new TreeSet<>();
|
||||
//再次添加新实例
|
||||
String taskId = taskDO.getTaskId();
|
||||
Set<String> instanceKeys = sourceInstanceSnapshot.get(taskId);
|
||||
for (Instance instance : sourceInstances) {
|
||||
if (needSync(instance.getMetadata())) {
|
||||
String instanceKey = composeInstanceKey(instance.getIp(), instance.getPort());
|
||||
if (CollectionUtils.isEmpty(instanceKeys) || !instanceKeys.contains(instanceKey)) {
|
||||
register(taskDO, instance);
|
||||
}
|
||||
latestSyncInstance.add(instanceKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (CollectionUtils.isNotEmpty(latestSyncInstance)) {
|
||||
log.info("任务Id:{},已同步实例个数:{}", taskId, latestSyncInstance.size());
|
||||
sourceInstanceSnapshot.put(taskId, latestSyncInstance);
|
||||
} else {
|
||||
// latestSyncInstance为空表示源集群中需要同步的所有实例(即非nacos-sync同步过来的实例)已经下线,清除本地持有快照
|
||||
sourceInstanceSnapshot.remove(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void removeInvalidInstance(TaskDO taskDO, List<Instance> sourceInstances) throws Exception {
|
||||
String taskId = taskDO.getTaskId();
|
||||
if (this.sourceInstanceSnapshot.containsKey(taskId)) {
|
||||
Set<String> oldInstanceKeys = this.sourceInstanceSnapshot.get(taskId);
|
||||
Set<String> newInstanceKeys = sourceInstances.stream()
|
||||
.map(instance -> composeInstanceKey(instance.getIp(), instance.getPort()))
|
||||
.collect(Collectors.toSet());
|
||||
oldInstanceKeys.removeAll(newInstanceKeys);
|
||||
if (CollectionUtils.isNotEmpty(oldInstanceKeys)) {
|
||||
log.info("任务Id:{},移除无效同步实例:{}", taskId, oldInstanceKeys);
|
||||
removeInvalidInstance(taskDO, oldInstanceKeys);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needDelete(Map<String, String> destMetaData, TaskDO taskDO) {
|
||||
return SyncService.super.needDelete(destMetaData, taskDO);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean needSync(Map<String, String> sourceMetaData) {
|
||||
return SyncService.super.needSync(sourceMetaData);
|
||||
}
|
||||
|
||||
public abstract String composeInstanceKey(String ip, int port);
|
||||
|
||||
public abstract void register(TaskDO taskDO, Instance instance);
|
||||
|
||||
public abstract void deregisterInstance(TaskDO taskDO) throws Exception;
|
||||
|
||||
public abstract void removeInvalidInstance(TaskDO taskDO, Set<String> invalidInstanceKeys) throws Exception;
|
||||
|
||||
}
|
|
@ -27,22 +27,18 @@ import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
|||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.ConsulUtils;
|
||||
import com.alibaba.nacossync.util.NacosUtils;
|
||||
import com.ecwid.consul.v1.ConsulClient;
|
||||
import com.ecwid.consul.v1.QueryParams;
|
||||
import com.ecwid.consul.v1.Response;
|
||||
import com.ecwid.consul.v1.health.model.HealthService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Consul 同步 Nacos
|
||||
*
|
||||
*
|
||||
* @author paderlol
|
||||
* @date: 2018-12-31 16:25
|
||||
*/
|
||||
|
@ -50,7 +46,8 @@ import java.util.Set;
|
|||
@NacosSyncService(sourceCluster = ClusterTypeEnum.CONSUL, destinationCluster = ClusterTypeEnum.NACOS)
|
||||
public class ConsulSyncToNacosServiceImpl implements SyncService {
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
private final ConsulServerHolder consulServerHolder;
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
@ -59,15 +56,14 @@ public class ConsulSyncToNacosServiceImpl implements SyncService {
|
|||
|
||||
private final SpecialSyncEventBus specialSyncEventBus;
|
||||
|
||||
|
||||
@Autowired
|
||||
public ConsulSyncToNacosServiceImpl(ConsulServerHolder consulServerHolder,
|
||||
SkyWalkerCacheServices skyWalkerCacheServices, NacosServerHolder nacosServerHolder,
|
||||
SpecialSyncEventBus specialSyncEventBus, MetricsManager metricsManager) {
|
||||
SpecialSyncEventBus specialSyncEventBus) {
|
||||
this.consulServerHolder = consulServerHolder;
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.specialSyncEventBus = specialSyncEventBus;
|
||||
this.metricsManager = metricsManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -75,15 +71,12 @@ public class ConsulSyncToNacosServiceImpl implements SyncService {
|
|||
|
||||
try {
|
||||
specialSyncEventBus.unsubscribe(taskDO);
|
||||
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
List<Instance> allInstances = destNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()));
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), null);
|
||||
List<Instance> allInstances = destNamingService.getAllInstances(taskDO.getServiceName());
|
||||
for (Instance instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()), instance.getIp(), instance.getPort());
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(), instance.getIp(), instance.getPort());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -96,17 +89,17 @@ public class ConsulSyncToNacosServiceImpl implements SyncService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean sync(TaskDO taskDO, Integer index) {
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
try {
|
||||
ConsulClient consulClient = consulServerHolder.get(taskDO.getSourceClusterId());
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
ConsulClient consulClient = consulServerHolder.get(taskDO.getSourceClusterId(), null);
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), null);
|
||||
Response<List<HealthService>> response =
|
||||
consulClient.getHealthServices(taskDO.getServiceName(), true, QueryParams.DEFAULT);
|
||||
List<HealthService> healthServiceList = response.getValue();
|
||||
Set<String> instanceKeys = new HashSet<>();
|
||||
overrideAllInstance(taskDO, destNamingService, healthServiceList, instanceKeys);
|
||||
cleanAllOldInstance(taskDO, destNamingService, instanceKeys);
|
||||
specialSyncEventBus.subscribe(taskDO, t->sync(t, index));
|
||||
specialSyncEventBus.subscribe(taskDO, this::sync);
|
||||
} catch (Exception e) {
|
||||
log.error("Sync task from consul to nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
|
@ -122,8 +115,7 @@ public class ConsulSyncToNacosServiceImpl implements SyncService {
|
|||
if (needDelete(instance.getMetadata(), taskDO)
|
||||
&& !instanceKeys.contains(composeInstanceKey(instance.getIp(), instance.getPort()))) {
|
||||
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()), instance.getIp(), instance.getPort());
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(), instance.getIp(), instance.getPort());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -133,7 +125,6 @@ public class ConsulSyncToNacosServiceImpl implements SyncService {
|
|||
for (HealthService healthService : healthServiceList) {
|
||||
if (needSync(ConsulUtils.transferMetadata(healthService.getService().getTags()))) {
|
||||
destNamingService.registerInstance(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
buildSyncInstance(healthService, taskDO));
|
||||
instanceKeys.add(composeInstanceKey(healthService.getService().getAddress(),
|
||||
healthService.getService().getPort()));
|
||||
|
@ -146,10 +137,10 @@ public class ConsulSyncToNacosServiceImpl implements SyncService {
|
|||
temp.setIp(instance.getService().getAddress());
|
||||
temp.setPort(instance.getService().getPort());
|
||||
Map<String, String> metaData = new HashMap<>(ConsulUtils.transferMetadata(instance.getService().getTags()));
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTER_ID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTERID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY, taskDO.getSourceClusterId());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTERID_KEY, taskDO.getSourceClusterId());
|
||||
temp.setMetadata(metaData);
|
||||
return temp;
|
||||
}
|
||||
|
|
|
@ -27,7 +27,6 @@ import com.alibaba.nacossync.extension.holder.EurekaServerHolder;
|
|||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.NacosUtils;
|
||||
import com.netflix.appinfo.InstanceInfo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
@ -39,7 +38,7 @@ import java.util.Map;
|
|||
|
||||
/**
|
||||
* eureka
|
||||
*
|
||||
*
|
||||
* @author paderlol
|
||||
* @date: 2018-12-31 16:25
|
||||
*/
|
||||
|
@ -72,10 +71,8 @@ public class EurekaSyncToNacosServiceImpl implements SyncService {
|
|||
|
||||
try {
|
||||
specialSyncEventBus.unsubscribe(taskDO);
|
||||
|
||||
EurekaNamingService eurekaNamingService = eurekaServerHolder.get(taskDO.getSourceClusterId());
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
|
||||
EurekaNamingService eurekaNamingService = eurekaServerHolder.get(taskDO.getSourceClusterId(), null);
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), null);
|
||||
List<InstanceInfo> eurekaInstances = eurekaNamingService.getApplications(taskDO.getServiceName());
|
||||
deleteAllInstanceFromEureka(taskDO, destNamingService, eurekaInstances);
|
||||
|
||||
|
@ -88,15 +85,12 @@ public class EurekaSyncToNacosServiceImpl implements SyncService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean sync(TaskDO taskDO,Integer index) {
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
try {
|
||||
|
||||
EurekaNamingService eurekaNamingService = eurekaServerHolder.get(taskDO.getSourceClusterId());
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
|
||||
EurekaNamingService eurekaNamingService = eurekaServerHolder.get(taskDO.getSourceClusterId(), null);
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), null);
|
||||
List<InstanceInfo> eurekaInstances = eurekaNamingService.getApplications(taskDO.getServiceName());
|
||||
List<Instance> nacosInstances = destNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()));
|
||||
List<Instance> nacosInstances = destNamingService.getAllInstances(taskDO.getServiceName());
|
||||
|
||||
if (CollectionUtils.isEmpty(eurekaInstances)) {
|
||||
// Clear all instance from Nacos
|
||||
|
@ -108,7 +102,7 @@ public class EurekaSyncToNacosServiceImpl implements SyncService {
|
|||
}
|
||||
addValidInstance(taskDO, destNamingService, eurekaInstances);
|
||||
}
|
||||
specialSyncEventBus.subscribe(taskDO, t->sync(t, index));
|
||||
specialSyncEventBus.subscribe(taskDO, this::sync);
|
||||
} catch (Exception e) {
|
||||
log.error("sync task from eureka to nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
|
@ -123,25 +117,18 @@ public class EurekaSyncToNacosServiceImpl implements SyncService {
|
|||
if (needSync(instance.getMetadata())) {
|
||||
log.info("Add service instance from Eureka, serviceName={}, Ip={}, port={}",
|
||||
instance.getAppName(), instance.getIPAddr(), instance.getPort());
|
||||
destNamingService.registerInstance(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()), buildSyncInstance(instance,
|
||||
taskDO));
|
||||
destNamingService.registerInstance(taskDO.getServiceName(), buildSyncInstance(instance, taskDO));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteAllInstanceFromEureka(TaskDO taskDO, NamingService destNamingService,
|
||||
List<InstanceInfo> eurekaInstances)
|
||||
throws NacosException {
|
||||
if (CollectionUtils.isEmpty(eurekaInstances)) {
|
||||
return;
|
||||
}
|
||||
|
||||
private void deleteAllInstanceFromEureka(TaskDO taskDO, NamingService destNamingService, List<InstanceInfo> eurekaInstances)
|
||||
throws NacosException {
|
||||
for (InstanceInfo instance : eurekaInstances) {
|
||||
if (needSync(instance.getMetadata())) {
|
||||
log.info("Delete service instance from Eureka, serviceName={}, Ip={}, port={}",
|
||||
instance.getAppName(), instance.getIPAddr(), instance.getPort());
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()), buildSyncInstance(instance, taskDO));
|
||||
instance.getAppName(), instance.getIPAddr(), instance.getPort());
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(), buildSyncInstance(instance, taskDO));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -152,8 +139,7 @@ public class EurekaSyncToNacosServiceImpl implements SyncService {
|
|||
if (!isExistInEurekaInstance(eurekaInstances, instance) && needDelete(instance.getMetadata(), taskDO)) {
|
||||
log.info("Remove invalid service instance from Nacos, serviceName={}, Ip={}, port={}",
|
||||
instance.getServiceName(), instance.getIp(), instance.getPort());
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()), instance.getIp(), instance.getPort());
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(), instance.getIp(), instance.getPort());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -168,8 +154,7 @@ public class EurekaSyncToNacosServiceImpl implements SyncService {
|
|||
throws NacosException {
|
||||
for (Instance instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(),
|
||||
NacosUtils.getGroupNameOrDefault(taskDO.getGroupName()), instance);
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(), instance);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -183,10 +168,10 @@ public class EurekaSyncToNacosServiceImpl implements SyncService {
|
|||
temp.setHealthy(true);
|
||||
|
||||
Map<String, String> metaData = new HashMap<>(instance.getMetadata());
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTER_ID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTERID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY, taskDO.getSourceClusterId());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTERID_KEY, taskDO.getSourceClusterId());
|
||||
temp.setMetadata(metaData);
|
||||
return temp;
|
||||
}
|
||||
|
|
|
@ -10,15 +10,21 @@
|
|||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.extension.impl;
|
||||
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.listener.EventListener;
|
||||
import com.alibaba.nacos.api.naming.listener.NamingEvent;
|
||||
import com.alibaba.nacos.api.naming.pojo.Instance;
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.constant.MetricsStatisticsType;
|
||||
import com.alibaba.nacossync.constant.SkyWalkerConstants;
|
||||
import com.alibaba.nacossync.extension.SyncService;
|
||||
import com.alibaba.nacossync.extension.annotation.NacosSyncService;
|
||||
import com.alibaba.nacossync.extension.holder.ConsulServerHolder;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.ConsulUtils;
|
||||
import com.ecwid.consul.v1.ConsulClient;
|
||||
|
@ -27,13 +33,15 @@ import com.ecwid.consul.v1.Response;
|
|||
import com.ecwid.consul.v1.agent.model.NewService;
|
||||
import com.ecwid.consul.v1.health.model.HealthService;
|
||||
import com.google.common.collect.Lists;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
|
@ -41,66 +49,107 @@ import java.util.stream.Collectors;
|
|||
*/
|
||||
@Slf4j
|
||||
@NacosSyncService(sourceCluster = ClusterTypeEnum.NACOS, destinationCluster = ClusterTypeEnum.CONSUL)
|
||||
public class NacosSyncToConsulServiceImpl extends AbstractNacosSync {
|
||||
|
||||
|
||||
private static final String DELIMITER = "=";
|
||||
|
||||
public class NacosSyncToConsulServiceImpl implements SyncService {
|
||||
private Map<String, EventListener> nacosListenerMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
private final ConsulServerHolder consulServerHolder;
|
||||
|
||||
public NacosSyncToConsulServiceImpl(SkyWalkerCacheServices skyWalkerCacheServices,
|
||||
ConsulServerHolder consulServerHolder) {
|
||||
|
||||
public NacosSyncToConsulServiceImpl(MetricsManager metricsManager, SkyWalkerCacheServices skyWalkerCacheServices,
|
||||
NacosServerHolder nacosServerHolder, ConsulServerHolder consulServerHolder) {
|
||||
this.metricsManager = metricsManager;
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.consulServerHolder = consulServerHolder;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String composeInstanceKey(String ip, int port) {
|
||||
return String.join(":", ip, String.valueOf(port));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void register(TaskDO taskDO, Instance instance) {
|
||||
ConsulClient consulClient = consulServerHolder.get(taskDO.getDestClusterId());
|
||||
consulClient.agentServiceRegister(buildSyncInstance(instance, taskDO));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deregisterInstance(TaskDO taskDO) throws Exception {
|
||||
ConsulClient consulClient = consulServerHolder.get(taskDO.getDestClusterId());
|
||||
Response<List<HealthService>> serviceResponse = consulClient.getHealthServices(taskDO.getServiceName(), true,
|
||||
QueryParams.DEFAULT);
|
||||
List<HealthService> healthServices = serviceResponse.getValue();
|
||||
for (HealthService healthService : healthServices) {
|
||||
|
||||
if (needDelete(ConsulUtils.transferMetadata(healthService.getService().getTags()), taskDO)) {
|
||||
consulClient.agentServiceDeregister(
|
||||
URLEncoder.encode(healthService.getService().getId(), StandardCharsets.UTF_8));
|
||||
public boolean delete(TaskDO taskDO) {
|
||||
try {
|
||||
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());
|
||||
ConsulClient consulClient = consulServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
|
||||
|
||||
sourceNamingService.unsubscribe(taskDO.getServiceName(), nacosListenerMap.get(taskDO.getTaskId()));
|
||||
|
||||
// 删除目标集群中同步的实例列表
|
||||
Response<List<HealthService>> serviceResponse =
|
||||
consulClient.getHealthServices(taskDO.getServiceName(), true, QueryParams.DEFAULT);
|
||||
List<HealthService> healthServices = serviceResponse.getValue();
|
||||
for (HealthService healthService : healthServices) {
|
||||
|
||||
if (needDelete(ConsulUtils.transferMetadata(healthService.getService().getTags()), taskDO)) {
|
||||
consulClient.agentServiceDeregister(URLEncoder
|
||||
.encode(healthService.getService().getId(), StandardCharsets.UTF_8.name()));
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("delete a task from nacos to nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.DELETE_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void removeInvalidInstance(TaskDO taskDO, Set<String> invalidInstanceKeys)
|
||||
throws UnsupportedEncodingException {
|
||||
ConsulClient consulClient = consulServerHolder.get(taskDO.getDestClusterId());
|
||||
Response<List<HealthService>> serviceResponse = consulClient.getHealthServices(taskDO.getServiceName(), true,
|
||||
QueryParams.DEFAULT);
|
||||
List<HealthService> healthServices = serviceResponse.getValue();
|
||||
for (HealthService healthService : healthServices) {
|
||||
|
||||
if (needDelete(ConsulUtils.transferMetadata(healthService.getService().getTags()), taskDO)
|
||||
&& !invalidInstanceKeys.contains(composeInstanceKey(healthService.getService().getAddress(),
|
||||
healthService.getService().getPort()))) {
|
||||
consulClient.agentServiceDeregister(
|
||||
URLEncoder.encode(healthService.getService().getId(), StandardCharsets.UTF_8));
|
||||
}
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
try {
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());
|
||||
ConsulClient consulClient = consulServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
|
||||
|
||||
nacosListenerMap.putIfAbsent(taskDO.getTaskId(), event -> {
|
||||
if (event instanceof NamingEvent) {
|
||||
try {
|
||||
Set<String> instanceKeySet = new HashSet();
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName());
|
||||
// 先将新的注册一遍
|
||||
for (Instance instance : sourceInstances) {
|
||||
if (needSync(instance.getMetadata())) {
|
||||
consulClient.agentServiceRegister(buildSyncInstance(instance, taskDO));
|
||||
instance.getInstanceId();
|
||||
instanceKeySet.add(composeInstanceKey(instance.getIp(), instance.getPort()));
|
||||
}
|
||||
}
|
||||
|
||||
// 再将不存在的删掉
|
||||
Response<List<HealthService>> serviceResponse =
|
||||
consulClient.getHealthServices(taskDO.getServiceName(), true, QueryParams.DEFAULT);
|
||||
List<HealthService> healthServices = serviceResponse.getValue();
|
||||
for (HealthService healthService : healthServices) {
|
||||
|
||||
if (needDelete(ConsulUtils.transferMetadata(healthService.getService().getTags()), taskDO)
|
||||
&& !instanceKeySet.contains(composeInstanceKey(healthService.getService().getAddress(),
|
||||
healthService.getService().getPort()))) {
|
||||
consulClient.agentServiceDeregister(URLEncoder
|
||||
.encode(healthService.getService().getId(), StandardCharsets.UTF_8.toString()));
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("event process fail, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
sourceNamingService.subscribe(taskDO.getServiceName(), nacosListenerMap.get(taskDO.getTaskId()));
|
||||
} catch (Exception e) {
|
||||
log.error("sync task from nacos to nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
private String composeInstanceKey(String ip, int port) {
|
||||
return ip + ":" + port;
|
||||
}
|
||||
|
||||
public NewService buildSyncInstance(Instance instance, TaskDO taskDO) {
|
||||
NewService newService = new NewService();
|
||||
newService.setAddress(instance.getIp());
|
||||
|
@ -109,13 +158,13 @@ public class NacosSyncToConsulServiceImpl extends AbstractNacosSync {
|
|||
newService.setId(instance.getInstanceId());
|
||||
List<String> tags = Lists.newArrayList();
|
||||
tags.addAll(instance.getMetadata().entrySet().stream()
|
||||
.map(entry -> String.join(DELIMITER, entry.getKey(), entry.getValue())).collect(Collectors.toList()));
|
||||
tags.add(String.join(DELIMITER, SkyWalkerConstants.DEST_CLUSTER_ID_KEY, taskDO.getDestClusterId()));
|
||||
tags.add(String.join(DELIMITER, SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode()));
|
||||
tags.add(String.join(DELIMITER, SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY, taskDO.getSourceClusterId()));
|
||||
.map(entry -> String.join("=", entry.getKey(), entry.getValue())).collect(Collectors.toList()));
|
||||
tags.add(String.join("=", SkyWalkerConstants.DEST_CLUSTERID_KEY, taskDO.getDestClusterId()));
|
||||
tags.add(String.join("=", SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode()));
|
||||
tags.add(String.join("=", SkyWalkerConstants.SOURCE_CLUSTERID_KEY, taskDO.getSourceClusterId()));
|
||||
newService.setTags(tags);
|
||||
return newService;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -10,17 +10,23 @@
|
|||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.extension.impl;
|
||||
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.listener.Event;
|
||||
import com.alibaba.nacos.api.naming.listener.EventListener;
|
||||
import com.alibaba.nacos.api.naming.listener.NamingEvent;
|
||||
import com.alibaba.nacos.api.naming.pojo.Instance;
|
||||
import com.alibaba.nacos.common.utils.CollectionUtils;
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.constant.MetricsStatisticsType;
|
||||
import com.alibaba.nacossync.constant.SkyWalkerConstants;
|
||||
import com.alibaba.nacossync.extension.SyncService;
|
||||
import com.alibaba.nacossync.extension.annotation.NacosSyncService;
|
||||
import com.alibaba.nacossync.extension.eureka.EurekaNamingService;
|
||||
import com.alibaba.nacossync.extension.holder.EurekaServerHolder;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.netflix.appinfo.DataCenterInfo;
|
||||
import com.netflix.appinfo.InstanceInfo;
|
||||
|
@ -28,96 +34,171 @@ import com.netflix.appinfo.LeaseInfo;
|
|||
import com.netflix.appinfo.MyDataCenterInfo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* @author zhanglong
|
||||
*/
|
||||
@Slf4j
|
||||
@NacosSyncService(sourceCluster = ClusterTypeEnum.NACOS, destinationCluster = ClusterTypeEnum.EUREKA)
|
||||
public class NacosSyncToEurekaServiceImpl extends AbstractNacosSync {
|
||||
|
||||
|
||||
private final EurekaServerHolder eurekaServerHolder;
|
||||
|
||||
public class NacosSyncToEurekaServiceImpl implements SyncService {
|
||||
private final Map<String, EventListener> nacosListenerMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
public NacosSyncToEurekaServiceImpl(EurekaServerHolder eurekaServerHolder,
|
||||
SkyWalkerCacheServices skyWalkerCacheServices) {
|
||||
this.eurekaServerHolder = eurekaServerHolder;
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
private final EurekaServerHolder eurekaServerHolder;
|
||||
|
||||
public NacosSyncToEurekaServiceImpl(MetricsManager metricsManager, SkyWalkerCacheServices skyWalkerCacheServices,
|
||||
NacosServerHolder nacosServerHolder, EurekaServerHolder eurekaServerHolder) {
|
||||
this.metricsManager = metricsManager;
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.eurekaServerHolder = eurekaServerHolder;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String composeInstanceKey(String ip, int port) {
|
||||
public boolean delete(TaskDO taskDO) {
|
||||
try {
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());
|
||||
EurekaNamingService destNamingService =
|
||||
eurekaServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
|
||||
|
||||
sourceNamingService.unsubscribe(taskDO.getServiceName(), nacosListenerMap.get(taskDO.getTaskId()));
|
||||
// 删除目标集群中同步的实例列表
|
||||
List<InstanceInfo> allInstances = destNamingService.getApplications(taskDO.getServiceName());
|
||||
if (allInstances != null) {
|
||||
for (InstanceInfo instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
destNamingService.deregisterInstance(instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("delete task from nacos to eureka was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.DELETE_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
try {
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());
|
||||
EurekaNamingService destNamingService =
|
||||
eurekaServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
|
||||
|
||||
nacosListenerMap.putIfAbsent(taskDO.getTaskId(), event -> {
|
||||
processNamingEvent(taskDO, sourceNamingService, destNamingService, event);
|
||||
});
|
||||
|
||||
sourceNamingService.subscribe(taskDO.getServiceName(), nacosListenerMap.get(taskDO.getTaskId()));
|
||||
} catch (Exception e) {
|
||||
log.error("sync task from eureka to nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void processNamingEvent(TaskDO taskDO, NamingService sourceNamingService,
|
||||
EurekaNamingService destNamingService, Event event) {
|
||||
if (event instanceof NamingEvent) {
|
||||
try {
|
||||
Set<String> instanceKeySet = new HashSet<>();
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName());
|
||||
// 先将新的注册一遍
|
||||
addAllNewInstance(taskDO, destNamingService, instanceKeySet, sourceInstances);
|
||||
// 再将不存在的删掉
|
||||
ifNecessaryDelete(taskDO, destNamingService, instanceKeySet);
|
||||
} catch (Exception e) {
|
||||
log.error("event process fail, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ifNecessaryDelete(TaskDO taskDO, EurekaNamingService destNamingService, Set<String> instanceKeySet) {
|
||||
List<InstanceInfo> allInstances = destNamingService.getApplications(taskDO.getServiceName());
|
||||
if (allInstances != null){
|
||||
for (InstanceInfo instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO) && !instanceKeySet.contains(composeInstanceKey(instance.getIPAddr(),
|
||||
instance.getPort()))) {
|
||||
destNamingService.deregisterInstance(instance);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addAllNewInstance(TaskDO taskDO, EurekaNamingService destNamingService, Set<String> instanceKeySet,
|
||||
List<Instance> sourceInstances) {
|
||||
for (Instance instance : sourceInstances) {
|
||||
if (needSync(instance.getMetadata())) {
|
||||
destNamingService.registerInstance(buildSyncInstance(instance, taskDO));
|
||||
instanceKeySet.add(composeInstanceKey(instance.getIp(), instance.getPort()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String composeInstanceKey(String ip, int port) {
|
||||
return ip + ":" + port;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void register(TaskDO taskDO, Instance instance) {
|
||||
EurekaNamingService destNamingService = eurekaServerHolder.get(taskDO.getDestClusterId());
|
||||
destNamingService.registerInstance(buildSyncInstance(instance, taskDO));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deregisterInstance(TaskDO taskDO) {
|
||||
EurekaNamingService destNamingService = eurekaServerHolder.get(taskDO.getDestClusterId());
|
||||
List<InstanceInfo> allInstances = destNamingService.getApplications(taskDO.getServiceName());
|
||||
if (allInstances != null) {
|
||||
for (InstanceInfo instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
destNamingService.deregisterInstance(instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeInvalidInstance(TaskDO taskDO, Set<String> invalidInstanceKeys) {
|
||||
EurekaNamingService destNamingService = eurekaServerHolder.get(taskDO.getDestClusterId());
|
||||
List<InstanceInfo> allInstances = destNamingService.getApplications(taskDO.getServiceName());
|
||||
if (CollectionUtils.isNotEmpty(allInstances)) {
|
||||
for (InstanceInfo instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO) && invalidInstanceKeys.contains(
|
||||
composeInstanceKey(instance.getIPAddr(), instance.getPort()))) {
|
||||
destNamingService.deregisterInstance(instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
private InstanceInfo buildSyncInstance(Instance instance, TaskDO taskDO) {
|
||||
DataCenterInfo dataCenterInfo = new MyDataCenterInfo(DataCenterInfo.Name.MyOwn);
|
||||
final Map<String, String> instanceMetadata = instance.getMetadata();
|
||||
HashMap<String, String> metadata = new HashMap<>(16);
|
||||
metadata.put(SkyWalkerConstants.DEST_CLUSTER_ID_KEY, taskDO.getDestClusterId());
|
||||
metadata.put(SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metadata.put(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY, taskDO.getSourceClusterId());
|
||||
metadata.put(SkyWalkerConstants.DEST_CLUSTERID_KEY, taskDO.getDestClusterId());
|
||||
metadata.put(SkyWalkerConstants.SYNC_SOURCE_KEY, skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metadata.put(SkyWalkerConstants.SOURCE_CLUSTERID_KEY, taskDO.getSourceClusterId());
|
||||
metadata.putAll(instanceMetadata);
|
||||
String homePageUrl = obtainHomePageUrl(instance, instanceMetadata);
|
||||
String serviceName = taskDO.getServiceName();
|
||||
|
||||
return new InstanceInfo(instance.getIp() + ":" + serviceName + ":" + instance.getPort(), serviceName, null,
|
||||
instance.getIp(), null, new InstanceInfo.PortWrapper(true, instance.getPort()), null,
|
||||
homePageUrl + "/actuator/env", homePageUrl + "/actuator/info", homePageUrl + "/actuator/health", null,
|
||||
serviceName, serviceName, 1, dataCenterInfo, instance.getIp(), InstanceInfo.InstanceStatus.UP,
|
||||
InstanceInfo.InstanceStatus.UNKNOWN, null, new LeaseInfo(30, 90, 0L, 0L, 0L, 0L, 0L), false, metadata,
|
||||
System.currentTimeMillis(), System.currentTimeMillis(), null, null);
|
||||
|
||||
return new InstanceInfo(
|
||||
instance.getIp() + ":" + serviceName + ":" + instance.getPort(),
|
||||
serviceName,
|
||||
null,
|
||||
instance.getIp(),
|
||||
null,
|
||||
new InstanceInfo.PortWrapper(true, instance.getPort()),
|
||||
null,
|
||||
homePageUrl+"/actuator/env",
|
||||
homePageUrl + "/actuator/info",
|
||||
homePageUrl + "/actuator/health",
|
||||
null,
|
||||
serviceName,
|
||||
serviceName,
|
||||
1,
|
||||
dataCenterInfo,
|
||||
instance.getIp(),
|
||||
InstanceInfo.InstanceStatus.UP,
|
||||
InstanceInfo.InstanceStatus.UNKNOWN,
|
||||
null,
|
||||
new LeaseInfo(30, 90,
|
||||
0L, 0L, 0L, 0L, 0L),
|
||||
false,
|
||||
metadata,
|
||||
System.currentTimeMillis(),
|
||||
System.currentTimeMillis(),
|
||||
null,
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
private String obtainHomePageUrl(Instance instance, Map<String, String> instanceMetadata) {
|
||||
final String managementContextPath = obtainManagementContextPath(instanceMetadata);
|
||||
final String managementContextPath =
|
||||
obtainManagementContextPath(instanceMetadata);
|
||||
final String managementPort = instanceMetadata.getOrDefault(SkyWalkerConstants.MANAGEMENT_PORT_KEY,
|
||||
String.valueOf(instance.getPort()));
|
||||
return String.format("http://%s:%s%s", instance.getIp(), managementPort, managementContextPath);
|
||||
String.valueOf(instance.getPort()));
|
||||
return String.format("http://%s:%s%s",instance.getIp(),managementPort,managementContextPath);
|
||||
}
|
||||
|
||||
|
||||
private String obtainManagementContextPath(Map<String, String> instanceMetadata) {
|
||||
final String path = instanceMetadata.getOrDefault(SkyWalkerConstants.MANAGEMENT_CONTEXT_PATH_KEY, "");
|
||||
if (path.endsWith("/")) {
|
||||
|
@ -125,6 +206,7 @@ public class NacosSyncToEurekaServiceImpl extends AbstractNacosSync {
|
|||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -10,9 +10,9 @@
|
|||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.extension.impl;
|
||||
|
||||
import com.alibaba.nacos.api.common.Constants;
|
||||
import com.alibaba.nacos.api.exception.NacosException;
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.listener.EventListener;
|
||||
|
@ -23,32 +23,24 @@ import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
|||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.constant.MetricsStatisticsType;
|
||||
import com.alibaba.nacossync.constant.SkyWalkerConstants;
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.extension.SyncService;
|
||||
import com.alibaba.nacossync.extension.annotation.NacosSyncService;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.BatchTaskExecutor;
|
||||
import com.alibaba.nacossync.util.StringUtils;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
|
||||
import com.alibaba.nacossync.util.Collections;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.alibaba.nacossync.constant.SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY;
|
||||
import static com.alibaba.nacossync.util.NacosUtils.getGroupNameOrDefault;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
/**
|
||||
* @author yangyshdan
|
||||
|
@ -57,173 +49,85 @@ import static com.alibaba.nacossync.util.NacosUtils.getGroupNameOrDefault;
|
|||
|
||||
@Slf4j
|
||||
@NacosSyncService(sourceCluster = ClusterTypeEnum.NACOS, destinationCluster = ClusterTypeEnum.NACOS)
|
||||
public class NacosSyncToNacosServiceImpl implements SyncService, InitializingBean, DisposableBean {
|
||||
|
||||
private final Map<String, EventListener> listenerMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<String, Integer> syncTaskTap = new ConcurrentHashMap<>();
|
||||
|
||||
private final ConcurrentHashMap<String, TaskDO> allSyncTaskMap = new ConcurrentHashMap<>();
|
||||
|
||||
private ScheduledExecutorService executorService;
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
public NacosSyncToNacosServiceImpl(MetricsManager metricsManager, NacosServerHolder nacosServerHolder,
|
||||
ClusterAccessService clusterAccessService, SkyWalkerCacheServices skyWalkerCacheServices) {
|
||||
this.metricsManager = metricsManager;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Due to network issues or other reasons, the Nacos Sync synchronization tasks may fail,
|
||||
* resulting in the target cluster's registry missing synchronized instances.
|
||||
* To prevent the target cluster's registry from missing synchronized instances for an extended period,
|
||||
* a fallback worker thread is started every 5 minutes to execute all synchronization tasks.
|
||||
*/
|
||||
@Override
|
||||
public void afterPropertiesSet() {
|
||||
initializeExecutorService();
|
||||
scheduleSyncTasks();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (executorService != null && !executorService.isShutdown()) {
|
||||
executorService.shutdown();
|
||||
try {
|
||||
if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
|
||||
executorService.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
executorService.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void initializeExecutorService() {
|
||||
executorService = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r);
|
||||
t.setDaemon(true);
|
||||
t.setName("com.alibaba.nacossync.basic.synctask");
|
||||
return t;
|
||||
});
|
||||
}
|
||||
|
||||
private void scheduleSyncTasks() {
|
||||
executorService.scheduleWithFixedDelay(this::executeSyncTasks, 0, 300, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private void executeSyncTasks() {
|
||||
if (allSyncTaskMap.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Collection<TaskDO> taskCollections = allSyncTaskMap.values();
|
||||
List<TaskDO> taskDOList = new ArrayList<>(taskCollections);
|
||||
|
||||
if (CollectionUtils.isNotEmpty(taskDOList)) {
|
||||
BatchTaskExecutor.batchOperation(taskDOList, this::executeTask);
|
||||
}
|
||||
}
|
||||
|
||||
private void executeTask(TaskDO task) {
|
||||
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||
String taskId = task.getTaskId();
|
||||
try {
|
||||
NamingService sourceNamingService = nacosServerHolder.get(task.getSourceClusterId());
|
||||
NamingService destNamingService = nacosServerHolder.get(task.getDestClusterId());
|
||||
doSync(taskId, task, sourceNamingService, destNamingService);
|
||||
} catch (NacosException e) {
|
||||
log.error("sync task from nacos to nacos failed, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
} catch (Exception e) {
|
||||
log.error("Unexpected error during sync task, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
} finally {
|
||||
stopwatch.stop();
|
||||
log.debug("Task execution time for taskId {}: {} ms", taskId, stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
||||
}
|
||||
}
|
||||
|
||||
public class NacosSyncToNacosServiceImpl implements SyncService {
|
||||
|
||||
private Map<String, EventListener> listenerMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<String, Set<String>> sourceInstanceSnapshot = new ConcurrentHashMap<>();
|
||||
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
@Autowired
|
||||
private SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
@Autowired
|
||||
private NacosServerHolder nacosServerHolder;
|
||||
|
||||
@Override
|
||||
public boolean delete(TaskDO taskDO) {
|
||||
try {
|
||||
NamingService sourceNamingService = nacosServerHolder.get(taskDO.getSourceClusterId());
|
||||
int level = clusterAccessService.findClusterLevel(taskDO.getSourceClusterId());
|
||||
String taskId = taskDO.getTaskId();
|
||||
|
||||
//Handle individual service
|
||||
if (StringUtils.isEmpty(taskId)) {
|
||||
log.warn("taskId is null data synchronization is not currently performed.{}", taskId);
|
||||
return false;
|
||||
}
|
||||
EventListener listener = listenerMap.remove(taskId);
|
||||
if (listener!= null) {
|
||||
sourceNamingService.unsubscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()), listener);
|
||||
}
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), new ArrayList<>(), false);
|
||||
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), taskDO.getNameSpace());
|
||||
//移除订阅
|
||||
sourceNamingService
|
||||
.unsubscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
listenerMap.remove(taskDO.getTaskId()));
|
||||
sourceInstanceSnapshot.remove(taskDO.getTaskId());
|
||||
|
||||
// 删除目标集群中同步的实例列表
|
||||
List<Instance> sourceInstances = sourceNamingService
|
||||
.getAllInstances(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
new ArrayList<>(), false);
|
||||
for (Instance instance : sourceInstances) {
|
||||
if (needSync(instance.getMetadata(), level, taskDO.getDestClusterId())) {
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), instance);
|
||||
if (needSync(instance.getMetadata())) {
|
||||
destNamingService
|
||||
.deregisterInstance(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
instance.getIp(),
|
||||
instance.getPort());
|
||||
}
|
||||
}
|
||||
// Remove all tasks that need to be synchronized.
|
||||
allSyncTaskMap.remove(taskId);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("delete task from nacos to nacos was failed, operationalId:{}", taskDO.getOperationId(), e);
|
||||
log.error("delete task from nacos to nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.DELETE_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean sync(TaskDO taskDO, Integer index) {
|
||||
log.info("Thread {} started synchronization at {}", Thread.currentThread().getId(), System.currentTimeMillis());
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
String taskId = taskDO.getTaskId();
|
||||
try {
|
||||
NamingService sourceNamingService = nacosServerHolder.get(taskDO.getSourceClusterId());
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
allSyncTaskMap.put(taskId, taskDO);
|
||||
// To prevent issues where tasks paused for synchronization, newly created tasks after deletion,
|
||||
// or resynchronization tasks do not receive new events and hence cannot synchronize,
|
||||
// perform a full synchronization of tasks before subscribing to events each time.
|
||||
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||
doSync(taskId, taskDO, sourceNamingService, destNamingService);
|
||||
log.debug("Time taken to synchronize a service registration: {} ms",
|
||||
stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), taskDO.getNameSpace());
|
||||
|
||||
this.listenerMap.putIfAbsent(taskId, event -> {
|
||||
if (event instanceof NamingEvent) {
|
||||
NamingEvent namingEvent = (NamingEvent) event;
|
||||
log.info("Detected changes in service {} information, taskId: {}, number of instances: {}, initiating synchronization",
|
||||
namingEvent.getServiceName(), taskId,
|
||||
namingEvent.getInstances() == null ? null : namingEvent.getInstances().size());
|
||||
try {
|
||||
doSync(taskId, taskDO, sourceNamingService, destNamingService);
|
||||
log.info("Detected synchronization end for service {}", namingEvent.getServiceName());
|
||||
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
new ArrayList<>(), false);
|
||||
// 先删除不存在的
|
||||
this.removeInvalidInstance(taskDO, destNamingService, sourceInstances);
|
||||
// 如果同步实例已经为空代表该服务所有实例已经下线,清除本地持有快照
|
||||
if (sourceInstances.isEmpty()) {
|
||||
sourceInstanceSnapshot.remove(taskId);
|
||||
return;
|
||||
}
|
||||
// 同步实例
|
||||
this.syncNewInstance(taskDO, destNamingService, sourceInstances);
|
||||
} catch (Exception e) {
|
||||
log.error("event process fail, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
sourceNamingService.subscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
listenerMap.get(taskId));
|
||||
listenerMap.get(taskId));
|
||||
} catch (Exception e) {
|
||||
log.error("sync task from nacos to nacos was failed, taskId:{}", taskId, e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
|
@ -231,192 +135,63 @@ public class NacosSyncToNacosServiceImpl implements SyncService, InitializingBea
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
private void doSync(String taskId, TaskDO taskDO, NamingService sourceNamingService,
|
||||
NamingService destNamingService) throws NacosException {
|
||||
if (syncTaskTap.putIfAbsent(taskId, 1) != null) {
|
||||
log.info("Task ID:{} - the previous synchronization task has not finished yet", taskId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), new ArrayList<>(), true);
|
||||
|
||||
int level = clusterAccessService.findClusterLevel(taskDO.getSourceClusterId());
|
||||
if (CollectionUtils.isNotEmpty(sourceInstances) && sourceInstances.get(0).isEphemeral()) {
|
||||
// Handle batch data synchronization of ephemeral instances, need to get all current service instances.
|
||||
// TODO: When the Client is version 1.x, execute the same synchronization method as persistent instances.
|
||||
handlerPersistenceInstance(taskDO, destNamingService, sourceInstances, level);
|
||||
} else if (CollectionUtils.isEmpty(sourceInstances)) {
|
||||
// If the current source cluster is empty, then directly deregister the instances in the target cluster.
|
||||
log.debug("service {} needs to sync ephemeral instance num is null: serviceName ", taskDO.getServiceName());
|
||||
processDeRegisterInstances(taskDO, destNamingService);
|
||||
} else {
|
||||
// Handle batch data synchronization of persistent instances.
|
||||
handlerPersistenceInstance(taskDO, destNamingService, sourceInstances, level);
|
||||
}
|
||||
} finally {
|
||||
syncTaskTap.remove(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
private void handlerPersistenceInstance(TaskDO taskDO, NamingService destNamingService,
|
||||
List<Instance> sourceInstances, int level) throws NacosException {
|
||||
List<Instance> needRegisterInstance = new ArrayList<>();
|
||||
|
||||
private void syncNewInstance(TaskDO taskDO, NamingService destNamingService,
|
||||
List<Instance> sourceInstances) throws NacosException {
|
||||
Set<String> latestSyncInstance = new TreeSet<>();
|
||||
//再次添加新实例
|
||||
String taskId = taskDO.getTaskId();
|
||||
Set<String> instanceKeys = sourceInstanceSnapshot.get(taskId);
|
||||
for (Instance instance : sourceInstances) {
|
||||
if (needSync(instance.getMetadata(), level, taskDO.getDestClusterId())) {
|
||||
needRegisterInstance.add(buildSyncInstance(instance, taskDO));
|
||||
if (needSync(instance.getMetadata())) {
|
||||
String instanceKey = composeInstanceKey(instance);
|
||||
if (CollectionUtils.isEmpty(instanceKeys) || !instanceKeys.contains(instanceKey)) {
|
||||
destNamingService.registerInstance(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
buildSyncInstance(instance, taskDO));
|
||||
}
|
||||
latestSyncInstance.add(instanceKey);
|
||||
|
||||
}
|
||||
}
|
||||
List<Instance> destAllInstances = destNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), new ArrayList<>(), true);
|
||||
|
||||
// Get the instances that the destination cluster has already synchronized
|
||||
List<Instance> destHasSyncInstances = destAllInstances.stream()
|
||||
.filter(instance -> hasSync(instance, taskDO.getSourceClusterId())).collect(Collectors.toList());
|
||||
|
||||
// The following two conversions are necessary because the Nacos Instance's equals method
|
||||
// is flawed and cannot be used for direct comparison.
|
||||
// The reason is that Instance's equals method compares Metadata using the toString method,
|
||||
// and Metadata is of type HashMap, which does not guarantee order in its toString representation.
|
||||
|
||||
// Convert destHasSyncInstances to a Map with the concatenated string as key
|
||||
Map<String, Instance> destInstanceMap = destHasSyncInstances.stream()
|
||||
.collect(Collectors.toMap(NacosSyncToNacosServiceImpl::getInstanceKey, instance -> instance));
|
||||
|
||||
// Convert newInstances to a Map with the concatenated string as key
|
||||
Map<String, Instance> needRegisterMap = needRegisterInstance.stream()
|
||||
.collect(Collectors.toMap(NacosSyncToNacosServiceImpl::getInstanceKey, instance -> instance));
|
||||
|
||||
// Remove instances from newInstanceMap that are present in destInstanceMap
|
||||
List<Instance> newInstances = removeSyncedInstances(destInstanceMap, needRegisterMap);
|
||||
|
||||
// Remove instances from destInstanceMap that are present in newInstanceMap
|
||||
List<Instance> invalidInstances = getInvalidInstances(destInstanceMap, needRegisterMap);
|
||||
|
||||
// Register each instance one by one. Take one instance at a time.
|
||||
for (Instance newInstance : newInstances) {
|
||||
destNamingService.registerInstance(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
newInstance);
|
||||
}
|
||||
|
||||
if (CollectionUtils.isNotEmpty(invalidInstances)) {
|
||||
log.info("taskId: {}, service {} deregistered, number of executions: {}", taskDO.getTaskId(), taskDO.getServiceName(),
|
||||
destHasSyncInstances.size());
|
||||
}
|
||||
for (Instance instance : invalidInstances) {
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
instance);
|
||||
if (CollectionUtils.isNotEmpty(latestSyncInstance)) {
|
||||
|
||||
log.info("任务Id:{},已同步实例个数:{}", taskId, latestSyncInstance.size());
|
||||
sourceInstanceSnapshot.put(taskId, latestSyncInstance);
|
||||
}
|
||||
}
|
||||
|
||||
private List<Instance> getInvalidInstances(Map<String, Instance> destInstanceMap, Map<String, Instance> needRegisterMap) {
|
||||
Map<String, Instance> destClone = new HashMap<>(destInstanceMap);
|
||||
|
||||
// Convert newInstances to a Map with the concatenated string as key
|
||||
Map<String, Instance> needRegisterClone = new HashMap<>(needRegisterMap);
|
||||
|
||||
// Remove instances from newInstanceMap that are present in destInstanceMap
|
||||
destClone.keySet().removeAll(needRegisterClone.keySet());
|
||||
|
||||
return new ArrayList<>(destClone.values());
|
||||
}
|
||||
|
||||
|
||||
public List<Instance> removeSyncedInstances(Map<String, Instance> destInstanceMap, Map<String, Instance> needRegisterMap) {
|
||||
// Convert destHasSyncInstances to a Map with the concatenated string as key
|
||||
|
||||
Map<String, Instance> destClone = new HashMap<>(destInstanceMap);
|
||||
|
||||
// Convert newInstances to a Map with the concatenated string as key
|
||||
Map<String, Instance> needRegisterClone = new HashMap<>(needRegisterMap);
|
||||
|
||||
// Remove instances from newInstanceMap that are present in destInstanceMap
|
||||
needRegisterClone.keySet().removeAll(destClone.keySet());
|
||||
|
||||
return new ArrayList<>(needRegisterClone.values());
|
||||
}
|
||||
|
||||
|
||||
private boolean hasSync(Instance instance, String sourceClusterId) {
|
||||
if (instance.getMetadata() != null) {
|
||||
String sourceClusterKey = instance.getMetadata().get(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY);
|
||||
return sourceClusterKey != null && sourceClusterKey.equals(sourceClusterId);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* When the number of instances that the source cluster needs to synchronize is 0,
|
||||
* if the target cluster still has instances synchronized with the source cluster,
|
||||
* perform unregistration.
|
||||
*
|
||||
* @param destNamingService Destination cluster naming service
|
||||
* @throws NacosException
|
||||
*/
|
||||
private void processDeRegisterInstances(TaskDO taskDO, NamingService destNamingService) throws NacosException {
|
||||
// If the instances in sourceInstances are empty, it means the instances are offline or do not exist.
|
||||
List<Instance> destInstances = destNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), new ArrayList<>(), false);
|
||||
// If the instances in the target cluster are also empty, then no operation is needed.
|
||||
if (CollectionUtils.isEmpty(destInstances)) {
|
||||
return;
|
||||
}
|
||||
destInstances = filterInstancesForRemoval(destInstances, taskDO.getSourceClusterId());
|
||||
if (CollectionUtils.isNotEmpty(destInstances)) {
|
||||
// Deregister each instance one by one. Take one instance at a time.
|
||||
// Need to handle redo, otherwise, it will be registered again.
|
||||
for (Instance destInstance : destInstances) {
|
||||
destNamingService.deregisterInstance(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), destInstance);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<Instance> filterInstancesForRemoval(List<Instance> destInstances, String sourceClusterId) {
|
||||
return destInstances.stream().filter(instance -> !instance.getMetadata().isEmpty())
|
||||
.filter(instance -> needDeregister(instance.getMetadata().get(SOURCE_CLUSTER_ID_KEY), sourceClusterId))
|
||||
|
||||
|
||||
private void removeInvalidInstance(TaskDO taskDO, NamingService destNamingService,
|
||||
List<Instance> sourceInstances) throws NacosException {
|
||||
String taskId = taskDO.getTaskId();
|
||||
if (this.sourceInstanceSnapshot.containsKey(taskId)) {
|
||||
Set<String> oldInstanceKeys = this.sourceInstanceSnapshot.get(taskId);
|
||||
List<String> newInstanceKeys = sourceInstances.stream().map(this::composeInstanceKey)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
}
|
||||
|
||||
private boolean needDeregister(String destClusterId, String sourceClusterId) {
|
||||
if (!StringUtils.isEmpty(destClusterId)) {
|
||||
return destClusterId.equals(sourceClusterId);
|
||||
Collection<String> instanceKeys = Collections.subtract(oldInstanceKeys, newInstanceKeys);
|
||||
for (String instanceKey : instanceKeys) {
|
||||
log.info("任务Id:{},移除无效同步实例:{}", taskId, instanceKey);
|
||||
String[] split = instanceKey.split(":", -1);
|
||||
destNamingService
|
||||
.deregisterInstance(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()), split[0],
|
||||
Integer.parseInt(split[1]));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean needSync(Map<String, String> sourceMetaData, int level, String destClusterId) {
|
||||
// Regular cluster (default)
|
||||
if (level == 0) {
|
||||
return SyncService.super.needSync(sourceMetaData);
|
||||
}
|
||||
// Central cluster, as long as the instance is not from the target cluster,
|
||||
// it needs to be synchronized (extended functionality)
|
||||
return !destClusterId.equals(sourceMetaData.get(SOURCE_CLUSTER_ID_KEY));
|
||||
|
||||
private String composeInstanceKey(Instance instance) {
|
||||
return instance.getIp() + ":" + instance.getPort();
|
||||
}
|
||||
|
||||
|
||||
|
||||
private String getGroupNameOrDefault(String groupName) {
|
||||
return StringUtils.defaultIfBlank(groupName, Constants.DEFAULT_GROUP);
|
||||
}
|
||||
|
||||
private Instance buildSyncInstance(Instance instance, TaskDO taskDO) {
|
||||
Instance temp = getInstance(instance);
|
||||
temp.addMetadata(SkyWalkerConstants.DEST_CLUSTER_ID_KEY, taskDO.getDestClusterId());
|
||||
temp.addMetadata(SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
temp.addMetadata(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY, taskDO.getSourceClusterId());
|
||||
//The flag is a synchronous instance
|
||||
temp.addMetadata(SkyWalkerConstants.SYNC_INSTANCE_TAG,
|
||||
taskDO.getSourceClusterId() + "@@" + taskDO.getVersion());
|
||||
return temp;
|
||||
}
|
||||
|
||||
private static Instance getInstance(Instance instance) {
|
||||
Instance temp = new Instance();
|
||||
temp.setInstanceId(instance.getInstanceId());
|
||||
temp.setIp(instance.getIp());
|
||||
temp.setPort(instance.getPort());
|
||||
temp.setClusterName(instance.getClusterName());
|
||||
|
@ -425,19 +200,15 @@ public class NacosSyncToNacosServiceImpl implements SyncService, InitializingBea
|
|||
temp.setHealthy(instance.isHealthy());
|
||||
temp.setWeight(instance.getWeight());
|
||||
temp.setEphemeral(instance.isEphemeral());
|
||||
Map<String, String> metaData = new HashMap<>(instance.getMetadata());
|
||||
Map<String, String> metaData = new HashMap<>();
|
||||
metaData.putAll(instance.getMetadata());
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTERID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTERID_KEY, taskDO.getSourceClusterId());
|
||||
temp.setMetadata(metaData);
|
||||
return temp;
|
||||
}
|
||||
|
||||
private static String getInstanceKey(Instance instance) {
|
||||
return String.join("|",
|
||||
instance.getIp(),
|
||||
String.valueOf(instance.getPort()),
|
||||
String.valueOf(instance.getWeight()),
|
||||
String.valueOf(instance.isHealthy()),
|
||||
String.valueOf(instance.isEphemeral()),
|
||||
instance.getClusterName(),
|
||||
instance.getServiceName());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -25,9 +25,12 @@ import com.alibaba.nacossync.extension.SyncService;
|
|||
import com.alibaba.nacossync.extension.annotation.NacosSyncService;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.extension.holder.ZookeeperServerHolder;
|
||||
import com.alibaba.nacossync.extension.impl.extend.NacosSyncToZookeeperServicesSharding;
|
||||
import com.alibaba.nacossync.extension.impl.extend.Sharding;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.DubboConstants;
|
||||
import com.alibaba.nacossync.util.ExpirySet;
|
||||
import com.google.common.collect.Sets;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
|
@ -35,17 +38,15 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCache;
|
|||
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
|
||||
import org.apache.curator.utils.CloseableUtils;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import static com.alibaba.nacossync.util.NacosUtils.getGroupNameOrDefault;
|
||||
import static com.alibaba.nacossync.util.StringUtils.convertDubboFullPathForZk;
|
||||
import static com.alibaba.nacossync.util.StringUtils.convertDubboProvidersPath;
|
||||
|
||||
|
@ -59,7 +60,8 @@ import static com.alibaba.nacossync.util.StringUtils.convertDubboProvidersPath;
|
|||
@NacosSyncService(sourceCluster = ClusterTypeEnum.NACOS, destinationCluster = ClusterTypeEnum.ZK)
|
||||
public class NacosSyncToZookeeperServiceImpl implements SyncService {
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
/**
|
||||
* @description The Nacos listener map.
|
||||
|
@ -91,33 +93,37 @@ public class NacosSyncToZookeeperServiceImpl implements SyncService {
|
|||
|
||||
private final ZookeeperServerHolder zookeeperServerHolder;
|
||||
|
||||
|
||||
private static ExpirySet<String> serviceNameSet = new ExpirySet<String>();
|
||||
|
||||
private static ExecutorService EXECUTOR = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
|
||||
@Resource(type = NacosSyncToZookeeperServicesSharding.class)
|
||||
private Sharding sharding;
|
||||
|
||||
@Autowired
|
||||
public NacosSyncToZookeeperServiceImpl(SkyWalkerCacheServices skyWalkerCacheServices,
|
||||
NacosServerHolder nacosServerHolder, ZookeeperServerHolder zookeeperServerHolder, MetricsManager metricsManager) {
|
||||
NacosServerHolder nacosServerHolder, ZookeeperServerHolder zookeeperServerHolder) {
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.zookeeperServerHolder = zookeeperServerHolder;
|
||||
this.metricsManager = metricsManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(TaskDO taskDO) {
|
||||
try {
|
||||
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId());
|
||||
//nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());//
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());//fix with no nameSpaceName
|
||||
EventListener eventListener = nacosListenerMap.remove(taskDO.getTaskId());
|
||||
PathChildrenCache pathChildrenCache = pathChildrenCacheMap.get(taskDO.getTaskId());
|
||||
sourceNamingService.unsubscribe(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
eventListener);
|
||||
sourceNamingService.unsubscribe(taskDO.getServiceName(), eventListener);
|
||||
CloseableUtils.closeQuietly(pathChildrenCache);
|
||||
Set<String> instanceUrlSet = instanceBackupMap.get(taskDO.getTaskId());
|
||||
CuratorFramework client = zookeeperServerHolder.get(taskDO.getDestClusterId());
|
||||
if(!instanceUrlSet.isEmpty()){
|
||||
for (String instanceUrl : instanceUrlSet) {
|
||||
client.delete().quietly().forPath(instanceUrl);
|
||||
}
|
||||
CuratorFramework client = zookeeperServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
|
||||
for (String instanceUrl : instanceUrlSet) {
|
||||
client.delete().quietly().forPath(instanceUrl);
|
||||
}
|
||||
sharding.stop(taskDO);
|
||||
} catch (Exception e) {
|
||||
log.error("delete task from nacos to zk was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.DELETE_ERROR);
|
||||
|
@ -127,88 +133,39 @@ public class NacosSyncToZookeeperServiceImpl implements SyncService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean sync(TaskDO taskDO, Integer index) {
|
||||
try {
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId());
|
||||
CuratorFramework client = zookeeperServerHolder.get(taskDO.getDestClusterId());
|
||||
nacosListenerMap.putIfAbsent(taskDO.getTaskId(), event -> {
|
||||
if (event instanceof NamingEvent) {
|
||||
try {
|
||||
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), new ArrayList<>(), true);
|
||||
Set<String> newInstanceUrlSet = getWaitingToAddInstance(taskDO, client, sourceInstances);
|
||||
|
||||
// fetch the instance backup
|
||||
deleteInvalidInstances(taskDO, client, newInstanceUrlSet);
|
||||
// replace the instance backup
|
||||
instanceBackupMap.put(taskDO.getTaskId(), newInstanceUrlSet);
|
||||
// try to compensate for the removed instance
|
||||
tryToCompensate(taskDO, sourceNamingService, sourceInstances);
|
||||
} catch (Exception e) {
|
||||
log.error("event process fail, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
sourceNamingService.subscribe(taskDO.getServiceName(),getGroupNameOrDefault(taskDO.getGroupName()),
|
||||
nacosListenerMap.get(taskDO.getTaskId()));
|
||||
} catch (Exception e) {
|
||||
log.error("sync task from nacos to zk was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
}
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
sharding.start(taskDO);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
private void tryToCompensate(TaskDO taskDO, NamingService sourceNamingService, List<Instance> sourceInstances) {
|
||||
if (!CollectionUtils.isEmpty(sourceInstances)) {
|
||||
final PathChildrenCache pathCache = getPathCache(taskDO);
|
||||
// Avoiding re-registration if there is already a listener registered.
|
||||
if (pathCache.getListenable().size() == 0) {
|
||||
registerCompensationListener(pathCache, taskDO, sourceNamingService);
|
||||
if (pathCache.getListenable().size() == 0) { // 防止重复注册
|
||||
pathCache.getListenable().addListener((zkClient, zkEvent) -> {
|
||||
if (zkEvent.getType() == PathChildrenCacheEvent.Type.CHILD_REMOVED) {
|
||||
List<Instance> allInstances =
|
||||
sourceNamingService.getAllInstances(taskDO.getServiceName());
|
||||
for (Instance instance : allInstances) {
|
||||
String instanceUrl = buildSyncInstance(instance, taskDO);
|
||||
String zkInstancePath = zkEvent.getData().getPath();
|
||||
if (zkInstancePath.equals(instanceUrl)) {
|
||||
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL)
|
||||
.forPath(zkInstancePath);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void registerCompensationListener(PathChildrenCache pathCache, TaskDO taskDO, NamingService sourceNamingService) {
|
||||
pathCache.getListenable().addListener((zkClient, zkEvent) -> {
|
||||
try {
|
||||
if (zkEvent.getType() == PathChildrenCacheEvent.Type.CHILD_REMOVED) {
|
||||
compensateOnChildRemoval(zkClient, zkEvent, sourceNamingService, taskDO);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error processing ZooKeeper event: {}", zkEvent.getType(), e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void compensateOnChildRemoval(CuratorFramework zkClient, PathChildrenCacheEvent zkEvent, NamingService sourceNamingService, TaskDO taskDO) {
|
||||
String zkInstancePath = null;
|
||||
try {
|
||||
List<Instance> allInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(), getGroupNameOrDefault(taskDO.getGroupName()));
|
||||
zkInstancePath = zkEvent.getData().getPath();
|
||||
for (Instance instance : allInstances) {
|
||||
String instanceUrl = buildSyncInstance(instance, taskDO);
|
||||
if (zkInstancePath.equals(instanceUrl)) {
|
||||
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL)
|
||||
.forPath(zkInstancePath);
|
||||
log.info("Compensated by re-creating the removed node at path: {}", zkInstancePath);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to compensate for the removed node at path: {}", zkInstancePath, e);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteInvalidInstances(TaskDO taskDO, CuratorFramework client, Set<String> newInstanceUrlSet)
|
||||
throws Exception {
|
||||
throws Exception {
|
||||
Set<String> instanceBackup =
|
||||
instanceBackupMap.getOrDefault(taskDO.getTaskId(), Sets.newHashSet());
|
||||
instanceBackupMap.getOrDefault(taskDO.getTaskId(), Sets.newHashSet());
|
||||
for (String instanceUrl : instanceBackup) {
|
||||
if (newInstanceUrlSet.contains(instanceUrl)) {
|
||||
continue;
|
||||
|
@ -218,15 +175,17 @@ public class NacosSyncToZookeeperServiceImpl implements SyncService {
|
|||
}
|
||||
|
||||
private HashSet<String> getWaitingToAddInstance(TaskDO taskDO, CuratorFramework client,
|
||||
List<Instance> sourceInstances) throws Exception {
|
||||
List<Instance> sourceInstances) throws Exception {
|
||||
HashSet<String> waitingToAddInstance = new HashSet<>();
|
||||
for (Instance instance : sourceInstances) {
|
||||
if (needSync(instance.getMetadata())) {
|
||||
log.info("nacos->zk ,real sync service :{},and instance :{}", instance.getServiceName(), instance.getIp());
|
||||
String instanceUrl = buildSyncInstance(instance, taskDO);
|
||||
if (null == client.checkExists().forPath(instanceUrl)) {
|
||||
client.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL)
|
||||
.forPath(instanceUrl);
|
||||
if (null != client.checkExists().forPath(instanceUrl)) {
|
||||
client.delete().quietly().forPath(instanceUrl);
|
||||
}
|
||||
client.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL)
|
||||
.forPath(instanceUrl);
|
||||
waitingToAddInstance.add(instanceUrl);
|
||||
}
|
||||
}
|
||||
|
@ -234,30 +193,31 @@ public class NacosSyncToZookeeperServiceImpl implements SyncService {
|
|||
}
|
||||
|
||||
protected String buildSyncInstance(Instance instance, TaskDO taskDO) throws UnsupportedEncodingException {
|
||||
Map<String, String> metaData = new HashMap<>(instance.getMetadata());
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTER_ID_KEY, taskDO.getDestClusterId());
|
||||
Map<String, String> metaData = new HashMap<>();
|
||||
metaData.putAll(instance.getMetadata());
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTERID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY, taskDO.getSourceClusterId());
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTERID_KEY, taskDO.getSourceClusterId());
|
||||
|
||||
String servicePath = monitorPath.computeIfAbsent(taskDO.getTaskId(),
|
||||
key -> convertDubboProvidersPath(metaData.get(DubboConstants.INTERFACE_KEY)));
|
||||
key -> convertDubboProvidersPath(metaData.get(DubboConstants.INTERFACE_KEY)));
|
||||
|
||||
return convertDubboFullPathForZk(metaData, servicePath, instance.getIp(), instance.getPort());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* fetch zk path cache
|
||||
* 获取zk path child 监听缓存类
|
||||
*
|
||||
* @param taskDO task instance
|
||||
* @return zk path cache
|
||||
* @param taskDO 任务对象
|
||||
* @return zk节点操作缓存对象
|
||||
*/
|
||||
private PathChildrenCache getPathCache(TaskDO taskDO) {
|
||||
return pathChildrenCacheMap.computeIfAbsent(taskDO.getTaskId(), (key) -> {
|
||||
try {
|
||||
PathChildrenCache pathChildrenCache = new PathChildrenCache(
|
||||
zookeeperServerHolder.get(taskDO.getDestClusterId()), monitorPath.get(key), false);
|
||||
zookeeperServerHolder.get(taskDO.getDestClusterId(), ""), monitorPath.get(key), false);
|
||||
pathChildrenCache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);
|
||||
return pathChildrenCache;
|
||||
} catch (Exception e) {
|
||||
|
@ -269,4 +229,74 @@ public class NacosSyncToZookeeperServiceImpl implements SyncService {
|
|||
}
|
||||
|
||||
|
||||
private class SyncThread implements Runnable {
|
||||
|
||||
NamingService sourceNamingService;
|
||||
|
||||
TaskDO taskDO;
|
||||
|
||||
CuratorFramework client;
|
||||
|
||||
SyncThread(NamingService sourceNamingService, TaskDO taskDO, CuratorFramework client) {
|
||||
this.sourceNamingService = sourceNamingService;
|
||||
this.taskDO = taskDO;
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
|
||||
//List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName());
|
||||
List<Instance> sourceInstances = sourceNamingService.getAllInstances(taskDO.getServiceName(), taskDO.getGroupName());//fix with no group
|
||||
Set<String> newInstanceUrlSet = getWaitingToAddInstance(taskDO, client, sourceInstances);
|
||||
|
||||
// 获取之前的备份 删除无效实例
|
||||
deleteInvalidInstances(taskDO, client, newInstanceUrlSet);
|
||||
// 替换当前备份为最新备份
|
||||
instanceBackupMap.put(taskDO.getTaskId(), newInstanceUrlSet);
|
||||
// 尝试恢复因为zk客户端意外断开导致的实例数据
|
||||
tryToCompensate(taskDO, sourceNamingService, filterNeedSync(sourceInstances));
|
||||
} catch (Exception e) {
|
||||
log.error("event process fail, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
|
||||
} finally {
|
||||
//serviceNameSet.remove(((NamingEvent) event).getServiceName());//如果考虑高实时性 可以手动remove 这样时间窗口的大小就不固定 依赖处理速度 窗口大小作为兜底
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<Instance> filterNeedSync(List<Instance> sourceInstances) {
|
||||
Iterator<Instance> iterator = sourceInstances.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
if (!needSync(iterator.next().getMetadata())) {
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
return sourceInstances;
|
||||
}
|
||||
|
||||
public boolean addSyncService(TaskDO taskDO) {
|
||||
try {
|
||||
NamingService sourceNamingService =
|
||||
//nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getGroupName());
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());//fix with no nameSpaceName
|
||||
CuratorFramework client = zookeeperServerHolder.get(taskDO.getDestClusterId(), taskDO.getGroupName());
|
||||
nacosListenerMap.putIfAbsent(taskDO.getTaskId(), event -> {
|
||||
if (event instanceof NamingEvent) {
|
||||
if (serviceNameSet.set(((NamingEvent) event).getServiceName())) {// add event merge
|
||||
EXECUTOR.execute(new SyncThread(sourceNamingService, taskDO, client));
|
||||
}
|
||||
}
|
||||
});
|
||||
sourceNamingService.subscribe(taskDO.getServiceName(), nacosListenerMap.get(taskDO.getTaskId()));
|
||||
} catch (Exception e) {
|
||||
log.error("sync task from nacos to zk was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -10,21 +10,25 @@
|
|||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.extension.impl;
|
||||
|
||||
import com.alibaba.nacos.api.exception.NacosException;
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.pojo.Instance;
|
||||
import com.alibaba.nacos.client.naming.NacosNamingService;
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.constant.MetricsStatisticsType;
|
||||
import com.alibaba.nacossync.constant.ShardingLogTypeEnum;
|
||||
import com.alibaba.nacossync.constant.SkyWalkerConstants;
|
||||
import com.alibaba.nacossync.extension.SyncService;
|
||||
import com.alibaba.nacossync.extension.annotation.NacosSyncService;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.extension.holder.ZookeeperServerHolder;
|
||||
import com.alibaba.nacossync.extension.impl.extend.Sharding;
|
||||
import com.alibaba.nacossync.extension.impl.extend.ZookeeperSyncToNacosServiceSharding;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.ShardingLog;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
@ -32,30 +36,16 @@ import org.apache.curator.framework.CuratorFramework;
|
|||
import org.apache.curator.framework.recipes.cache.TreeCache;
|
||||
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
|
||||
import org.apache.curator.utils.CloseableUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import javax.annotation.Resource;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static com.alibaba.nacossync.util.DubboConstants.ALL_SERVICE_NAME_PATTERN;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.DUBBO_PATH_FORMAT;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.DUBBO_ROOT_PATH;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.GROUP_KEY;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.INSTANCE_IP_KEY;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.INSTANCE_PORT_KEY;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.INTERFACE_KEY;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.PROTOCOL_KEY;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.VERSION_KEY;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.WEIGHT_KEY;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.ZOOKEEPER_SEPARATOR;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.createServiceName;
|
||||
import static com.alibaba.nacossync.util.NacosUtils.getGroupNameOrDefault;
|
||||
import static com.alibaba.nacossync.util.DubboConstants.*;
|
||||
import static com.alibaba.nacossync.util.StringUtils.parseIpAndPortString;
|
||||
import static com.alibaba.nacossync.util.StringUtils.parseQueryString;
|
||||
|
||||
|
@ -67,55 +57,70 @@ import static com.alibaba.nacossync.util.StringUtils.parseQueryString;
|
|||
@Slf4j
|
||||
@NacosSyncService(sourceCluster = ClusterTypeEnum.ZK, destinationCluster = ClusterTypeEnum.NACOS)
|
||||
public class ZookeeperSyncToNacosServiceImpl implements SyncService {
|
||||
|
||||
private static final String DEFAULT_WEIGHT = "1.0";
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
|
||||
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
/**
|
||||
* Listener cache of Zookeeper format taskId -> PathChildrenCache instance
|
||||
*/
|
||||
private final Map<String, TreeCache> treeCacheMap = new ConcurrentHashMap<>();
|
||||
|
||||
private Map<String, TreeCache> treeCacheMap = new ConcurrentHashMap<>();
|
||||
/**
|
||||
* service name cache
|
||||
*/
|
||||
private final Map<String, String> nacosServiceNameMap = new ConcurrentHashMap<>();
|
||||
|
||||
private Map<String, String> nacosServiceNameMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final ZookeeperServerHolder zookeeperServerHolder;
|
||||
|
||||
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
|
||||
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
|
||||
|
||||
@Resource(type = ZookeeperSyncToNacosServiceSharding.class)
|
||||
private Sharding sharding;
|
||||
|
||||
//排除/dobbo下面的所有非服务节点
|
||||
private static final List<String> IGNORED_DUBBO_PATH = Stream.of("mapping", "metadata", "yellow").collect(Collectors.toList());
|
||||
|
||||
@Autowired
|
||||
public ZookeeperSyncToNacosServiceImpl(ZookeeperServerHolder zookeeperServerHolder,
|
||||
NacosServerHolder nacosServerHolder, SkyWalkerCacheServices skyWalkerCacheServices,
|
||||
MetricsManager metricsManager) {
|
||||
NacosServerHolder nacosServerHolder, SkyWalkerCacheServices skyWalkerCacheServices) {
|
||||
this.zookeeperServerHolder = zookeeperServerHolder;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.metricsManager = metricsManager;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean sync(TaskDO taskDO, Integer index) {
|
||||
public boolean sync(TaskDO taskDO) {
|
||||
try {
|
||||
if (treeCacheMap.containsKey(taskDO.getTaskId())) {
|
||||
return true;
|
||||
}
|
||||
if (!initializeTreeCache(taskDO)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
if (destNamingService == null) {
|
||||
logAndRecordSyncError("Failed to obtain NamingService for destination clusterId: {}", taskDO.getDestClusterId(), null);
|
||||
return false;
|
||||
}
|
||||
TreeCache treeCache = getTreeCache(taskDO);
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), null);
|
||||
// 初次执行任务统一注册所有实例
|
||||
registerAllInstances(taskDO, destNamingService);
|
||||
setupListener(taskDO, destNamingService);
|
||||
//注册ZK监听
|
||||
Objects.requireNonNull(treeCache).getListenable().addListener((client, event) -> {
|
||||
try {
|
||||
String path = event.getData().getPath();
|
||||
if (!com.alibaba.nacossync.util.StringUtils.isDubboProviderPath(path)) {
|
||||
return;
|
||||
}
|
||||
Map<String, String> queryParam = parseQueryString(path);
|
||||
//add sharding
|
||||
if (!isProcess(taskDO, destNamingService, queryParam.get(INTERFACE_KEY)))
|
||||
return;
|
||||
if (isMatch(taskDO, queryParam) && needSync(queryParam)) {
|
||||
log.info("sync sharding Zookeeper to Nacos serviceName:{},local servicesName :{}", queryParam.get(INTERFACE_KEY), sharding.getLocalServices(null));
|
||||
processEvent(taskDO, destNamingService, event, path, queryParam);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("event process from Zookeeper to Nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
}
|
||||
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error("sync task from Zookeeper to Nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
|
@ -123,232 +128,59 @@ public class ZookeeperSyncToNacosServiceImpl implements SyncService {
|
|||
}
|
||||
return true;
|
||||
}
|
||||
private boolean initializeTreeCache(TaskDO taskDO) {
|
||||
TreeCache treeCache = getTreeCache(taskDO);
|
||||
if (treeCache == null) {
|
||||
logAndRecordSyncError("Failed to obtain TreeCache for taskId: {}", taskDO.getTaskId(), null);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
private void logAndRecordSyncError(String message, String taskId, Exception e) {
|
||||
if (e != null) {
|
||||
log.error(message, taskId, e);
|
||||
} else {
|
||||
log.error(message, taskId);
|
||||
}
|
||||
metricsManager.recordError(MetricsStatisticsType.SYNC_ERROR);
|
||||
}
|
||||
|
||||
private void setupListener(TaskDO taskDO, NamingService destNamingService) {
|
||||
TreeCache treeCache = Objects.requireNonNull(getTreeCache(taskDO));
|
||||
treeCache.getListenable().addListener((client, event) -> {
|
||||
try {
|
||||
// INITIALIZED is a special event that is not triggered by the Zookeeper server
|
||||
if(event.getData()==null){
|
||||
log.warn("TreeCache event data is null, taskId:{}", taskDO.getTaskId());
|
||||
return;
|
||||
}
|
||||
String path = event.getData().getPath();
|
||||
Map<String, String> queryParam = parseQueryString(path);
|
||||
if (isMatch(taskDO, queryParam) && needSync(queryParam)) {
|
||||
processEvent(taskDO, destNamingService, event, path, queryParam);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logAndRecordSyncError("Event process from Zookeeper to Nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
public boolean delete(TaskDO taskDO) {
|
||||
if (taskDO.getServiceName() == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
CloseableUtils.closeQuietly(treeCacheMap.get(taskDO.getTaskId()));
|
||||
|
||||
try {
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId());
|
||||
if (destNamingService == null) {
|
||||
log.error("Failed to obtain NamingService for destination clusterId: {}", taskDO.getDestClusterId());
|
||||
return false;
|
||||
}
|
||||
|
||||
Set<String> serviceNames = getServiceNamesToDelete(taskDO);
|
||||
deleteInstances(serviceNames, destNamingService, taskDO);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Delete task from Zookeeper to Nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.DELETE_ERROR);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private Set<String> getServiceNamesToDelete(TaskDO taskDO) {
|
||||
if (!ALL_SERVICE_NAME_PATTERN.equals(taskDO.getServiceName())) {
|
||||
return new HashSet<>(Collections.singleton(taskDO.getServiceName()));
|
||||
} else {
|
||||
return new HashSet<>(nacosServiceNameMap.keySet());
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteInstances(Set<String> serviceNames, NamingService destNamingService, TaskDO taskDO) {
|
||||
for (String serviceName : serviceNames) {
|
||||
try {
|
||||
List<Instance> allInstances = destNamingService.getAllInstances(serviceName,
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), new ArrayList<>(), true);
|
||||
for (Instance instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
destNamingService.deregisterInstance(instance.getServiceName(),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), instance.getIp(), instance.getPort());
|
||||
}
|
||||
}
|
||||
nacosServiceNameMap.remove(serviceName);
|
||||
} catch (NacosException e) {
|
||||
log.error("Failed to deregister service instance for serviceName: {}", serviceName, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* fetch the Path cache when the task sync
|
||||
*/
|
||||
protected TreeCache getTreeCache(TaskDO taskDO) {
|
||||
return treeCacheMap.computeIfAbsent(taskDO.getTaskId(), (key) -> {
|
||||
try {
|
||||
TreeCache treeCache = new TreeCache(zookeeperServerHolder.get(taskDO.getSourceClusterId()),
|
||||
DUBBO_ROOT_PATH);
|
||||
treeCache.start();
|
||||
return treeCache;
|
||||
} catch (Exception e) {
|
||||
log.error("zookeeper path children cache start failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the instance information needs to be synchronized based on the dubbo version, grouping name,
|
||||
* and service name.
|
||||
*/
|
||||
protected boolean isMatch(TaskDO taskDO, Map<String, String> queryParam) {
|
||||
return isVersionMatch(taskDO, queryParam) &&
|
||||
isGroupMatch(taskDO, queryParam) &&
|
||||
isServiceMatch(taskDO, queryParam) ||
|
||||
isMatchAllServices(taskDO);
|
||||
}
|
||||
|
||||
private boolean isVersionMatch(TaskDO task, Map<String, String> queryParam) {
|
||||
return StringUtils.isBlank(task.getVersion()) || StringUtils.equals(task.getVersion(), queryParam.get(VERSION_KEY));
|
||||
}
|
||||
|
||||
private boolean isGroupMatch(TaskDO task, Map<String, String> queryParam) {
|
||||
return StringUtils.isBlank(task.getGroupName()) || StringUtils.equals(task.getGroupName(), queryParam.get(GROUP_KEY));
|
||||
}
|
||||
|
||||
private boolean isServiceMatch(TaskDO task, Map<String, String> queryParam) {
|
||||
return StringUtils.isNotBlank(task.getServiceName()) && StringUtils.equals(task.getServiceName(), queryParam.get(INTERFACE_KEY));
|
||||
}
|
||||
|
||||
private boolean isMatchAllServices(TaskDO task) {
|
||||
return StringUtils.isNotBlank(task.getServiceName()) && StringUtils.equals(task.getServiceName(), ALL_SERVICE_NAME_PATTERN);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Builds a synchronized Nacos instance from Zookeeper data.
|
||||
*
|
||||
* @param queryParam Parameters obtained from the query string.
|
||||
* @param ipAndPortMap IP and port information.
|
||||
* @param taskDO Task details.
|
||||
* @return A fully configured Nacos instance.
|
||||
*/
|
||||
protected Instance buildSyncInstance(Map<String, String> queryParam, Map<String, String> ipAndPortMap, TaskDO taskDO) {
|
||||
Instance instance = new Instance();
|
||||
instance.setIp(ipAndPortMap.get(INSTANCE_IP_KEY));
|
||||
instance.setPort(Integer.parseInt(ipAndPortMap.get(INSTANCE_PORT_KEY)));
|
||||
instance.setServiceName(getServiceNameFromCache(taskDO.getTaskId(), queryParam));
|
||||
instance.setWeight(parseWeight(queryParam));
|
||||
instance.setHealthy(true);
|
||||
instance.setMetadata(buildMetadata(queryParam, ipAndPortMap, taskDO));
|
||||
return instance;
|
||||
}
|
||||
|
||||
|
||||
private double parseWeight(Map<String, String> queryParam) {
|
||||
try {
|
||||
return Double.parseDouble(queryParam.getOrDefault(WEIGHT_KEY, DEFAULT_WEIGHT));
|
||||
} catch (NumberFormatException e) {
|
||||
log.error("Error parsing weight: {}", queryParam.get(WEIGHT_KEY), e);
|
||||
return Double.parseDouble(DEFAULT_WEIGHT); // Default weight in case of error
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> buildMetadata(Map<String, String> queryParam, Map<String, String> ipAndPortMap, TaskDO taskDO) {
|
||||
Map<String, String> metaData = new HashMap<>(queryParam);
|
||||
metaData.put(PROTOCOL_KEY, ipAndPortMap.get(PROTOCOL_KEY));
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTER_ID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.SYNC_SOURCE_KEY, skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTER_ID_KEY, taskDO.getSourceClusterId());
|
||||
return metaData;
|
||||
}
|
||||
|
||||
|
||||
private void processEvent(TaskDO taskDO, NamingService destNamingService, TreeCacheEvent event, String path,
|
||||
Map<String, String> queryParam) throws NacosException {
|
||||
if (!com.alibaba.nacossync.util.StringUtils.isDubboProviderPath(path)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, String> queryParam) throws NacosException {
|
||||
|
||||
Map<String, String> ipAndPortParam = parseIpAndPortString(path);
|
||||
if (ipAndPortParam.isEmpty()) {
|
||||
log.error("Invalid IP and Port data extracted from path: {}", path);
|
||||
return;
|
||||
}
|
||||
Instance instance = buildSyncInstance(queryParam, ipAndPortParam, taskDO);
|
||||
String serviceName = queryParam.get(INTERFACE_KEY);
|
||||
if (serviceName == null || serviceName.isEmpty()) {
|
||||
log.error("Service name is missing in the query parameters.");
|
||||
return;
|
||||
}
|
||||
switch (event.getType()) {
|
||||
case NODE_ADDED:
|
||||
case NODE_UPDATED:
|
||||
|
||||
destNamingService.registerInstance(getServiceNameFromCache(serviceName, queryParam),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), instance);
|
||||
|
||||
destNamingService.registerInstance(
|
||||
getServiceNameFromCache(serviceName, queryParam), instance);
|
||||
//getServiceNameFromCache(serviceName, queryParam, instance), instance);
|
||||
log.info("syn add service : {} ,instance:{}", serviceName, instance);
|
||||
break;
|
||||
case NODE_REMOVED:
|
||||
|
||||
destNamingService.deregisterInstance(getServiceNameFromCache(serviceName, queryParam),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), ipAndPortParam.get(INSTANCE_IP_KEY),
|
||||
|
||||
destNamingService.deregisterInstance(
|
||||
getServiceNameFromCache(serviceName, queryParam),
|
||||
ipAndPortParam.get(INSTANCE_IP_KEY),
|
||||
Integer.parseInt(ipAndPortParam.get(INSTANCE_PORT_KEY)));
|
||||
nacosServiceNameMap.remove(serviceName);
|
||||
log.info("syn delete service : {} ,instance:{}", serviceName, instance);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void registerAllInstances(TaskDO taskDO, NamingService destNamingService) throws Exception {
|
||||
CuratorFramework zk = zookeeperServerHolder.get(taskDO.getSourceClusterId());
|
||||
CuratorFramework zk = zookeeperServerHolder.get(taskDO.getSourceClusterId(), "");
|
||||
sharding.start(taskDO);//幂等 可重复添加
|
||||
if (!ALL_SERVICE_NAME_PATTERN.equals(taskDO.getServiceName())) {
|
||||
registerALLInstances0(taskDO, destNamingService, zk, taskDO.getServiceName());
|
||||
sharding.doSharding(null, new ArrayList<>(Arrays.asList(taskDO.getServiceName())));
|
||||
TreeSet<String> shardingServices = sharding.getLocalServices(null);
|
||||
if (shardingServices.contains(taskDO.getServiceName())) {
|
||||
registerALLInstances0(taskDO, destNamingService, zk, taskDO.getServiceName());
|
||||
}
|
||||
} else {
|
||||
// 同步全部
|
||||
List<String> serviceList = zk.getChildren().forPath(DUBBO_ROOT_PATH);
|
||||
sharding.doSharding(null, filterNoProviderPath(serviceList));
|
||||
TreeSet<String> shardingServices = sharding.getLocalServices(null);
|
||||
for (String serviceName : serviceList) {
|
||||
registerALLInstances0(taskDO, destNamingService, zk, serviceName);
|
||||
if (shardingServices.contains(serviceName) && !IGNORED_DUBBO_PATH.contains(serviceName))//add
|
||||
registerALLInstances0(taskDO, destNamingService, zk, serviceName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void registerALLInstances0(TaskDO taskDO, NamingService destNamingService, CuratorFramework zk,
|
||||
String serviceName) throws Exception {
|
||||
String serviceName) throws Exception {
|
||||
String path = String.format(DUBBO_PATH_FORMAT, serviceName);
|
||||
if (zk.getChildren() == null) {
|
||||
return;
|
||||
|
@ -360,12 +192,117 @@ public class ZookeeperSyncToNacosServiceImpl implements SyncService {
|
|||
Map<String, String> ipAndPortParam = parseIpAndPortString(path + ZOOKEEPER_SEPARATOR + provider);
|
||||
Instance instance = buildSyncInstance(queryParam, ipAndPortParam, taskDO);
|
||||
destNamingService.registerInstance(getServiceNameFromCache(serviceName, queryParam),
|
||||
getGroupNameOrDefault(taskDO.getGroupName()), instance);
|
||||
instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public boolean delete(TaskDO taskDO) {
|
||||
if (taskDO.getServiceName() == null) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
|
||||
CloseableUtils.closeQuietly(treeCacheMap.get(taskDO.getTaskId()));
|
||||
NamingService destNamingService = nacosServerHolder.get(taskDO.getDestClusterId(), null);
|
||||
if (!ALL_SERVICE_NAME_PATTERN.equals(taskDO.getServiceName())) {
|
||||
if (nacosServiceNameMap.containsKey(taskDO.getServiceName())) {
|
||||
List<Instance> allInstances =
|
||||
destNamingService.getAllInstances(nacosServiceNameMap.get(taskDO.getServiceName()));
|
||||
for (Instance instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
destNamingService.deregisterInstance(instance.getServiceName(), instance.getIp(),
|
||||
instance.getPort());
|
||||
}
|
||||
nacosServiceNameMap.remove(taskDO.getServiceName());
|
||||
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Set<String> serviceNames = nacosServiceNameMap.keySet();
|
||||
for (String serviceName : serviceNames) {
|
||||
|
||||
if (nacosServiceNameMap.containsKey(serviceName)) {
|
||||
List<Instance> allInstances =
|
||||
destNamingService.getAllInstances(serviceName);
|
||||
for (Instance instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
destNamingService.deregisterInstance(instance.getServiceName(), instance.getIp(),
|
||||
instance.getPort());
|
||||
}
|
||||
nacosServiceNameMap.remove(serviceName);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("delete task from zookeeper to nacos was failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
metricsManager.recordError(MetricsStatisticsType.DELETE_ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* fetch the Path cache when the task sync
|
||||
*/
|
||||
protected TreeCache getTreeCache(TaskDO taskDO) {
|
||||
return treeCacheMap.computeIfAbsent(taskDO.getTaskId(), (key) -> {
|
||||
try {
|
||||
TreeCache treeCache =
|
||||
new TreeCache(zookeeperServerHolder.get(taskDO.getSourceClusterId(), ""),
|
||||
DUBBO_ROOT_PATH);
|
||||
treeCache.start();
|
||||
return treeCache;
|
||||
} catch (Exception e) {
|
||||
log.error("zookeeper path children cache start failed, taskId:{}", taskDO.getTaskId(), e);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* The instance information that needs to be synchronized is matched based on the dubbo version and the grouping
|
||||
* name
|
||||
*/
|
||||
protected boolean isMatch(TaskDO taskDO, Map<String, String> queryParam) {
|
||||
Predicate<TaskDO> isVersionEq = (task) -> StringUtils.isBlank(taskDO.getVersion())
|
||||
|| StringUtils.equals(task.getVersion(), queryParam.get(VERSION_KEY));
|
||||
Predicate<TaskDO> isGroupEq = (task) -> StringUtils.isBlank(taskDO.getGroupName()) || StringUtils.isBlank(queryParam.get(GROUP_KEY)) //fix
|
||||
|| StringUtils.equals(task.getGroupName(), queryParam.get(GROUP_KEY));
|
||||
return isVersionEq.and(isGroupEq).test(taskDO);
|
||||
}
|
||||
|
||||
/**
|
||||
* create Nacos service instance
|
||||
*
|
||||
* @param queryParam dubbo metadata
|
||||
* @param ipAndPortMap dubbo ip and address
|
||||
*/
|
||||
protected Instance buildSyncInstance(Map<String, String> queryParam, Map<String, String> ipAndPortMap,
|
||||
TaskDO taskDO) {
|
||||
Instance temp = new Instance();
|
||||
temp.setIp(ipAndPortMap.get(INSTANCE_IP_KEY));
|
||||
temp.setPort(Integer.parseInt(ipAndPortMap.get(INSTANCE_PORT_KEY)));
|
||||
temp.setServiceName(getServiceNameFromCache(taskDO.getTaskId(), queryParam));
|
||||
temp.setWeight(Double.parseDouble(queryParam.get(WEIGHT_KEY) == null ? "1.0" : queryParam.get(WEIGHT_KEY)));
|
||||
temp.setHealthy(true);
|
||||
|
||||
Map<String, String> metaData = new HashMap<>(queryParam);
|
||||
metaData.put(PROTOCOL_KEY, ipAndPortMap.get(PROTOCOL_KEY));
|
||||
metaData.put(SkyWalkerConstants.DEST_CLUSTERID_KEY, taskDO.getDestClusterId());
|
||||
metaData.put(SkyWalkerConstants.SYNC_SOURCE_KEY,
|
||||
skyWalkerCacheServices.getClusterType(taskDO.getSourceClusterId()).getCode());
|
||||
metaData.put(SkyWalkerConstants.SOURCE_CLUSTERID_KEY, taskDO.getSourceClusterId());
|
||||
temp.setMetadata(metaData);
|
||||
return temp;
|
||||
}
|
||||
|
||||
/**
|
||||
* cteate Dubbo service name
|
||||
*
|
||||
|
@ -375,5 +312,71 @@ public class ZookeeperSyncToNacosServiceImpl implements SyncService {
|
|||
protected String getServiceNameFromCache(String serviceName, Map<String, String> queryParam) {
|
||||
return nacosServiceNameMap.computeIfAbsent(serviceName, (key) -> createServiceName(queryParam));
|
||||
}
|
||||
|
||||
|
||||
private List<String> filterNoProviderPath(List<String> sourceInstances) {
|
||||
Iterator<String> iterator = sourceInstances.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
if (IGNORED_DUBBO_PATH.contains(iterator.next())) {
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
return sourceInstances;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消service下的instance注册,防止在server变化的时候,server之前sharding的service被分配到其他server上,这里需要手动将这部分instance下线;
|
||||
* (这里没有直接调用Nacos的deregisterInstance是因为在当前分布式下存在时序问题,可能导致该任务误删除其他server刚注册上的instance,这里使用停止发送心跳的方法,让instance自己下线)
|
||||
*
|
||||
* @param namingService
|
||||
* @param serviceNames
|
||||
*/
|
||||
private void deregisterService(NamingService namingService, Queue<ShardingLog> serviceNames, TaskDO taskDO) {
|
||||
log.info("zk->nacos current deal with serviceNames:" + sharding.getLocalServices(null));
|
||||
log.info("zk->nacos current change serviceNames count:" + serviceNames.size());
|
||||
while (!serviceNames.isEmpty()) {
|
||||
ShardingLog shardingLog = serviceNames.poll();
|
||||
if (!shardingLog.getType().equals(ShardingLogTypeEnum.DELETE.getType())) {
|
||||
log.info("zk->nacos current add serviceName:{},will skip...", shardingLog.getServiceName());
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
List<Instance> allInstances =
|
||||
namingService.getAllInstances(nacosServiceNameMap.get(shardingLog.getServiceName()));
|
||||
for (Instance instance : allInstances) {
|
||||
if (needDelete(instance.getMetadata(), taskDO)) {
|
||||
log.info("zk->nacos current will stop beat:" + instance.getIp() + instance.getPort() + " ,key:" + instance.getServiceName());
|
||||
((NacosNamingService) namingService).getBeatReactor().removeBeatInfo(instance.getServiceName(), instance.getIp(), instance.getPort());
|
||||
}
|
||||
nacosServiceNameMap.remove(shardingLog.getServiceName());
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("deregisterService faild ,cause by:{}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断是否本机处理的service
|
||||
*
|
||||
* @param taskDO
|
||||
* @param destNamingService
|
||||
* @param serviceName
|
||||
* @return
|
||||
*/
|
||||
private boolean isProcess(TaskDO taskDO, NamingService destNamingService, String serviceName) {
|
||||
try {
|
||||
if (IGNORED_DUBBO_PATH.contains(serviceName))
|
||||
return false;
|
||||
sharding.doSharding(null, new ArrayList<>(Arrays.asList(serviceName)));
|
||||
deregisterService(destNamingService, sharding.getChangeService(), taskDO);
|
||||
if (sharding.getLocalServices(null).contains(serviceName)) {
|
||||
return true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("zk->nacos sharding faild ,taskid:{}", taskDO.getId(), e);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,210 @@
|
|||
package com.alibaba.nacossync.extension.impl.extend;
|
||||
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.client.naming.utils.NetUtils;
|
||||
import com.alibaba.nacossync.constant.ShardingLogTypeEnum;
|
||||
import com.alibaba.nacossync.extension.SyncManagerService;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.extension.impl.NacosSyncToZookeeperServiceImpl;
|
||||
import com.alibaba.nacossync.extension.sharding.ConsistentHashServiceSharding;
|
||||
import com.alibaba.nacossync.extension.sharding.ServiceSharding;
|
||||
import com.alibaba.nacossync.pojo.ShardingLog;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.DubboConstants;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/29.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class NacosSyncToZookeeperServicesSharding implements Sharding {
|
||||
|
||||
@Autowired
|
||||
private NacosServerHolder nacosServerHolder;
|
||||
|
||||
public static final int DEFAULT_SERVICE_PAGENO = 1;
|
||||
|
||||
public static final int DEFAULT_SERVICE_PAGE_SIZE = 10000;
|
||||
|
||||
private volatile String serviceListMd5;
|
||||
|
||||
@Autowired
|
||||
private SyncManagerService syncManagerService;
|
||||
|
||||
@Lazy
|
||||
@Resource(type = ConsistentHashServiceSharding.class)
|
||||
private ServiceSharding serviceSharding;
|
||||
|
||||
private static final String SHARDING_KEY_NAME = NacosSyncToZookeeperServicesSharding.class.getName();
|
||||
|
||||
private static final long DEFAULT_SERVICES_CHANGE_THREAD_DELAY = 10;
|
||||
|
||||
private static final long DEFAULT_SERVICES_CHANGE_THREAD_INTERVAL = 5;
|
||||
|
||||
//add cache taskDO
|
||||
private Map<String, TaskDO> taskDOMap = new ConcurrentHashMap<>();
|
||||
|
||||
@Value("${server.port}")
|
||||
private String serverPort;
|
||||
|
||||
|
||||
private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
|
||||
@Override
|
||||
public Thread newThread(Runnable r) {
|
||||
Thread thread = new Thread(r);
|
||||
thread.setDaemon(true);
|
||||
thread.setName(" com.alibaba.nacossync.sharding.getServiceName");
|
||||
return thread;
|
||||
}
|
||||
});
|
||||
|
||||
protected boolean servicesIschanged(TaskDO taskDO) throws Exception {
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());
|
||||
List<String> serviceNames = sourceNamingService.getServicesOfServer(DEFAULT_SERVICE_PAGENO, DEFAULT_SERVICE_PAGE_SIZE, SkyWalkerUtil.getGroupName(taskDO.getGroupName())).getData();
|
||||
Collections.sort(serviceNames);
|
||||
String md5 = SkyWalkerUtil.StringToMd5(serviceNames.toString());
|
||||
if (!md5.equals(serviceListMd5)) {
|
||||
serviceListMd5 = md5;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected synchronized void reSubscribeService(TaskDO taskDO) {
|
||||
log.error("reSubscribe start");
|
||||
if (Objects.isNull(taskDO)) return;
|
||||
try {
|
||||
NamingService sourceNamingService =
|
||||
nacosServerHolder.get(taskDO.getSourceClusterId(), taskDO.getNameSpace());
|
||||
List<String> serviceNames = sourceNamingService.getServicesOfServer(DEFAULT_SERVICE_PAGENO, DEFAULT_SERVICE_PAGE_SIZE, taskDO.getGroupName()).getData();//如果使用同一个groupName暂时没问题,如果配置了多个group,需要升级sdk1.4+,支持按照*的group查询
|
||||
serviceSharding.sharding(SHARDING_KEY_NAME, serviceNames);
|
||||
} catch (Exception e) {
|
||||
log.error("reSubscribe faild,task id:{}", taskDO.getId(), e);
|
||||
}
|
||||
if (!serviceSharding.getChangeServices(SHARDING_KEY_NAME).isEmpty()) {
|
||||
try {
|
||||
syncChangedServices();
|
||||
} catch (Exception e) {
|
||||
log.error("reSubscribe -->delete service faild,task id:{}", taskDO.getId(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//暂时不支持多source_cluster_id,多nammespace维度(默认取第一个task的source_cluster_id和namespace)
|
||||
@Override
|
||||
public void start(TaskDO taskDO) {
|
||||
taskDOMap.putIfAbsent(taskDO.getServiceName(), taskDO);
|
||||
if (!serviceSharding.addServerChange(SHARDING_KEY_NAME, this)) {
|
||||
return;
|
||||
}
|
||||
executorService.scheduleWithFixedDelay(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
if (servicesIschanged(taskDO)) {
|
||||
reSubscribeService(taskDO);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("schedule reSubscribe service thread faild ,task id :", taskDO.getId(), e);
|
||||
}
|
||||
}
|
||||
}, DEFAULT_SERVICES_CHANGE_THREAD_DELAY, DEFAULT_SERVICES_CHANGE_THREAD_INTERVAL, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onServerChange() {
|
||||
if (taskDOMap.size() > 0) {
|
||||
for (TaskDO taskDO : taskDOMap.values()) {//任意取一个taskDo
|
||||
reSubscribeService(taskDO);
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Queue<ShardingLog> getChangeService() {
|
||||
return serviceSharding.getChangeServices(SHARDING_KEY_NAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doSharding(String key, List<String> serviceNames) {
|
||||
serviceSharding.sharding(SHARDING_KEY_NAME, serviceNames);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TreeSet<String> getLocalServices(String key) {
|
||||
return serviceSharding.getLocalServices(key);
|
||||
}
|
||||
|
||||
|
||||
private void syncAddedServices(String serviceName) {
|
||||
if (taskDOMap.containsKey(DubboConstants.ALL_SERVICE_NAME_PATTERN)) {//如果有配置为* 的 则不用处理单独配置serviceName的task
|
||||
TaskDO taskDO = buildNewTaskDo(taskDOMap.get(DubboConstants.ALL_SERVICE_NAME_PATTERN), serviceName);
|
||||
((NacosSyncToZookeeperServiceImpl) syncManagerService.getSyncService(taskDO.getSourceClusterId(), taskDO.getDestClusterId())).addSyncService(taskDO);
|
||||
log.info("reSubscribe ,{} is add", serviceName);
|
||||
return;
|
||||
}
|
||||
if (taskDOMap.containsKey(serviceName)) {//如果有配置变更的serviceName,而且sharding到本server则处理
|
||||
TaskDO taskDO = buildNewTaskDo(taskDOMap.get(serviceName), serviceName);
|
||||
((NacosSyncToZookeeperServiceImpl) syncManagerService.getSyncService(taskDO.getSourceClusterId(), taskDO.getDestClusterId())).addSyncService(taskDO);
|
||||
log.info("reSubscribe ,{} is add", serviceName);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void syncRemovedServices(String serviceName) {
|
||||
if (taskDOMap.containsKey(DubboConstants.ALL_SERVICE_NAME_PATTERN)) {
|
||||
TaskDO taskDO = buildNewTaskDo(taskDOMap.get(DubboConstants.ALL_SERVICE_NAME_PATTERN), serviceName);
|
||||
syncManagerService.getSyncService(taskDO.getSourceClusterId(), taskDO.getDestClusterId()).delete(taskDO);
|
||||
log.info("reSubscribe ,{} is remove", serviceName);
|
||||
return;
|
||||
}
|
||||
if (taskDOMap.containsKey(serviceName)) {
|
||||
TaskDO taskDO = buildNewTaskDo(taskDOMap.get(serviceName), serviceName);
|
||||
syncManagerService.getSyncService(taskDO.getSourceClusterId(), taskDO.getDestClusterId()).delete(taskDO);
|
||||
log.info("reSubscribe ,{} is remove", serviceName);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void syncChangedServices() {
|
||||
log.info("reSubscribe ,local ip: {},current sharding service:{}", NetUtils.localIP() + ":" + serverPort, serviceSharding.getLocalServices(SHARDING_KEY_NAME).toString());
|
||||
while (!serviceSharding.getChangeServices(SHARDING_KEY_NAME).isEmpty()) {
|
||||
ShardingLog shardingLog = serviceSharding.getChangeServices(SHARDING_KEY_NAME).poll();
|
||||
if (shardingLog.getType().equals(ShardingLogTypeEnum.ADD.getType())) {
|
||||
syncAddedServices(shardingLog.getServiceName());
|
||||
}
|
||||
if (shardingLog.getType().equals(ShardingLogTypeEnum.DELETE.getType())) {
|
||||
syncRemovedServices(shardingLog.getServiceName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop(TaskDO taskDO) {
|
||||
if (taskDOMap.containsKey(taskDO.getServiceName())) {
|
||||
taskDOMap.remove(taskDO.getServiceName());
|
||||
}
|
||||
}
|
||||
|
||||
private TaskDO buildNewTaskDo(TaskDO taskDO, String serviceName) {
|
||||
TaskDO taskDO1 = new TaskDO();
|
||||
BeanUtils.copyProperties(taskDO, taskDO1);
|
||||
taskDO1.setTaskId(serviceName);//需要一个key替换以前的taskid很多封装维度,暂时使用serviceName
|
||||
taskDO1.setServiceName(serviceName);
|
||||
return taskDO1;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,26 @@
|
|||
package com.alibaba.nacossync.extension.impl.extend;
|
||||
|
||||
import com.alibaba.nacossync.pojo.ShardingLog;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Queue;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/30.
|
||||
*/
|
||||
public interface Sharding {
|
||||
|
||||
public void onServerChange();
|
||||
|
||||
public void start(TaskDO taskDO);
|
||||
|
||||
public void stop(TaskDO taskDO);
|
||||
|
||||
public void doSharding(String key, List<String> serviceNames);
|
||||
|
||||
public TreeSet<String> getLocalServices(String key);
|
||||
|
||||
public Queue<ShardingLog> getChangeService();
|
||||
}
|
|
@ -0,0 +1,84 @@
|
|||
package com.alibaba.nacossync.extension.impl.extend;
|
||||
|
||||
import com.alibaba.nacossync.extension.sharding.ConsistentHashServiceSharding;
|
||||
import com.alibaba.nacossync.extension.sharding.ServiceSharding;
|
||||
import com.alibaba.nacossync.pojo.ShardingLog;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Queue;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/30.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ZookeeperSyncToNacosServiceSharding implements Sharding {
|
||||
|
||||
private static final String SHARDING_KEY_NAME = ZookeeperSyncToNacosServiceSharding.class.getName();
|
||||
|
||||
private volatile String serviceListMd5;
|
||||
|
||||
private volatile boolean serverChange = false;
|
||||
|
||||
@Lazy
|
||||
@Resource(type = ConsistentHashServiceSharding.class)
|
||||
private ServiceSharding serviceSharding;
|
||||
|
||||
|
||||
@Override
|
||||
public void onServerChange() {
|
||||
serverChange = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(TaskDO taskDO) {
|
||||
serviceSharding.addServerChange(SHARDING_KEY_NAME, this);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Queue<ShardingLog> getChangeService() {
|
||||
return serviceSharding.getChangeServices(SHARDING_KEY_NAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doSharding(String key, List<String> serviceNames) {
|
||||
try {
|
||||
if (servicesIschanged(serviceNames) || serverChange) {
|
||||
log.info("zk ->nacos reshading start");
|
||||
serverChange = false;
|
||||
serviceSharding.sharding(SHARDING_KEY_NAME, serviceNames);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("zk ->nacos reshading faild.", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public TreeSet<String> getLocalServices(String key) {
|
||||
return serviceSharding.getLocalServices(SHARDING_KEY_NAME);
|
||||
}
|
||||
|
||||
protected boolean servicesIschanged(List<String> serviceNames) throws Exception {//zk区分不了是service变化还是instance变化
|
||||
Collections.sort(serviceNames);
|
||||
String md5 = SkyWalkerUtil.StringToMd5(serviceNames.toString());
|
||||
if (!md5.equals(serviceListMd5)) {
|
||||
serviceListMd5 = md5;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop(TaskDO taskDO) {
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,148 @@
|
|||
package com.alibaba.nacossync.extension.sharding;
|
||||
|
||||
import com.alibaba.nacos.api.naming.listener.Event;
|
||||
import com.alibaba.nacos.api.naming.listener.EventListener;
|
||||
import com.alibaba.nacos.client.naming.utils.NetUtils;
|
||||
import com.alibaba.nacossync.constant.ShardingLogTypeEnum;
|
||||
import com.alibaba.nacossync.extension.impl.extend.Sharding;
|
||||
import com.alibaba.nacossync.pojo.ShardingLog;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/27.
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class AbstractServiceSharding implements ServiceSharding, InitializingBean {
|
||||
|
||||
protected volatile List<String> servers = new LinkedList<String>();
|
||||
|
||||
private Map<String, ConcurrentLinkedQueue<ShardingLog>> localServicesChangeMap = new ConcurrentHashMap<String, ConcurrentLinkedQueue<ShardingLog>>();
|
||||
|
||||
private volatile Map<String, TreeSet<String>> localServicesMap = new ConcurrentHashMap<String, TreeSet<String>>();
|
||||
|
||||
private final static String LOCAL_IP = NetUtils.localIP();
|
||||
|
||||
private volatile String serverListMd5;
|
||||
|
||||
private Map<String, Sharding> serverListens = new ConcurrentHashMap<String, Sharding>();
|
||||
|
||||
@Value("${server.port}")
|
||||
private String serverPort;
|
||||
|
||||
@Lazy
|
||||
@Resource(type = NacosServersManager.class)
|
||||
private ServersManager serversManager;
|
||||
|
||||
protected List<String> getServers() {
|
||||
return servers;
|
||||
}
|
||||
|
||||
protected void listenServer() {
|
||||
try {
|
||||
serversManager.subscribeServers(new EventListener() {
|
||||
@Override
|
||||
public void onEvent(Event event) {
|
||||
try {
|
||||
shadingServers();
|
||||
for (Sharding sharding : serverListens.values()) {
|
||||
sharding.onServerChange();
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("subscribe servers faild.", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error("subscribe servers faild.", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void shadingServers() throws Exception {
|
||||
List<String> serversList = serversManager.getServers();
|
||||
Collections.sort(serversList);
|
||||
String md5 = SkyWalkerUtil.StringToMd5(serversList.toString());
|
||||
if (!md5.equals(serverListMd5)) {
|
||||
servers = serversList;
|
||||
serverListMd5 = md5;
|
||||
doSharding();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//need fix: 暂时同步话解决
|
||||
protected void shadingServices(String key, List<String> serviceNames) {
|
||||
if (!localServicesMap.containsKey(key)) {
|
||||
TreeSet<String> localServicesSet = new TreeSet<String>();
|
||||
localServicesMap.putIfAbsent(key, localServicesSet);
|
||||
}
|
||||
if (!localServicesChangeMap.containsKey(key)) {
|
||||
ConcurrentLinkedQueue<ShardingLog> removeQueue = new ConcurrentLinkedQueue<ShardingLog>();
|
||||
localServicesChangeMap.putIfAbsent(key, removeQueue);
|
||||
}
|
||||
TreeSet<String> localServices = localServicesMap.get(key);
|
||||
try {
|
||||
for (String serviceName : serviceNames) {
|
||||
if (getShardingServer(serviceName).equals(LOCAL_IP + ":" + serverPort)) {
|
||||
if (!localServices.contains(serviceName)) {
|
||||
localServicesMap.get(key).add(serviceName);
|
||||
localServicesChangeMap.get(key).offer(new ShardingLog(serviceName, ShardingLogTypeEnum.ADD.getType()));
|
||||
}
|
||||
} else {
|
||||
if (localServices.contains(serviceName)) {
|
||||
localServicesMap.get(key).remove(serviceName);
|
||||
localServicesChangeMap.get(key).offer(new ShardingLog(serviceName, ShardingLogTypeEnum.DELETE.getType()));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("shading services faild.", e);
|
||||
}
|
||||
}
|
||||
|
||||
//need fix:按照service维度做sharding,但是在service维度存在zk->nacos nacos->zk两种service,而目前避免环的处理在instance维度的metadata中,如果每次做sharding都去判断instance消耗太大,而且也不能完全避免service中存在多种源的instance,故目前做法是按照zk和nacos注册上的所有
|
||||
//serviceName List做sharding,可能存在sharding不均衡问题,如导致大部分的service都落在一个node上的可能
|
||||
@Override
|
||||
public void sharding(String key, List<String> serviceNames) {
|
||||
try {
|
||||
shadingServices(key, serviceNames);
|
||||
} catch (Exception e) {
|
||||
log.error("sharding faild. sharding key is:{}", key, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean addServerChange(String key, Sharding sharding) {
|
||||
return serverListens.putIfAbsent(key, sharding) == null ? true : false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TreeSet<String> getLocalServices(String key) {
|
||||
return localServicesMap.get(key);
|
||||
}
|
||||
|
||||
protected abstract void doSharding();
|
||||
|
||||
protected abstract String getShardingServer(String key);
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() throws Exception {
|
||||
listenServer();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Queue<ShardingLog> getChangeServices(String key) {
|
||||
return localServicesChangeMap.get(key);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,78 @@
|
|||
package com.alibaba.nacossync.extension.sharding;
|
||||
|
||||
import com.alibaba.nacos.common.utils.StringUtils;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/27.
|
||||
*/
|
||||
@Service
|
||||
@Lazy
|
||||
public class ConsistentHashServiceSharding extends AbstractServiceSharding {
|
||||
|
||||
public static final String HASH_NODES = "-hash.vn.nodes";
|
||||
|
||||
private List<String> nodes = new LinkedList<String>();
|
||||
|
||||
private SortedMap<Integer, String> virtualNodes = new TreeMap<Integer, String>();
|
||||
|
||||
private static final int VIRTUAL_COUNT = 100;
|
||||
|
||||
public ConsistentHashServiceSharding() {
|
||||
super();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void doSharding() {
|
||||
List<String> servers = getServers();
|
||||
nodes.clear();
|
||||
virtualNodes.clear();
|
||||
for (String node : servers) {
|
||||
nodes.add(node);
|
||||
}
|
||||
for (String node : nodes) {
|
||||
for (int i = 0; i < VIRTUAL_COUNT; i++) {
|
||||
String virtualNodeName = node + HASH_NODES + String.valueOf(i);
|
||||
virtualNodes.put(getHash(virtualNodeName), virtualNodeName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getShardingServer(String key) {
|
||||
int hash = getHash(SkyWalkerUtil.StringToMd5(key));
|
||||
SortedMap<Integer, String> subMap = virtualNodes.tailMap(hash);
|
||||
String virtualNode;
|
||||
if (subMap.isEmpty()) {
|
||||
Integer i = virtualNodes.firstKey();
|
||||
virtualNode = virtualNodes.get(i);
|
||||
} else {
|
||||
Integer i = subMap.firstKey();
|
||||
virtualNode = subMap.get(i);
|
||||
}
|
||||
if (StringUtils.isNotBlank(virtualNode)) {
|
||||
return virtualNode.substring(0, virtualNode.indexOf("-"));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private int getHash(String str) {
|
||||
final int p = 16777619;
|
||||
int hash = (int) 2166136261L;
|
||||
for (int i = 0; i < str.length(); i++)
|
||||
hash = (hash ^ str.charAt(i)) * p;
|
||||
hash += hash << 13;
|
||||
hash ^= hash >> 7;
|
||||
hash += hash << 3;
|
||||
hash ^= hash >> 17;
|
||||
hash += hash << 5;
|
||||
if (hash < 0)
|
||||
hash = Math.abs(hash);
|
||||
return hash;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,106 @@
|
|||
package com.alibaba.nacossync.extension.sharding;
|
||||
|
||||
import com.alibaba.nacos.api.PropertyKeyConst;
|
||||
import com.alibaba.nacos.api.naming.NamingFactory;
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.listener.EventListener;
|
||||
import com.alibaba.nacos.api.naming.pojo.Instance;
|
||||
import com.alibaba.nacos.client.naming.utils.NetUtils;
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import com.alibaba.nacossync.util.StringUtils;
|
||||
import com.google.common.base.Joiner;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/27.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
@Lazy
|
||||
public class NacosServersManager implements ServersManager<EventListener>, InitializingBean {
|
||||
|
||||
private NamingService namingService;
|
||||
|
||||
@Value("${server.port}")
|
||||
private String serverPort;
|
||||
|
||||
@Autowired
|
||||
private SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
//不动现有业务表和逻辑基础上,指定一个固定的key--"SHARDINGKEY" 生成md5值 作为默认连接的nacos路径
|
||||
private static final String DEFAULT_SHARDING_NACOS_KEY = "3ac01a8c7501f121ab01efb920aa4764";
|
||||
|
||||
private static final String DEFAULT_SHARDING_NACOS_NAMESPACES = "public";
|
||||
|
||||
private static final String DEFAULT_SHARDING_NACOS_GOURPID = "shadinggroup";
|
||||
|
||||
private static final String DEFAULT_SHARDING_NACOS_SERVICENAME = "com.dmall.sharding";
|
||||
|
||||
@Value("${sharding.nacos.url}")
|
||||
private String shardingNacosUrl;
|
||||
|
||||
@Value("${sharding.nacos.namespace}")
|
||||
private String shardingNacosnameSpace;
|
||||
|
||||
@Value("${sharding.nacos.groupname}")
|
||||
private String shardingNacosGroupName;
|
||||
|
||||
@Value("${sharding.nacos.servicename}")
|
||||
private String shardingNacosServiceName;
|
||||
|
||||
@Override
|
||||
public List<String> getServers() throws Exception {
|
||||
List<Instance> instanceList = namingService.getAllInstances(DEFAULT_SHARDING_NACOS_SERVICENAME, DEFAULT_SHARDING_NACOS_GOURPID);
|
||||
List<String> serverList = new LinkedList<String>();
|
||||
for (Instance instance : instanceList) {
|
||||
serverList.add(instance.getIp() + ":" + instance.getPort());
|
||||
}
|
||||
return serverList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void subscribeServers(EventListener listener) throws Exception {
|
||||
namingService.subscribe(DEFAULT_SHARDING_NACOS_SERVICENAME, DEFAULT_SHARDING_NACOS_GOURPID, listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void register(String ip, int port) throws Exception {
|
||||
namingService.registerInstance(DEFAULT_SHARDING_NACOS_SERVICENAME, DEFAULT_SHARDING_NACOS_GOURPID, ip, port);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() throws Exception {
|
||||
try {
|
||||
log.info("start init nacos servers.");
|
||||
if (StringUtils.isEmpty(shardingNacosUrl)) {
|
||||
shardingNacosUrl = DEFAULT_SHARDING_NACOS_KEY;
|
||||
}
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(PropertyKeyConst.SERVER_ADDR, getNacosUrl());
|
||||
properties.setProperty(PropertyKeyConst.NAMESPACE, StringUtils.isEmpty(shardingNacosnameSpace) ? DEFAULT_SHARDING_NACOS_NAMESPACES : shardingNacosnameSpace);
|
||||
namingService = NamingFactory.createNamingService(properties);
|
||||
register(NetUtils.localIP(), Integer.parseInt(serverPort));
|
||||
log.info("init nacos servers sucess.");
|
||||
} catch (Exception e) {
|
||||
log.info("init nacos faild .", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String getNacosUrl() {
|
||||
if (!StringUtils.isIPV4AndPorts(shardingNacosUrl, ",")) {
|
||||
List<String> allClusterConnectKey = skyWalkerCacheServices
|
||||
.getAllClusterConnectKey(StringUtils.isEmpty(shardingNacosUrl) ? DEFAULT_SHARDING_NACOS_KEY : shardingNacosUrl);
|
||||
return Joiner.on(",").join(allClusterConnectKey);
|
||||
}
|
||||
return shardingNacosUrl;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
package com.alibaba.nacossync.extension.sharding;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/27.
|
||||
*/
|
||||
public interface ServersManager<T> {
|
||||
|
||||
public List<String> getServers() throws Exception;
|
||||
|
||||
public void subscribeServers(T listener) throws Exception;
|
||||
|
||||
public void register(String ip, int port) throws Exception;
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
package com.alibaba.nacossync.extension.sharding;
|
||||
|
||||
import com.alibaba.nacossync.extension.impl.extend.Sharding;
|
||||
import com.alibaba.nacossync.pojo.ShardingLog;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Queue;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/10/27.
|
||||
*/
|
||||
public interface ServiceSharding {
|
||||
|
||||
public void sharding(String key, List<String> serviceNames);
|
||||
|
||||
public TreeSet<String> getLocalServices(String key);
|
||||
|
||||
public boolean addServerChange(String name, Sharding sharding);
|
||||
|
||||
public Queue<ShardingLog> getChangeServices(String key);
|
||||
}
|
|
@ -23,19 +23,15 @@ import java.util.concurrent.TimeUnit;
|
|||
@Service
|
||||
public class MetricsManager implements CommandLineRunner {
|
||||
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
@Autowired
|
||||
private SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
@Autowired
|
||||
private ClusterAccessService clusterAccessService;
|
||||
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
public MetricsManager(SkyWalkerCacheServices skyWalkerCacheServices, ClusterAccessService clusterAccessService,
|
||||
TaskAccessService taskAccessService) {
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
this.taskAccessService = taskAccessService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Callback used to run the bean.
|
||||
*
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
package com.alibaba.nacossync.pojo;
|
||||
|
||||
/**
|
||||
* Created by maj on 2020/11/18.
|
||||
*/
|
||||
public class ShardingLog {
|
||||
|
||||
private String serviceName;
|
||||
|
||||
private String type;
|
||||
|
||||
public ShardingLog() {
|
||||
}
|
||||
|
||||
public ShardingLog(String serviceName, String type) {
|
||||
this.serviceName = serviceName;
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public String getServiceName() {
|
||||
return serviceName;
|
||||
}
|
||||
|
||||
public void setServiceName(String serviceName) {
|
||||
this.serviceName = serviceName;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
}
|
|
@ -17,28 +17,17 @@
|
|||
package com.alibaba.nacossync.pojo.model;
|
||||
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.proxy.HibernateProxy;
|
||||
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
import javax.persistence.*;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: EnvDO.java, v 0.1 2018-09-25 PM 4:17 NacosSync Exp $$
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@RequiredArgsConstructor
|
||||
@Data
|
||||
@Entity
|
||||
@Table(name = "cluster")
|
||||
public class ClusterDO implements Serializable {
|
||||
|
@ -77,34 +66,4 @@ public class ClusterDO implements Serializable {
|
|||
*/
|
||||
private String password;
|
||||
|
||||
private String namespace;
|
||||
|
||||
private Integer clusterLevel;
|
||||
|
||||
@Override
|
||||
public final boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null) {
|
||||
return false;
|
||||
}
|
||||
Class<?> oEffectiveClass =
|
||||
o instanceof HibernateProxy ? ((HibernateProxy) o).getHibernateLazyInitializer().getPersistentClass()
|
||||
: o.getClass();
|
||||
Class<?> thisEffectiveClass =
|
||||
this instanceof HibernateProxy ? ((HibernateProxy) this).getHibernateLazyInitializer()
|
||||
.getPersistentClass() : this.getClass();
|
||||
if (thisEffectiveClass != oEffectiveClass) {
|
||||
return false;
|
||||
}
|
||||
ClusterDO clusterDO = (ClusterDO) o;
|
||||
return getId() != null && Objects.equals(getId(), clusterDO.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public final int hashCode() {
|
||||
return this instanceof HibernateProxy ? ((HibernateProxy) this).getHibernateLazyInitializer()
|
||||
.getPersistentClass().hashCode() : getClass().hashCode();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -16,28 +16,15 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.pojo.model;
|
||||
|
||||
import javax.persistence.*;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.proxy.HibernateProxy;
|
||||
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import java.util.Objects;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: SystemConfig.java, v 0.1 2018-09-26 上午1:48 NacosSync Exp $$
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@RequiredArgsConstructor
|
||||
@Data
|
||||
@Entity
|
||||
@Table(name = "system_config")
|
||||
public class SystemConfigDO {
|
||||
|
@ -47,31 +34,5 @@ public class SystemConfigDO {
|
|||
private String configKey;
|
||||
private String configValue;
|
||||
private String configDesc;
|
||||
|
||||
@Override
|
||||
public final boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null) {
|
||||
return false;
|
||||
}
|
||||
Class<?> oEffectiveClass =
|
||||
o instanceof HibernateProxy ? ((HibernateProxy) o).getHibernateLazyInitializer().getPersistentClass()
|
||||
: o.getClass();
|
||||
Class<?> thisEffectiveClass =
|
||||
this instanceof HibernateProxy ? ((HibernateProxy) this).getHibernateLazyInitializer()
|
||||
.getPersistentClass() : this.getClass();
|
||||
if (thisEffectiveClass != oEffectiveClass) {
|
||||
return false;
|
||||
}
|
||||
SystemConfigDO that = (SystemConfigDO) o;
|
||||
return getId() != null && Objects.equals(getId(), that.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public final int hashCode() {
|
||||
return this instanceof HibernateProxy ? ((HibernateProxy) this).getHibernateLazyInitializer()
|
||||
.getPersistentClass().hashCode() : getClass().hashCode();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -12,29 +12,16 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.pojo.model;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.proxy.HibernateProxy;
|
||||
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import javax.persistence.*;
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: TaskDo.java, v 0.1 2018-09-24 PM11:53 NacosSync Exp $$
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@RequiredArgsConstructor
|
||||
@Data
|
||||
@Entity
|
||||
@Table(name = "task")
|
||||
public class TaskDO implements Serializable {
|
||||
|
@ -83,31 +70,4 @@ public class TaskDO implements Serializable {
|
|||
* operation id,The operation id follow when the task status changes
|
||||
*/
|
||||
private String operationId;
|
||||
|
||||
@Override
|
||||
public final boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null) {
|
||||
return false;
|
||||
}
|
||||
Class<?> oEffectiveClass =
|
||||
o instanceof HibernateProxy ? ((HibernateProxy) o).getHibernateLazyInitializer().getPersistentClass()
|
||||
: o.getClass();
|
||||
Class<?> thisEffectiveClass =
|
||||
this instanceof HibernateProxy ? ((HibernateProxy) this).getHibernateLazyInitializer()
|
||||
.getPersistentClass() : this.getClass();
|
||||
if (thisEffectiveClass != oEffectiveClass) {
|
||||
return false;
|
||||
}
|
||||
TaskDO taskDO = (TaskDO) o;
|
||||
return getId() != null && Objects.equals(getId(), taskDO.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public final int hashCode() {
|
||||
return this instanceof HibernateProxy ? ((HibernateProxy) this).getHibernateLazyInitializer()
|
||||
.getPersistentClass().hashCode() : getClass().hashCode();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,10 +17,10 @@
|
|||
package com.alibaba.nacossync.pojo.request;
|
||||
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: AddClusterRequest.java, v 0.1 2018-09-25 PM 10:27 NacosSync Exp $$
|
||||
|
@ -52,6 +52,5 @@ public class ClusterAddRequest extends BaseRequest {
|
|||
* The password of the Nacos.
|
||||
*/
|
||||
private String password;
|
||||
private String namespace;
|
||||
|
||||
}
|
||||
|
|
|
@ -1,44 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.pojo.request;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: TaskAddAllRequest.java, v 0.1 2022-03-23 AM12:13 NacosSync Exp $$
|
||||
*/
|
||||
@Data
|
||||
public class TaskAddAllRequest extends BaseRequest {
|
||||
|
||||
/**
|
||||
* eg: b7bacb110199d5bb83b9757038fadeb0 .
|
||||
*/
|
||||
private String sourceClusterId;
|
||||
|
||||
/**
|
||||
* eg: bbdad57833a0e4f0981f6f3349005617 .
|
||||
*/
|
||||
private String destClusterId;
|
||||
|
||||
/**
|
||||
* whether to exclude subscriber.
|
||||
*/
|
||||
private boolean excludeConsumer = true;
|
||||
|
||||
}
|
|
@ -14,53 +14,33 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.pojo.view;
|
||||
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: ClusterModel.java, v 0.1 2018-09-25 下午11:09 NacosSync Exp $$
|
||||
*/
|
||||
@Data
|
||||
public class ClusterModel implements Serializable {
|
||||
|
||||
|
||||
private String clusterId;
|
||||
|
||||
/**
|
||||
* json format,["192.168.1:8080","192.168.2?key=1"]
|
||||
*/
|
||||
private String connectKeyList;
|
||||
|
||||
/**
|
||||
* cluster name, eg:cluster of ShangHai(edas-sh)
|
||||
*/
|
||||
private String clusterName;
|
||||
|
||||
/**
|
||||
* cluster type, eg cluster of CS,cluster of Nacos,
|
||||
*
|
||||
* @see ClusterTypeEnum
|
||||
*/
|
||||
private String clusterType;
|
||||
|
||||
private String namespace;
|
||||
|
||||
private String userName;
|
||||
|
||||
public static ClusterModel from(ClusterDO clusterDO) {
|
||||
ClusterModel clusterModel = new ClusterModel();
|
||||
clusterModel.setClusterId(clusterDO.getClusterId());
|
||||
clusterModel.setConnectKeyList(clusterDO.getConnectKeyList());
|
||||
clusterModel.setClusterType(clusterDO.getClusterType());
|
||||
clusterModel.setClusterName(clusterDO.getClusterName());
|
||||
clusterModel.setNamespace(clusterDO.getNamespace());
|
||||
clusterModel.setUserName(clusterDO.getUserName());
|
||||
return clusterModel;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,10 +14,8 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.pojo.view;
|
||||
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
|
@ -26,27 +24,13 @@ import lombok.Data;
|
|||
*/
|
||||
@Data
|
||||
public class TaskModel {
|
||||
|
||||
|
||||
private String taskId;
|
||||
|
||||
private String sourceClusterId;
|
||||
|
||||
private String destClusterId;
|
||||
|
||||
private String serviceName;
|
||||
|
||||
private String groupName;
|
||||
|
||||
private String taskStatus;
|
||||
|
||||
public static TaskModel from(TaskDO taskDO) {
|
||||
TaskModel taskModel = new TaskModel();
|
||||
taskModel.setDestClusterId(taskDO.getDestClusterId());
|
||||
taskModel.setGroupName(taskDO.getGroupName());
|
||||
taskModel.setServiceName(taskDO.getServiceName());
|
||||
taskModel.setSourceClusterId(taskDO.getSourceClusterId());
|
||||
taskModel.setTaskStatus(taskDO.getTaskStatus());
|
||||
taskModel.setTaskId(taskDO.getTaskId());
|
||||
return taskModel;
|
||||
}
|
||||
private String nameSpace;
|
||||
|
||||
}
|
||||
|
|
|
@ -45,7 +45,8 @@ public class SkyWalkerTemplate {
|
|||
}
|
||||
|
||||
private static <T extends BaseResult> void initExceptionResult(T result, Throwable e) {
|
||||
if (e instanceof SkyWalkerException skyWalkerException) {
|
||||
if (e instanceof SkyWalkerException) {
|
||||
SkyWalkerException skyWalkerException = (SkyWalkerException) e;
|
||||
if (null != skyWalkerException.getResultCode()) {
|
||||
result.setResultCode(skyWalkerException.getResultCode().getCode());
|
||||
}
|
||||
|
|
|
@ -14,20 +14,21 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.alibaba.nacossync.constant.ClusterTypeEnum;
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.exception.SkyWalkerException;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import com.alibaba.nacossync.pojo.request.ClusterAddRequest;
|
||||
import com.alibaba.nacossync.pojo.result.ClusterAddResult;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
|
@ -37,53 +38,47 @@ import org.springframework.stereotype.Service;
|
|||
@Slf4j
|
||||
@Service
|
||||
public class ClusterAddProcessor implements Processor<ClusterAddRequest, ClusterAddResult> {
|
||||
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public ClusterAddProcessor(ClusterAccessService clusterAccessService, ObjectMapper objectMapper) {
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
@Autowired
|
||||
private ClusterAccessService clusterAccessService;
|
||||
|
||||
@Override
|
||||
public void process(ClusterAddRequest clusterAddRequest, ClusterAddResult clusterAddResult, Object... others)
|
||||
throws Exception {
|
||||
public void process(ClusterAddRequest clusterAddRequest, ClusterAddResult clusterAddResult,
|
||||
Object... others) throws Exception {
|
||||
ClusterDO clusterDO = new ClusterDO();
|
||||
|
||||
if (null == clusterAddRequest.getConnectKeyList() || clusterAddRequest.getConnectKeyList().isEmpty()) {
|
||||
|
||||
|
||||
if (null == clusterAddRequest.getConnectKeyList() || 0 == clusterAddRequest.getConnectKeyList().size()) {
|
||||
|
||||
throw new SkyWalkerException("集群列表不能为空!");
|
||||
}
|
||||
|
||||
if (StringUtils.isBlank(clusterAddRequest.getClusterName()) || StringUtils.isBlank(
|
||||
clusterAddRequest.getClusterType())) {
|
||||
|
||||
|
||||
if (StringUtils.isBlank(clusterAddRequest.getClusterName()) || StringUtils
|
||||
.isBlank(clusterAddRequest.getClusterType())) {
|
||||
|
||||
throw new SkyWalkerException("集群名字或者类型不能为空!");
|
||||
}
|
||||
|
||||
|
||||
if (!ClusterTypeEnum.contains(clusterAddRequest.getClusterType())) {
|
||||
|
||||
|
||||
throw new SkyWalkerException("集群类型不存在:" + clusterAddRequest.getClusterType());
|
||||
}
|
||||
|
||||
|
||||
String clusterId = SkyWalkerUtil.generateClusterId(clusterAddRequest);
|
||||
|
||||
|
||||
if (null != clusterAccessService.findByClusterId(clusterId)) {
|
||||
|
||||
|
||||
throw new SkyWalkerException("重复插入,clusterId已存在:" + clusterId);
|
||||
}
|
||||
|
||||
|
||||
clusterDO.setClusterId(clusterId);
|
||||
clusterDO.setClusterName(clusterAddRequest.getClusterName());
|
||||
clusterDO.setClusterType(clusterAddRequest.getClusterType());
|
||||
clusterDO.setConnectKeyList(objectMapper.writeValueAsString(clusterAddRequest.getConnectKeyList()));
|
||||
clusterDO.setConnectKeyList(JSONObject.toJSONString(clusterAddRequest.getConnectKeyList()));
|
||||
clusterDO.setUserName(clusterAddRequest.getUserName());
|
||||
clusterDO.setPassword(clusterAddRequest.getPassword());
|
||||
clusterDO.setNamespace(clusterAddRequest.getNamespace());
|
||||
clusterDO.setClusterLevel(0);
|
||||
clusterAccessService.insert(clusterDO);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,43 +14,31 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.exception.SkyWalkerException;
|
||||
import com.alibaba.nacossync.pojo.request.ClusterDeleteRequest;
|
||||
import com.alibaba.nacossync.pojo.result.ClusterDeleteResult;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.pojo.result.ClusterDeleteResult;
|
||||
import com.alibaba.nacossync.pojo.request.ClusterDeleteRequest;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: ClusterDeleteProcessor.java, v 0.1 2018-09-30 PM2:43 NacosSync Exp $$
|
||||
*/
|
||||
@Service
|
||||
public class ClusterDeleteProcessor implements Processor<ClusterDeleteRequest, ClusterDeleteResult> {
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
public ClusterDeleteProcessor(ClusterAccessService clusterAccessService, TaskAccessService taskAccessService) {
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
this.taskAccessService = taskAccessService;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private ClusterAccessService clusterAccessService;
|
||||
|
||||
@Override
|
||||
public void process(ClusterDeleteRequest clusterDeleteRequest, ClusterDeleteResult clusterDeleteResult,
|
||||
Object... others) throws Exception {
|
||||
int count = taskAccessService.countByDestClusterIdOrSourceClusterId(clusterDeleteRequest.getClusterId(),
|
||||
clusterDeleteRequest.getClusterId());
|
||||
if (count > 0) {
|
||||
throw new SkyWalkerException(String.format("集群下有%d个任务,请先删除任务", count));
|
||||
}
|
||||
|
||||
public void process(ClusterDeleteRequest clusterDeleteRequest,
|
||||
ClusterDeleteResult clusterDeleteResult, Object... others) throws Exception {
|
||||
|
||||
clusterAccessService.deleteByClusterId(clusterDeleteRequest.getClusterId());
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,37 +14,44 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.pojo.result.ClusterDetailQueryResult;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import com.alibaba.nacossync.pojo.request.ClusterDetailQueryRequest;
|
||||
import com.alibaba.nacossync.pojo.result.ClusterDetailQueryResult;
|
||||
import com.alibaba.nacossync.pojo.view.ClusterModel;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: ClusterDetailQueryProcessor.java, v 0.1 2018-09-30 PM2:39 NacosSync Exp $$
|
||||
*/
|
||||
@Service
|
||||
public class ClusterDetailQueryProcessor implements Processor<ClusterDetailQueryRequest, ClusterDetailQueryResult> {
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
public ClusterDetailQueryProcessor(ClusterAccessService clusterAccessService) {
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
}
|
||||
|
||||
public class ClusterDetailQueryProcessor
|
||||
implements
|
||||
Processor<ClusterDetailQueryRequest, ClusterDetailQueryResult> {
|
||||
@Autowired
|
||||
private ClusterAccessService clusterAccessService;
|
||||
|
||||
@Override
|
||||
public void process(ClusterDetailQueryRequest clusterDetailQueryRequest,
|
||||
ClusterDetailQueryResult clusterDetailQueryResult, Object... others) throws Exception {
|
||||
|
||||
ClusterDO clusterDO = clusterAccessService.findByClusterId(clusterDetailQueryRequest.getClusterId());
|
||||
|
||||
clusterDetailQueryResult.setClusterModel(ClusterModel.from(clusterDO));
|
||||
|
||||
ClusterDetailQueryResult clusterDetailQueryResult, Object... others)
|
||||
throws Exception {
|
||||
|
||||
ClusterDO clusterDO = clusterAccessService.findByClusterId(clusterDetailQueryRequest
|
||||
.getClusterId());
|
||||
|
||||
ClusterModel clusterModel = new ClusterModel();
|
||||
clusterModel.setClusterId(clusterDO.getClusterId());
|
||||
clusterModel.setConnectKeyList(clusterDO.getConnectKeyList());
|
||||
clusterModel.setClusterType(clusterDO.getClusterType());
|
||||
clusterModel.setClusterName(clusterDO.getClusterName());
|
||||
|
||||
clusterDetailQueryResult.setClusterModel(clusterModel);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,60 +14,70 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.alibaba.nacossync.pojo.QueryCondition;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import com.alibaba.nacossync.pojo.request.ClusterListQueryRequest;
|
||||
import com.alibaba.nacossync.pojo.result.ClusterListQueryResult;
|
||||
import com.alibaba.nacossync.pojo.view.ClusterModel;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.pojo.result.ClusterListQueryResult;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import com.alibaba.nacossync.pojo.request.ClusterListQueryRequest;
|
||||
import com.alibaba.nacossync.pojo.view.ClusterModel;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: ClusterListQueryProcessor.java, v 0.1 2018-09-30 PM2:33 NacosSync Exp $$
|
||||
*/
|
||||
@Service
|
||||
public class ClusterListQueryProcessor implements Processor<ClusterListQueryRequest, ClusterListQueryResult> {
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
public ClusterListQueryProcessor(ClusterAccessService clusterAccessService) {
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
}
|
||||
|
||||
public class ClusterListQueryProcessor implements
|
||||
Processor<ClusterListQueryRequest, ClusterListQueryResult> {
|
||||
|
||||
@Autowired
|
||||
private ClusterAccessService clusterAccessService;
|
||||
|
||||
@Override
|
||||
public void process(ClusterListQueryRequest clusterListQueryRequest, ClusterListQueryResult clusterListQueryResult,
|
||||
Object... others) {
|
||||
|
||||
public void process(ClusterListQueryRequest clusterListQueryRequest,
|
||||
ClusterListQueryResult clusterListQueryResult, Object... others) {
|
||||
|
||||
Page<ClusterDO> clusterDOS;
|
||||
|
||||
|
||||
if (StringUtils.isNotBlank(clusterListQueryRequest.getClusterName())) {
|
||||
|
||||
|
||||
QueryCondition queryCondition = new QueryCondition();
|
||||
queryCondition.setServiceName(clusterListQueryRequest.getClusterName());
|
||||
clusterDOS = clusterAccessService.findPageCriteria(clusterListQueryRequest.getPageNum() - 1,
|
||||
clusterListQueryRequest.getPageSize(), queryCondition);
|
||||
|
||||
|
||||
} else {
|
||||
|
||||
|
||||
clusterDOS = clusterAccessService.findPageNoCriteria(clusterListQueryRequest.getPageNum() - 1,
|
||||
clusterListQueryRequest.getPageSize());
|
||||
|
||||
|
||||
}
|
||||
List<ClusterModel> clusterModels = clusterDOS.stream().map(ClusterModel::from).toList();
|
||||
|
||||
List<ClusterModel> clusterModels = new ArrayList<>();
|
||||
clusterDOS.forEach(clusterDO -> {
|
||||
|
||||
ClusterModel clusterModel = new ClusterModel();
|
||||
clusterModel.setClusterId(clusterDO.getClusterId());
|
||||
clusterModel.setClusterName(clusterDO.getClusterName());
|
||||
clusterModel.setClusterType(clusterDO.getClusterType());
|
||||
clusterModel.setConnectKeyList(clusterDO.getConnectKeyList());
|
||||
clusterModels.add(clusterModel);
|
||||
});
|
||||
|
||||
clusterListQueryResult.setClusterModels(clusterModels);
|
||||
clusterListQueryResult.setTotalPage(clusterDOS.getTotalPages());
|
||||
clusterListQueryResult.setCurrentSize(clusterModels.size());
|
||||
clusterListQueryResult.setTotalSize(clusterDOS.getTotalElements());
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,250 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.nacos.api.exception.NacosException;
|
||||
import com.alibaba.nacos.api.naming.CommonParams;
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.client.naming.NacosNamingService;
|
||||
import com.alibaba.nacos.client.naming.net.NamingProxy;
|
||||
import com.alibaba.nacos.client.naming.utils.UtilAndComs;
|
||||
import com.alibaba.nacos.common.utils.HttpMethod;
|
||||
import com.alibaba.nacos.common.utils.JacksonUtils;
|
||||
import com.alibaba.nacos.common.utils.StringUtils;
|
||||
import com.alibaba.nacossync.constant.TaskStatusEnum;
|
||||
import com.alibaba.nacossync.dao.ClusterAccessService;
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.exception.SkyWalkerException;
|
||||
import com.alibaba.nacossync.extension.SyncManagerService;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.pojo.model.ClusterDO;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.pojo.request.TaskAddAllRequest;
|
||||
import com.alibaba.nacossync.pojo.request.TaskAddRequest;
|
||||
import com.alibaba.nacossync.pojo.result.TaskAddResult;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.ReflectionUtils;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import static com.alibaba.nacossync.constant.SkyWalkerConstants.GROUP_NAME_PARAM;
|
||||
import static com.alibaba.nacossync.constant.SkyWalkerConstants.PAGE_NO;
|
||||
import static com.alibaba.nacossync.constant.SkyWalkerConstants.PAGE_SIZE;
|
||||
import static com.alibaba.nacossync.constant.SkyWalkerConstants.SERVICE_NAME_PARAM;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
* @version $Id: TaskAddAllProcessor.java, v 0.1 2022-03-23 PM11:40 NacosSync Exp $$
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class TaskAddAllProcessor implements Processor<TaskAddAllRequest, TaskAddResult> {
|
||||
|
||||
private static final String CONSUMER_PREFIX = "consumers:";
|
||||
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
|
||||
private final SyncManagerService syncManagerService;
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
private final ClusterAccessService clusterAccessService;
|
||||
|
||||
public TaskAddAllProcessor(NacosServerHolder nacosServerHolder, SyncManagerService syncManagerService,
|
||||
TaskAccessService taskAccessService, ClusterAccessService clusterAccessService) {
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.syncManagerService = syncManagerService;
|
||||
this.taskAccessService = taskAccessService;
|
||||
this.clusterAccessService = clusterAccessService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(TaskAddAllRequest addAllRequest, TaskAddResult taskAddResult, Object... others)
|
||||
throws Exception {
|
||||
|
||||
ClusterDO destCluster = clusterAccessService.findByClusterId(addAllRequest.getDestClusterId());
|
||||
|
||||
ClusterDO sourceCluster = clusterAccessService.findByClusterId(addAllRequest.getSourceClusterId());
|
||||
|
||||
if (Objects.isNull(destCluster) || Objects.isNull(sourceCluster)) {
|
||||
throw new SkyWalkerException("Please check if the source or target cluster exists.");
|
||||
}
|
||||
|
||||
if (Objects.isNull(syncManagerService.getSyncService(sourceCluster.getClusterId(), destCluster.getClusterId()))) {
|
||||
throw new SkyWalkerException("current sync type not supported.");
|
||||
}
|
||||
// TODO 目前仅支持 Nacos 为源的同步类型,待完善更多类型支持。
|
||||
final NamingService sourceNamingService = nacosServerHolder.get(sourceCluster.getClusterId());
|
||||
if (sourceNamingService == null) {
|
||||
throw new SkyWalkerException("only support sync type that the source of the Nacos.");
|
||||
}
|
||||
|
||||
final EnhanceNamingService enhanceNamingService = new EnhanceNamingService(sourceNamingService);
|
||||
final CatalogServiceResult catalogServiceResult = enhanceNamingService.catalogServices(null, null);
|
||||
if (catalogServiceResult == null || catalogServiceResult.getCount() <= 0) {
|
||||
throw new SkyWalkerException("sourceCluster data empty");
|
||||
}
|
||||
|
||||
for (ServiceView serviceView : catalogServiceResult.getServiceList()) {
|
||||
// exclude subscriber
|
||||
if (addAllRequest.isExcludeConsumer() && serviceView.getName().startsWith(CONSUMER_PREFIX)) {
|
||||
continue;
|
||||
}
|
||||
TaskAddRequest taskAddRequest = new TaskAddRequest();
|
||||
taskAddRequest.setSourceClusterId(sourceCluster.getClusterId());
|
||||
taskAddRequest.setDestClusterId(destCluster.getClusterId());
|
||||
taskAddRequest.setServiceName(serviceView.getName());
|
||||
taskAddRequest.setGroupName(serviceView.getGroupName());
|
||||
this.dealTask(addAllRequest, taskAddRequest);
|
||||
}
|
||||
}
|
||||
|
||||
private void dealTask(TaskAddAllRequest addAllRequest, TaskAddRequest taskAddRequest) throws Exception {
|
||||
|
||||
String taskId = SkyWalkerUtil.generateTaskId(taskAddRequest);
|
||||
TaskDO taskDO = taskAccessService.findByTaskId(taskId);
|
||||
if (null == taskDO) {
|
||||
taskDO = new TaskDO();
|
||||
taskDO.setTaskId(taskId);
|
||||
taskDO.setDestClusterId(addAllRequest.getDestClusterId());
|
||||
taskDO.setSourceClusterId(addAllRequest.getSourceClusterId());
|
||||
taskDO.setServiceName(taskAddRequest.getServiceName());
|
||||
taskDO.setVersion(taskAddRequest.getVersion());
|
||||
taskDO.setGroupName(taskAddRequest.getGroupName());
|
||||
taskDO.setNameSpace(taskAddRequest.getNameSpace());
|
||||
taskDO.setTaskStatus(TaskStatusEnum.SYNC.getCode());
|
||||
taskDO.setWorkerIp(SkyWalkerUtil.getLocalIp());
|
||||
taskDO.setOperationId(SkyWalkerUtil.generateOperationId());
|
||||
|
||||
} else {
|
||||
taskDO.setTaskStatus(TaskStatusEnum.SYNC.getCode());
|
||||
taskDO.setOperationId(SkyWalkerUtil.generateOperationId());
|
||||
}
|
||||
taskAccessService.addTask(taskDO);
|
||||
}
|
||||
|
||||
static class EnhanceNamingService {
|
||||
|
||||
protected NamingService delegate;
|
||||
|
||||
protected NamingProxy serverProxy;
|
||||
|
||||
protected EnhanceNamingService(NamingService namingService) {
|
||||
if (!(namingService instanceof NacosNamingService)) {
|
||||
throw new IllegalArgumentException(
|
||||
"namingService only support instance of com.alibaba.nacos.client.naming.NacosNamingService.");
|
||||
}
|
||||
this.delegate = namingService;
|
||||
|
||||
// serverProxy
|
||||
final Field serverProxyField = ReflectionUtils.findField(NacosNamingService.class, "serverProxy");
|
||||
assert serverProxyField != null;
|
||||
ReflectionUtils.makeAccessible(serverProxyField);
|
||||
this.serverProxy = (NamingProxy) ReflectionUtils.getField(serverProxyField, delegate);
|
||||
}
|
||||
|
||||
public CatalogServiceResult catalogServices(@Nullable String serviceName, @Nullable String group)
|
||||
throws NacosException {
|
||||
int pageNo = 1; // start with 1
|
||||
int pageSize = 100;
|
||||
|
||||
final CatalogServiceResult result = catalogServices(serviceName, group, pageNo, pageSize);
|
||||
|
||||
CatalogServiceResult tmpResult = result;
|
||||
|
||||
while (Objects.nonNull(tmpResult) && tmpResult.serviceList.size() >= pageSize) {
|
||||
pageNo++;
|
||||
tmpResult = catalogServices(serviceName, group, pageNo, pageSize);
|
||||
|
||||
if (tmpResult != null) {
|
||||
result.serviceList.addAll(tmpResult.serviceList);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @see com.alibaba.nacos.client.naming.core.HostReactor#getServiceInfoDirectlyFromServer(String, String)
|
||||
*/
|
||||
public CatalogServiceResult catalogServices(@Nullable String serviceName, @Nullable String group, int pageNo,
|
||||
int pageSize) throws NacosException {
|
||||
|
||||
// pageNo
|
||||
// pageSize
|
||||
// serviceNameParam
|
||||
// groupNameParam
|
||||
final Map<String, String> params = new HashMap<>(8);
|
||||
params.put(CommonParams.NAMESPACE_ID, serverProxy.getNamespaceId());
|
||||
params.put(SERVICE_NAME_PARAM, serviceName);
|
||||
params.put(GROUP_NAME_PARAM, group);
|
||||
params.put(PAGE_NO, String.valueOf(pageNo));
|
||||
params.put(PAGE_SIZE, String.valueOf(pageSize));
|
||||
|
||||
final String result = this.serverProxy.reqApi(UtilAndComs.nacosUrlBase + "/catalog/services", params,
|
||||
HttpMethod.GET);
|
||||
if (StringUtils.isNotEmpty(result)) {
|
||||
return JacksonUtils.toObj(result, CatalogServiceResult.class);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy from Nacos Server.
|
||||
*/
|
||||
@Data
|
||||
static class ServiceView {
|
||||
|
||||
private String name;
|
||||
|
||||
private String groupName;
|
||||
|
||||
private int clusterCount;
|
||||
|
||||
private int ipCount;
|
||||
|
||||
private int healthyInstanceCount;
|
||||
|
||||
private String triggerFlag;
|
||||
|
||||
}
|
||||
|
||||
@Data
|
||||
static class CatalogServiceResult {
|
||||
|
||||
/**
|
||||
* count,not equal serviceList.size .
|
||||
*/
|
||||
private int count;
|
||||
|
||||
private List<ServiceView> serviceList;
|
||||
|
||||
}
|
||||
|
||||
}
|
|
@ -89,7 +89,7 @@ public class TaskAddProcessor implements Processor<TaskAddRequest, TaskAddResult
|
|||
taskDO.setTaskStatus(TaskStatusEnum.SYNC.getCode());
|
||||
taskDO.setWorkerIp(SkyWalkerUtil.getLocalIp());
|
||||
taskDO.setOperationId(SkyWalkerUtil.generateOperationId());
|
||||
|
||||
|
||||
} else {
|
||||
|
||||
taskDO.setTaskStatus(TaskStatusEnum.SYNC.getCode());
|
||||
|
|
|
@ -16,12 +16,19 @@
|
|||
*/
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.pojo.request.TaskDeleteInBatchRequest;
|
||||
import com.alibaba.nacossync.pojo.result.BaseResult;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
|
||||
/**
|
||||
|
@ -33,15 +40,20 @@ import org.springframework.stereotype.Service;
|
|||
@Service
|
||||
public class TaskDeleteInBatchProcessor implements Processor<TaskDeleteInBatchRequest, BaseResult> {
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
public TaskDeleteInBatchProcessor(TaskAccessService taskAccessService) {
|
||||
this.taskAccessService = taskAccessService;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
|
||||
@Override
|
||||
public void process(TaskDeleteInBatchRequest taskBatchDeleteRequest, BaseResult baseResult,
|
||||
Object... others) {
|
||||
//
|
||||
// String[] taskIds= taskBatchDeleteRequest.getTaskIds();
|
||||
// List<TaskDO> taskDOs = new ArrayList<TaskDO>();
|
||||
// for (String taskId : taskIds) {
|
||||
// TaskDO taskDO = new TaskDO();
|
||||
// taskDO.setTaskId(taskId);
|
||||
// taskDOs.add(taskDO);
|
||||
// }
|
||||
taskAccessService.deleteTaskInBatch(taskBatchDeleteRequest.getTaskIds());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,12 +14,9 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.nacossync.constant.SkyWalkerConstants;
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.event.DeleteAllSubTaskEvent;
|
||||
import com.alibaba.nacossync.event.DeleteTaskEvent;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.pojo.request.TaskDeleteRequest;
|
||||
|
@ -27,6 +24,7 @@ import com.alibaba.nacossync.pojo.result.BaseResult;
|
|||
import com.alibaba.nacossync.template.Processor;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
|
@ -36,28 +34,18 @@ import org.springframework.stereotype.Service;
|
|||
@Slf4j
|
||||
@Service
|
||||
public class TaskDeleteProcessor implements Processor<TaskDeleteRequest, BaseResult> {
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
private final EventBus eventBus;
|
||||
|
||||
public TaskDeleteProcessor(TaskAccessService taskAccessService, EventBus eventBus) {
|
||||
this.taskAccessService = taskAccessService;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
@Autowired
|
||||
private EventBus eventBus;
|
||||
|
||||
@Override
|
||||
public void process(TaskDeleteRequest taskDeleteRequest, BaseResult baseResult, Object... others) {
|
||||
public void process(TaskDeleteRequest taskDeleteRequest, BaseResult baseResult,
|
||||
Object... others) {
|
||||
TaskDO taskDO = taskAccessService.findByTaskId(taskDeleteRequest.getTaskId());
|
||||
// delete all sub task when ServiceName is all
|
||||
if (SkyWalkerConstants.NACOS_ALL_SERVICE_NAME.equalsIgnoreCase(taskDO.getServiceName())) {
|
||||
eventBus.post(new DeleteAllSubTaskEvent(taskDO));
|
||||
} else {
|
||||
eventBus.post(new DeleteTaskEvent(taskDO));
|
||||
}
|
||||
log.info("删除同步任务数据之前,发出一个同步事件:{}", taskDO);
|
||||
eventBus.post(new DeleteTaskEvent(taskDO));
|
||||
log.info("删除同步任务数据之前,发出一个同步事件:" + taskDO);
|
||||
taskAccessService.deleteTaskById(taskDeleteRequest.getTaskId());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -14,17 +14,17 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.exception.SkyWalkerException;
|
||||
import com.alibaba.nacossync.pojo.result.TaskDetailQueryResult;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.pojo.request.TaskDetailQueryRequest;
|
||||
import com.alibaba.nacossync.pojo.result.TaskDetailQueryResult;
|
||||
import com.alibaba.nacossync.pojo.view.TaskModel;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
|
@ -34,23 +34,30 @@ import org.springframework.stereotype.Service;
|
|||
@Slf4j
|
||||
@Service
|
||||
public class TaskDetailProcessor implements Processor<TaskDetailQueryRequest, TaskDetailQueryResult> {
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
public TaskDetailProcessor(TaskAccessService taskAccessService) {
|
||||
this.taskAccessService = taskAccessService;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
|
||||
@Override
|
||||
public void process(TaskDetailQueryRequest taskDetailQueryRequest, TaskDetailQueryResult taskDetailQueryResult,
|
||||
Object... others) throws Exception {
|
||||
|
||||
public void process(TaskDetailQueryRequest taskDetailQueryRequest, TaskDetailQueryResult taskDetailQueryResult, Object... others)
|
||||
throws Exception {
|
||||
|
||||
TaskDO taskDO = taskAccessService.findByTaskId(taskDetailQueryRequest.getTaskId());
|
||||
|
||||
|
||||
if (null == taskDO) {
|
||||
throw new SkyWalkerException("taskDo is null,taskId :" + taskDetailQueryRequest.getTaskId());
|
||||
}
|
||||
|
||||
taskDetailQueryResult.setTaskModel(TaskModel.from(taskDO));
|
||||
|
||||
TaskModel taskModel = new TaskModel();
|
||||
|
||||
taskModel.setDestClusterId(taskDO.getDestClusterId());
|
||||
taskModel.setGroupName(taskDO.getGroupName());
|
||||
taskModel.setServiceName(taskDO.getServiceName());
|
||||
taskModel.setSourceClusterId(taskDO.getSourceClusterId());
|
||||
taskModel.setTaskStatus(taskDO.getTaskStatus());
|
||||
taskModel.setTaskId(taskDO.getTaskId());
|
||||
taskModel.setNameSpace(taskDO.getNameSpace());
|
||||
|
||||
taskDetailQueryResult.setTaskModel(taskModel);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,23 +14,25 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.alibaba.nacossync.pojo.QueryCondition;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.pojo.request.TaskListQueryRequest;
|
||||
import com.alibaba.nacossync.pojo.result.TaskListQueryResult;
|
||||
import com.alibaba.nacossync.pojo.view.TaskModel;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.pojo.result.TaskListQueryResult;
|
||||
import com.alibaba.nacossync.pojo.request.TaskListQueryRequest;
|
||||
import com.alibaba.nacossync.pojo.view.TaskModel;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -39,34 +41,43 @@ import java.util.List;
|
|||
@Service
|
||||
@Slf4j
|
||||
public class TaskListQueryProcessor implements Processor<TaskListQueryRequest, TaskListQueryResult> {
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
public TaskListQueryProcessor(TaskAccessService taskAccessService) {
|
||||
this.taskAccessService = taskAccessService;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
|
||||
@Override
|
||||
public void process(TaskListQueryRequest taskListQueryRequest, TaskListQueryResult taskListQueryResult,
|
||||
Object... others) {
|
||||
|
||||
public void process(TaskListQueryRequest taskListQueryRequest,
|
||||
TaskListQueryResult taskListQueryResult, Object... others) {
|
||||
|
||||
Page<TaskDO> taskDOPage;
|
||||
|
||||
|
||||
if (StringUtils.isNotBlank(taskListQueryRequest.getServiceName())) {
|
||||
|
||||
|
||||
QueryCondition queryCondition = new QueryCondition();
|
||||
queryCondition.setServiceName(taskListQueryRequest.getServiceName());
|
||||
taskDOPage = taskAccessService.findPageCriteria(taskListQueryRequest.getPageNum() - 1,
|
||||
taskListQueryRequest.getPageSize(), queryCondition);
|
||||
} else {
|
||||
|
||||
|
||||
taskDOPage = taskAccessService.findPageNoCriteria(taskListQueryRequest.getPageNum() - 1,
|
||||
taskListQueryRequest.getPageSize());
|
||||
|
||||
|
||||
}
|
||||
|
||||
List<TaskModel> taskList = taskDOPage.stream().map(TaskModel::from).toList();
|
||||
|
||||
|
||||
List<TaskModel> taskList = new ArrayList<>();
|
||||
|
||||
taskDOPage.forEach(taskDO -> {
|
||||
TaskModel taskModel = new TaskModel();
|
||||
taskModel.setTaskId(taskDO.getTaskId());
|
||||
taskModel.setDestClusterId(taskDO.getDestClusterId());
|
||||
taskModel.setSourceClusterId(taskDO.getSourceClusterId());
|
||||
taskModel.setServiceName(taskDO.getServiceName());
|
||||
taskModel.setGroupName(taskDO.getGroupName());
|
||||
taskModel.setTaskStatus(taskDO.getTaskStatus());
|
||||
taskModel.setNameSpace(taskDO.getNameSpace());
|
||||
taskList.add(taskModel);
|
||||
});
|
||||
|
||||
taskListQueryResult.setTaskModels(taskList);
|
||||
taskListQueryResult.setTotalPage(taskDOPage.getTotalPages());
|
||||
taskListQueryResult.setTotalSize(taskDOPage.getTotalElements());
|
||||
|
|
|
@ -14,19 +14,21 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.template.processor;
|
||||
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.constant.TaskStatusEnum;
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.exception.SkyWalkerException;
|
||||
import com.alibaba.nacossync.pojo.result.BaseResult;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.pojo.request.TaskUpdateRequest;
|
||||
import com.alibaba.nacossync.pojo.result.BaseResult;
|
||||
import com.alibaba.nacossync.template.Processor;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -35,33 +37,29 @@ import org.springframework.stereotype.Service;
|
|||
@Slf4j
|
||||
@Service
|
||||
public class TaskUpdateProcessor implements Processor<TaskUpdateRequest, BaseResult> {
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
public TaskUpdateProcessor(TaskAccessService taskAccessService) {
|
||||
this.taskAccessService = taskAccessService;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
|
||||
@Override
|
||||
public void process(TaskUpdateRequest taskUpdateRequest, BaseResult baseResult, Object... others) throws Exception {
|
||||
|
||||
public void process(TaskUpdateRequest taskUpdateRequest, BaseResult baseResult,
|
||||
Object... others) throws Exception {
|
||||
|
||||
TaskDO taskDO = taskAccessService.findByTaskId(taskUpdateRequest.getTaskId());
|
||||
|
||||
|
||||
if (!TaskStatusEnum.contains(taskUpdateRequest.getTaskStatus())) {
|
||||
throw new SkyWalkerException(
|
||||
"taskUpdateRequest.getTaskStatus() is not exist , value is :" + taskUpdateRequest.getTaskStatus());
|
||||
"taskUpdateRequest.getTaskStatus() is not exist , value is :"
|
||||
+ taskUpdateRequest.getTaskStatus());
|
||||
}
|
||||
|
||||
|
||||
if (null == taskDO) {
|
||||
throw new SkyWalkerException("taskDo is null ,taskId is :" + taskUpdateRequest.getTaskId());
|
||||
throw new SkyWalkerException("taskDo is null ,taskId is :"
|
||||
+ taskUpdateRequest.getTaskId());
|
||||
}
|
||||
|
||||
|
||||
taskDO.setTaskStatus(taskUpdateRequest.getTaskStatus());
|
||||
|
||||
taskDO.setOperationId(SkyWalkerUtil.generateOperationId());
|
||||
|
||||
|
||||
taskAccessService.addTask(taskDO);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -1,252 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.timer;
|
||||
|
||||
import com.alibaba.nacos.api.naming.NamingService;
|
||||
import com.alibaba.nacos.api.naming.pojo.ListView;
|
||||
import com.alibaba.nacos.client.naming.utils.CollectionUtils;
|
||||
import com.alibaba.nacossync.constant.MetricsStatisticsType;
|
||||
import com.alibaba.nacossync.constant.TaskStatusEnum;
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.event.DeleteAllSubTaskEvent;
|
||||
import com.alibaba.nacossync.event.DeleteTaskEvent;
|
||||
import com.alibaba.nacossync.event.SyncTaskEvent;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.alibaba.nacossync.util.BatchTaskExecutor;
|
||||
import com.alibaba.nacossync.util.SkyWalkerUtil;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* when the database task service name is empty, check all the services in the cluster and create a synchronization
|
||||
* task.
|
||||
*/
|
||||
@Slf4j
|
||||
public class CheckRunningStatusAllNacosThread implements Runnable {
|
||||
|
||||
private final Map<String, Set<String>> subTaskService = new ConcurrentHashMap<>();
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
|
||||
private final EventBus eventBus;
|
||||
|
||||
|
||||
public CheckRunningStatusAllNacosThread(MetricsManager metricsManager, TaskAccessService taskAccessService,
|
||||
NacosServerHolder nacosServerHolder, EventBus eventBus) {
|
||||
this.metricsManager = metricsManager;
|
||||
this.taskAccessService = taskAccessService;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
this.eventBus = eventBus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronize data based on the ns level.
|
||||
*/
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
try {
|
||||
List<TaskDO> tasks = taskAccessService.findAllByServiceNameEqualAll();
|
||||
if (CollectionUtils.isEmpty(tasks)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the set of all task IDs
|
||||
Set<String> taskIdSet = tasks.stream().map(TaskDO::getTaskId).collect(Collectors.toSet());
|
||||
|
||||
// Filter and handle sub-tasks that need to be deleted. This handles the case where tasks have been deleted
|
||||
// but sub-tasks still exist.
|
||||
subTaskService.entrySet().stream()
|
||||
.filter(entry -> shouldDeleteSubTasks(entry, taskIdSet))
|
||||
.forEach(entry -> postDeleteAllSubTaskEvent(entry.getKey()));
|
||||
|
||||
// Handle regular tasks
|
||||
tasks.forEach(this::processTask);
|
||||
} catch (Exception e) {
|
||||
log.warn("CheckRunningStatusThread Exception", e);
|
||||
metricsManager.recordError(MetricsStatisticsType.DISPATCHER_TASK);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Listens for the event of deleting all sub-tasks and handles the delete operation.
|
||||
*
|
||||
* @param deleteAllSubTaskEvent The event object containing the task information to be deleted.
|
||||
*/
|
||||
public void listenerDeleteAllTaskEvent(DeleteAllSubTaskEvent deleteAllSubTaskEvent) {
|
||||
// Retrieve the task object
|
||||
TaskDO task = deleteAllSubTaskEvent.getTaskDO();
|
||||
|
||||
// Retrieve the task ID
|
||||
String taskId = task.getTaskId();
|
||||
|
||||
// Remove the set of service names corresponding to the task ID from subTaskService
|
||||
Set<String> serviceNameSet = subTaskService.remove(taskId);
|
||||
|
||||
// If the set of service names is empty, return immediately
|
||||
if (CollectionUtils.isEmpty(serviceNameSet)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Build the list of sub-tasks pending removal
|
||||
List<TaskDO> servicesPendingRemoval = serviceNameSet.stream()
|
||||
.map(serviceName -> buildSubTaskDO(task, serviceName))
|
||||
.toList();
|
||||
|
||||
// Handle the removal of the pending sub-tasks
|
||||
handleRemoval(servicesPendingRemoval, serviceNameSet);
|
||||
}
|
||||
|
||||
|
||||
private boolean shouldDeleteSubTasks(Map.Entry<String, Set<String>> entry, Set<String> taskIdSet) {
|
||||
return !taskIdSet.contains(entry.getKey()) && !entry.getValue().isEmpty();
|
||||
}
|
||||
|
||||
private void postDeleteAllSubTaskEvent(String taskId) {
|
||||
TaskDO taskDO = new TaskDO();
|
||||
taskDO.setTaskId(taskId);
|
||||
eventBus.post(new DeleteAllSubTaskEvent(taskDO));
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes the given task by determining the services that need to be inserted and removed,
|
||||
* and performs the corresponding operations.
|
||||
*
|
||||
* @param task The task object to be processed.
|
||||
*/
|
||||
private void processTask(TaskDO task) {
|
||||
// Retrieve the set of services for the task, creating a new set if it does not exist
|
||||
Set<String> serviceSet = subTaskService.computeIfAbsent(task.getTaskId(), k -> ConcurrentHashMap.newKeySet());
|
||||
|
||||
// Get the list of all service names associated with the task
|
||||
List<String> serviceNameList = getAllServiceName(task);
|
||||
|
||||
// Determine the services that need to be inserted (those not in the current service set)
|
||||
List<TaskDO> servicesPendingInsertion = serviceNameList.stream()
|
||||
.filter(serviceName -> !serviceSet.contains(serviceName))
|
||||
.map(serviceName -> buildSubTaskDO(task, serviceName))
|
||||
.toList();
|
||||
|
||||
// Determine the services that need to be removed (those in the current service set but not in the service name list)
|
||||
List<TaskDO> servicesPendingRemoval = serviceSet.stream()
|
||||
.filter(serviceName -> !serviceNameList.contains(serviceName))
|
||||
.map(serviceName -> buildSubTaskDO(task, serviceName))
|
||||
.toList();
|
||||
|
||||
|
||||
// If all lists are empty, there is nothing to process
|
||||
if (CollectionUtils.isEmpty(serviceNameList) && CollectionUtils.isEmpty(servicesPendingInsertion)
|
||||
&& CollectionUtils.isEmpty(servicesPendingRemoval)) {
|
||||
log.debug("No service found for task: {}", task.getTaskId());
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
// If the task status is SYNC, handle the insertion of services
|
||||
if (TaskStatusEnum.SYNC.getCode().equals(task.getTaskStatus())) {
|
||||
handleInsertion(servicesPendingInsertion, serviceSet);
|
||||
}
|
||||
// Handle the removal of services
|
||||
handleRemoval(servicesPendingRemoval, serviceSet);
|
||||
|
||||
if (TaskStatusEnum.DELETE.getCode().equals(task.getTaskStatus())) {
|
||||
List<TaskDO> allSubTasks = serviceNameList.stream().map(serviceName -> buildSubTaskDO(task, serviceName))
|
||||
.collect(Collectors.toList());
|
||||
handleRemoval(allSubTasks, serviceSet);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles the insertion of services.
|
||||
*
|
||||
* @param servicesPendingInsertion The list of services to be inserted.
|
||||
* @param serviceSet The set of services.
|
||||
*/
|
||||
private void handleInsertion(List<TaskDO> servicesPendingInsertion, Set<String> serviceSet) {
|
||||
BatchTaskExecutor.batchOperation(servicesPendingInsertion, t -> {
|
||||
eventBus.post(new SyncTaskEvent(t));
|
||||
serviceSet.add(t.getServiceName());
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Handles the removal of services.
|
||||
*
|
||||
* @param servicesPendingRemoval The list of services to be removed.
|
||||
* @param serviceSet The set of services.
|
||||
*/
|
||||
private void handleRemoval(List<TaskDO> servicesPendingRemoval, Set<String> serviceSet) {
|
||||
BatchTaskExecutor.batchOperation(servicesPendingRemoval, t -> {
|
||||
eventBus.post(new DeleteTaskEvent(t));
|
||||
serviceSet.remove(t.getServiceName());
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Builds a sub-task object for the given task and service name.
|
||||
*
|
||||
* @param serviceName The service name.
|
||||
* @return The constructed sub-task object.
|
||||
*/
|
||||
private static TaskDO buildSubTaskDO(TaskDO taskDO, String serviceName) {
|
||||
TaskDO task = new TaskDO();
|
||||
|
||||
BeanUtils.copyProperties(taskDO, task);
|
||||
task.setTaskId(SkyWalkerUtil.generateTaskId(serviceName, taskDO.getGroupName(), taskDO.getSourceClusterId(),
|
||||
taskDO.getDestClusterId()));
|
||||
task.setServiceName(serviceName);
|
||||
return task;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves all service names associated with the given task.
|
||||
*
|
||||
* @return A list of service names.
|
||||
*/
|
||||
private List<String> getAllServiceName(TaskDO taskDO) {
|
||||
NamingService namingService = nacosServerHolder.get(taskDO.getSourceClusterId());
|
||||
if (namingService == null) {
|
||||
log.warn("naming service is null or not found, clusterId:{}", taskDO.getSourceClusterId());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
try {
|
||||
ListView<String> servicesOfServer = namingService.getServicesOfServer(0, Integer.MAX_VALUE,
|
||||
taskDO.getGroupName());
|
||||
return servicesOfServer.getData();
|
||||
} catch (Exception e) {
|
||||
log.error("query service list failure", e);
|
||||
}
|
||||
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
|
@ -14,23 +14,24 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.alibaba.nacossync.timer;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.alibaba.nacossync.cache.SkyWalkerCacheServices;
|
||||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.pojo.FinishedTask;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
/**
|
||||
* @author NacosSync
|
||||
|
@ -39,53 +40,57 @@ import java.util.stream.StreamSupport;
|
|||
@Slf4j
|
||||
@Service
|
||||
public class CleanExceedOperationIdTimer implements CommandLineRunner {
|
||||
|
||||
private static final long INITIAL_DELAY = 0;
|
||||
|
||||
private static final long PERIOD = 12;
|
||||
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
|
||||
private final ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
public CleanExceedOperationIdTimer(SkyWalkerCacheServices skyWalkerCacheServices,
|
||||
TaskAccessService taskAccessService, ScheduledExecutorService scheduledExecutorService) {
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.taskAccessService = taskAccessService;
|
||||
this.scheduledExecutorService = scheduledExecutorService;
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
private SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
|
||||
@Autowired
|
||||
private ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
/** Clean up the OperationId cache once every 12 hours */
|
||||
scheduledExecutorService.scheduleWithFixedDelay(new CleanExceedOperationIdThread(), INITIAL_DELAY, PERIOD,
|
||||
TimeUnit.HOURS);
|
||||
log.info("CleanExceedOperationIdTimer has started successfully");
|
||||
|
||||
scheduledExecutorService.scheduleWithFixedDelay(new CleanExceedOperationIdThread(), 0, 12,
|
||||
TimeUnit.HOURS);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private class CleanExceedOperationIdThread implements Runnable {
|
||||
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
|
||||
try {
|
||||
|
||||
Map<String, FinishedTask> finishedTaskMap = skyWalkerCacheServices.getFinishedTaskMap();
|
||||
Set<String> operationIds = getDbOperations(taskAccessService.findAll());
|
||||
finishedTaskMap.keySet().removeIf(operationId -> !operationIds.contains(operationId));
|
||||
|
||||
|
||||
Map<String, FinishedTask> finishedTaskMap = skyWalkerCacheServices
|
||||
.getFinishedTaskMap();
|
||||
|
||||
Iterable<TaskDO> taskDOS = taskAccessService.findAll();
|
||||
|
||||
Set<String> operationIds = getDbOperations(taskDOS);
|
||||
|
||||
for (String operationId : finishedTaskMap.keySet()) {
|
||||
|
||||
if (!operationIds.contains(operationId)) {
|
||||
|
||||
finishedTaskMap.remove(operationId);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("CleanExceedOperationIdThread Exception", e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
private Set<String> getDbOperations(Iterable<TaskDO> taskDOS) {
|
||||
return StreamSupport.stream(taskDOS.spliterator(), false).map(TaskDO::getOperationId)
|
||||
.collect(Collectors.toSet());
|
||||
Set<String> operationIds = new HashSet<>();
|
||||
|
||||
taskDOS.forEach(taskDO -> operationIds.add(taskDO.getOperationId()));
|
||||
return operationIds;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -22,15 +22,14 @@ import com.alibaba.nacossync.constant.TaskStatusEnum;
|
|||
import com.alibaba.nacossync.dao.TaskAccessService;
|
||||
import com.alibaba.nacossync.event.DeleteTaskEvent;
|
||||
import com.alibaba.nacossync.event.SyncTaskEvent;
|
||||
import com.alibaba.nacossync.extension.holder.NacosServerHolder;
|
||||
import com.alibaba.nacossync.monitor.MetricsManager;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
@ -41,44 +40,27 @@ import java.util.concurrent.TimeUnit;
|
|||
@Slf4j
|
||||
@Service
|
||||
public class QuerySyncTaskTimer implements CommandLineRunner {
|
||||
|
||||
private static final int INITIAL_DELAY = 0;
|
||||
|
||||
private static final int DELAY = 3000;
|
||||
|
||||
private final MetricsManager metricsManager;
|
||||
@Autowired
|
||||
private MetricsManager metricsManager;
|
||||
|
||||
private final SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
@Autowired
|
||||
private SkyWalkerCacheServices skyWalkerCacheServices;
|
||||
|
||||
private final TaskAccessService taskAccessService;
|
||||
@Autowired
|
||||
private TaskAccessService taskAccessService;
|
||||
|
||||
private final EventBus eventBus;
|
||||
@Autowired
|
||||
private EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
private ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
private final ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
private final NacosServerHolder nacosServerHolder;
|
||||
|
||||
public QuerySyncTaskTimer(MetricsManager metricsManager, SkyWalkerCacheServices skyWalkerCacheServices,
|
||||
TaskAccessService taskAccessService, EventBus eventBus, ScheduledExecutorService scheduledExecutorService,
|
||||
NacosServerHolder nacosServerHolder) {
|
||||
this.metricsManager = metricsManager;
|
||||
this.skyWalkerCacheServices = skyWalkerCacheServices;
|
||||
this.taskAccessService = taskAccessService;
|
||||
this.eventBus = eventBus;
|
||||
this.scheduledExecutorService = scheduledExecutorService;
|
||||
this.nacosServerHolder = nacosServerHolder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
/** Fetch the task list from the database every 3 seconds */
|
||||
scheduledExecutorService.scheduleWithFixedDelay(new CheckRunningStatusThread(), INITIAL_DELAY, DELAY,
|
||||
scheduledExecutorService.scheduleWithFixedDelay(new CheckRunningStatusThread(), 0, 3000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
scheduledExecutorService.scheduleWithFixedDelay(new CheckRunningStatusAllNacosThread(metricsManager,
|
||||
taskAccessService, nacosServerHolder, eventBus), INITIAL_DELAY, DELAY,
|
||||
TimeUnit.MILLISECONDS);
|
||||
log.info("QuerySyncTaskTimer has started successfully");
|
||||
|
||||
}
|
||||
|
||||
private class CheckRunningStatusThread implements Runnable {
|
||||
|
@ -86,10 +68,10 @@ public class QuerySyncTaskTimer implements CommandLineRunner {
|
|||
@Override
|
||||
public void run() {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
Long start = System.currentTimeMillis();
|
||||
try {
|
||||
|
||||
List<TaskDO> taskDOS = taskAccessService.findAllByServiceNameNotEqualAll();
|
||||
Iterable<TaskDO> taskDOS = taskAccessService.findAll();
|
||||
|
||||
taskDOS.forEach(taskDO -> {
|
||||
|
||||
|
@ -101,13 +83,13 @@ public class QuerySyncTaskTimer implements CommandLineRunner {
|
|||
if (TaskStatusEnum.SYNC.getCode().equals(taskDO.getTaskStatus())) {
|
||||
|
||||
eventBus.post(new SyncTaskEvent(taskDO));
|
||||
log.info("从数据库中查询到一个同步任务,发出一个同步事件:{}", taskDO);
|
||||
log.info("从数据库中查询到一个同步任务,发出一个同步事件:" + taskDO);
|
||||
}
|
||||
|
||||
if (TaskStatusEnum.DELETE.getCode().equals(taskDO.getTaskStatus())) {
|
||||
|
||||
eventBus.post(new DeleteTaskEvent(taskDO));
|
||||
log.info("从数据库中查询到一个删除任务,发出一个同步事件:{}", taskDO);
|
||||
log.info("从数据库中查询到一个删除任务,发出一个同步事件:" + taskDO);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
@ -17,6 +17,7 @@ import com.alibaba.nacossync.extension.event.SpecialSyncEvent;
|
|||
import com.alibaba.nacossync.extension.event.SpecialSyncEventBus;
|
||||
import com.google.common.eventbus.EventBus;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
|
@ -32,25 +33,20 @@ import java.util.concurrent.TimeUnit;
|
|||
@Service
|
||||
public class SpecialSyncEventTimer implements CommandLineRunner {
|
||||
|
||||
private final SpecialSyncEventBus specialSyncEventBus;
|
||||
@Autowired
|
||||
private SpecialSyncEventBus specialSyncEventBus;
|
||||
|
||||
private final EventBus eventBus;
|
||||
@Autowired
|
||||
private EventBus eventBus;
|
||||
|
||||
@Autowired
|
||||
private ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
private final ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
public SpecialSyncEventTimer(SpecialSyncEventBus specialSyncEventBus, EventBus eventBus,
|
||||
ScheduledExecutorService scheduledExecutorService) {
|
||||
this.specialSyncEventBus = specialSyncEventBus;
|
||||
this.eventBus = eventBus;
|
||||
this.scheduledExecutorService = scheduledExecutorService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) throws Exception {
|
||||
|
||||
scheduledExecutorService.scheduleWithFixedDelay(new SpecialSyncEventTimer.SpecialSyncEventThread(), 0, 3000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
log.info("SpecialSyncEventTimer has started successfully");
|
||||
}
|
||||
|
||||
private class SpecialSyncEventThread implements Runnable {
|
||||
|
@ -62,9 +58,9 @@ public class SpecialSyncEventTimer implements CommandLineRunner {
|
|||
allSpecialSyncEvent.stream()
|
||||
.filter(specialSyncEvent -> TaskStatusEnum.SYNC.getCode()
|
||||
.equals(specialSyncEvent.getTaskDO().getTaskStatus()))
|
||||
.forEach(eventBus::post);
|
||||
.forEach(specialSyncEvent -> eventBus.post(specialSyncEvent));
|
||||
} catch (Exception e) {
|
||||
log.error("Exception occurred while processing special sync events", e);
|
||||
log.warn("SpecialSyncEventThread Exception", e);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,104 +0,0 @@
|
|||
package com.alibaba.nacossync.util;
|
||||
|
||||
import com.alibaba.nacos.client.naming.utils.CollectionUtils;
|
||||
import com.alibaba.nacossync.pojo.model.TaskDO;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
@Slf4j
|
||||
public class BatchTaskExecutor {
|
||||
|
||||
private static final int MAX_THREAD_NUM = 200;
|
||||
private static final ExecutorService executorService = Executors.newFixedThreadPool(MAX_THREAD_NUM);
|
||||
|
||||
/**
|
||||
* Batch operation method
|
||||
*
|
||||
* @param items Task list
|
||||
* @param operation Operation to be executed
|
||||
*/
|
||||
public static void batchOperation(List<TaskDO> items, Consumer<TaskDO> operation) {
|
||||
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||
|
||||
List<Tuple<Integer, List<TaskDO>>> taskGroupList = averageAssign(items, MAX_THREAD_NUM);
|
||||
|
||||
// Create a CompletableFuture for each task group
|
||||
CompletableFuture<?>[] futures = taskGroupList.stream().map(tuple -> CompletableFuture.runAsync(() -> {
|
||||
for (TaskDO taskDO : tuple.getT2()) {
|
||||
try {
|
||||
// Add timeout control for each task to avoid long-running tasks
|
||||
CompletableFuture.runAsync(() -> operation.accept(taskDO), executorService)
|
||||
.orTimeout(5, TimeUnit.SECONDS) // Task timeout set to 5 seconds
|
||||
.exceptionally(ex -> {
|
||||
log.error("Task execution timed out: {}", taskDO.getServiceName(), ex);
|
||||
return null;
|
||||
}).join();
|
||||
} catch (Exception e) {
|
||||
log.error("Error occurred during task execution: {}", taskDO.getServiceName(), e);
|
||||
}
|
||||
}
|
||||
}, executorService)).toArray(CompletableFuture[]::new);
|
||||
|
||||
try {
|
||||
// Wait for all tasks to complete
|
||||
CompletableFuture.allOf(futures).join();
|
||||
} catch (Exception e) {
|
||||
log.error("Error occurred during sync operation", e);
|
||||
} finally {
|
||||
log.info("Total sync tasks: {}, Execution time: {} ms", items.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Divide a list into n sublists, mainly implemented by offset
|
||||
* @param source collection to be divided
|
||||
* @param limit maximum value
|
||||
* @return list after division
|
||||
* @param <T> object type
|
||||
*/
|
||||
private static <T> List<Tuple<Integer, List<T>>> averageAssign(List<T> source, int limit) {
|
||||
if (CollectionUtils.isEmpty(source)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
int size = source.size();
|
||||
int listCount = (int) Math.ceil((double) size / limit); // Calculate the number of sublists
|
||||
int remainder = size % listCount; // Calculate the number of remaining elements after even distribution
|
||||
List<Tuple<Integer, List<T>>> result = new ArrayList<>(listCount); // Initialize the result list with the expected size
|
||||
|
||||
for (int i = 0, assigned = 0; i < listCount; i++) {
|
||||
int sublistSize = size / listCount + (remainder-- > 0 ? 1 : 0); // Determine the size of each sublist, distribute remaining elements
|
||||
List<T> sublist = new ArrayList<>(source.subList(assigned, assigned + sublistSize)); // Create the sublist
|
||||
result.add(Tuple.of(i, sublist)); // Add the sublist to the result
|
||||
assigned += sublistSize; // Update the assigned index
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the executor service to avoid resource leakage
|
||||
*/
|
||||
public static void shutdown() {
|
||||
executorService.shutdown();
|
||||
try {
|
||||
if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
|
||||
executorService.shutdownNow();
|
||||
if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
|
||||
log.error("Executor service did not terminate");
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException ie) {
|
||||
executorService.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue