dragonflyoss
/
client
mirror of https://github.com/dragonflyoss/client.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add log for collecter spawn (#204) 

Signed-off-by: Gaius <gaius.qi@gmail.com>
This commit is contained in:
Gaius 2024-01-09 12:31:47 +08:00 committed by GitHub
parent 65769f452d
commit 55a42d449b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 98 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/grpc/scheduler.rs
View File

@ -31,7 +31,7 @@ use std::sync::Arc;
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tokio::task::JoinSet; use tokio::task::JoinSet;
use tonic::transport::Channel; use tonic::transport::Channel;
use tracing::{error, info, instrument}; use tracing::{error, info, instrument, Instrument};
#[derive(Debug, Copy, Clone, Hash, PartialEq)] #[derive(Debug, Copy, Clone, Hash, PartialEq)]
struct VNode { struct VNode {
@ -152,6 +152,8 @@ impl SchedulerClient {
addr: SocketAddr, addr: SocketAddr,
request: tonic::Request<AnnounceHostRequest>, request: tonic::Request<AnnounceHostRequest>,
) -> Result<()> { ) -> Result<()> {
info!("announce host to {:?}", addr);
// Connect to the scheduler. // Connect to the scheduler.
let channel = Channel::from_shared(format!("http://{}", addr)) let channel = Channel::from_shared(format!("http://{}", addr))
.map_err(|_| Error::InvalidURI(addr.to_string()))? .map_err(|_| Error::InvalidURI(addr.to_string()))?
@ -164,7 +166,7 @@ impl SchedulerClient {
Ok(()) Ok(())
} }
join_set.spawn(announce_host(*available_scheduler_addr, request)); join_set.spawn(announce_host(*available_scheduler_addr, request).in_current_span());
} }
while let Some(message) = join_set.join_next().await { while let Some(message) = join_set.join_next().await {
@ -192,6 +194,8 @@ impl SchedulerClient {
addr: SocketAddr, addr: SocketAddr,
request: tonic::Request<AnnounceHostRequest>, request: tonic::Request<AnnounceHostRequest>,
) -> Result<()> { ) -> Result<()> {
info!("announce host to {}", addr);
// Connect to the scheduler. // Connect to the scheduler.
let channel = Channel::from_shared(format!("http://{}", addr)) let channel = Channel::from_shared(format!("http://{}", addr))
.map_err(|_| Error::InvalidURI(addr.to_string()))? .map_err(|_| Error::InvalidURI(addr.to_string()))?
@ -204,7 +208,7 @@ impl SchedulerClient {
Ok(()) Ok(())
} }
join_set.spawn(announce_host(*available_scheduler_addr, request)); join_set.spawn(announce_host(*available_scheduler_addr, request).in_current_span());
} }
while let Some(message) = join_set.join_next().await { while let Some(message) = join_set.join_next().await {
@ -231,6 +235,8 @@ impl SchedulerClient {
addr: SocketAddr, addr: SocketAddr,
request: tonic::Request<LeaveHostRequest>, request: tonic::Request<LeaveHostRequest>,
) -> Result<()> { ) -> Result<()> {
info!("leave host from {}", addr);
// Connect to the scheduler. // Connect to the scheduler.
let channel = Channel::from_shared(format!("http://{}", addr)) let channel = Channel::from_shared(format!("http://{}", addr))
.map_err(|_| Error::InvalidURI(addr.to_string()))? .map_err(|_| Error::InvalidURI(addr.to_string()))?
@ -243,7 +249,7 @@ impl SchedulerClient {
Ok(()) Ok(())
} }
join_set.spawn(leave_host(*available_scheduler_addr, request)); join_set.spawn(leave_host(*available_scheduler_addr, request).in_current_span());
} }
while let Some(message) = join_set.join_next().await { while let Some(message) = join_set.join_next().await {

87
src/task/mod.rs
View File

@ -45,7 +45,7 @@ use tokio::time::sleep;
use tokio_stream::{wrappers::ReceiverStream, StreamExt}; use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tonic::Request; use tonic::Request;
use tonic::Status; use tonic::Status;
use tracing::{error, info}; use tracing::{error, info, Instrument};
pub mod piece; pub mod piece;
pub mod piece_collector; pub mod piece_collector;
@ -695,6 +695,12 @@ impl Task {
storage: Arc<Storage>, storage: Arc<Storage>,
semaphore: Arc<Semaphore>, semaphore: Arc<Semaphore>,
) -> ClientResult<metadata::Piece> { ) -> ClientResult<metadata::Piece> {
info!(
"start to download piece {} from remote peer {:?}",
storage.piece_id(task_id.as_str(), number),
parent.id.clone()
);
let _permit = semaphore.acquire().await.map_err(|err| { let _permit = semaphore.acquire().await.map_err(|err| {
error!("acquire semaphore error: {:?}", err); error!("acquire semaphore error: {:?}", err);
Error::DownloadFromRemotePeerFailed(DownloadFromRemotePeerFailed { Error::DownloadFromRemotePeerFailed(DownloadFromRemotePeerFailed {
@ -722,14 +728,17 @@ impl Task {
Ok(metadata) Ok(metadata)
} }
join_set.spawn(download_from_remote_peer( join_set.spawn(
task_id.to_string(), download_from_remote_peer(
collect_piece.number, task_id.to_string(),
collect_piece.parent.clone(), collect_piece.number,
self.piece.clone(), collect_piece.parent.clone(),
self.storage.clone(), self.piece.clone(),
semaphore.clone(), self.storage.clone(),
)); semaphore.clone(),
)
.in_current_span(),
);
} }
// Initialize the finished pieces. // Initialize the finished pieces.
@ -890,8 +899,14 @@ impl Task {
length: u64, length: u64,
header: HeaderMap, header: HeaderMap,
piece: Arc<piece::Piece>, piece: Arc<piece::Piece>,
storage: Arc<Storage>,
semaphore: Arc<Semaphore>, semaphore: Arc<Semaphore>,
) -> ClientResult<metadata::Piece> { ) -> ClientResult<metadata::Piece> {
info!(
"start to download piece {} from source",
storage.piece_id(task_id.as_str(), number)
);
let _permit = semaphore.acquire().await?; let _permit = semaphore.acquire().await?;
let metadata = piece let metadata = piece
@ -908,16 +923,20 @@ impl Task {
Ok(metadata) Ok(metadata)
} }
join_set.spawn(download_from_source( join_set.spawn(
task_id.to_string(), download_from_source(
interested_piece.number, task_id.to_string(),
url.clone(), interested_piece.number,
interested_piece.offset, url.clone(),
interested_piece.length, interested_piece.offset,
header.clone(), interested_piece.length,
self.piece.clone(), header.clone(),
semaphore.clone(), self.piece.clone(),
)); self.storage.clone(),
semaphore.clone(),
)
.in_current_span(),
);
} }
// Wait for the pieces to be downloaded. // Wait for the pieces to be downloaded.
@ -1209,8 +1228,14 @@ impl Task {
length: u64, length: u64,
header: HeaderMap, header: HeaderMap,
piece: Arc<piece::Piece>, piece: Arc<piece::Piece>,
storage: Arc<Storage>,
semaphore: Arc<Semaphore>, semaphore: Arc<Semaphore>,
) -> ClientResult<metadata::Piece> { ) -> ClientResult<metadata::Piece> {
info!(
"start to download piece {} from source",
storage.piece_id(task_id.as_str(), number)
);
let _permit = semaphore.acquire().await?; let _permit = semaphore.acquire().await?;
let metadata = piece let metadata = piece
@ -1227,16 +1252,20 @@ impl Task {
Ok(metadata) Ok(metadata)
} }
join_set.spawn(download_from_source( join_set.spawn(
task_id.to_string(), download_from_source(
interested_piece.number, task_id.to_string(),
url.clone(), interested_piece.number,
interested_piece.offset, url.clone(),
interested_piece.length, interested_piece.offset,
header.clone(), interested_piece.length,
self.piece.clone(), header.clone(),
semaphore.clone(), self.piece.clone(),
)); self.storage.clone(),
semaphore.clone(),
)
.in_current_span(),
);
} }
// Wait for the pieces to be downloaded. // Wait for the pieces to be downloaded.

43
src/task/piece_collector.rs
View File

@ -26,7 +26,7 @@ use std::time::Duration;
use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::sync::mpsc::{self, Receiver, Sender};
use tokio::task::JoinSet; use tokio::task::JoinSet;
use tokio_stream::StreamExt; use tokio_stream::StreamExt;
use tracing::{error, info}; use tracing::{error, info, Instrument};
// CollectedPiece is the piece collected from a peer. // CollectedPiece is the piece collected from a peer.
pub struct CollectedPiece { pub struct CollectedPiece {
@ -101,7 +101,7 @@ impl PieceCollector {
.await .await
.unwrap_or_else(|err| { .unwrap_or_else(|err| {
error!("collect pieces failed: {}", err); error!("collect pieces failed: {}", err);
}) });
}); });
collected_piece_rx collected_piece_rx
@ -128,6 +128,8 @@ impl PieceCollector {
collected_piece_tx: Sender<CollectedPiece>, collected_piece_tx: Sender<CollectedPiece>,
collected_piece_timeout: Duration, collected_piece_timeout: Duration,
) -> Result<Peer> { ) -> Result<Peer> {
info!("sync pieces from parent {}", parent.id);
// If candidate_parent.host is None, skip it. // If candidate_parent.host is None, skip it.
let host = parent.host.clone().ok_or_else(|| { let host = parent.host.clone().ok_or_else(|| {
error!("peer {:?} host is empty", parent); error!("peer {:?} host is empty", parent);
@ -136,7 +138,15 @@ impl PieceCollector {
// Create a dfdaemon client. // Create a dfdaemon client.
let dfdaemon_upload_client = let dfdaemon_upload_client =
DfdaemonUploadClient::new(format!("http://{}:{}", host.ip, host.port)).await?; DfdaemonUploadClient::new(format!("http://{}:{}", host.ip, host.port))
.await
.map_err(|err| {
error!(
"create dfdaemon upload client from parent {} failed: {}",
parent.id, err
);
err
})?;
let response = dfdaemon_upload_client let response = dfdaemon_upload_client
.sync_pieces(SyncPiecesRequest { .sync_pieces(SyncPiecesRequest {
@ -146,7 +156,11 @@ impl PieceCollector {
.map(|piece| piece.number) .map(|piece| piece.number)
.collect(), .collect(),
}) })
.await?; .await
.map_err(|err| {
error!("sync pieces from parent {} failed: {}", parent.id, err);
err
})?;
// If the response repeating timeout exceeds the piece download timeout, the stream will return error. // If the response repeating timeout exceeds the piece download timeout, the stream will return error.
let out_stream = response.into_inner().timeout(collected_piece_timeout); let out_stream = response.into_inner().timeout(collected_piece_timeout);
@ -194,15 +208,18 @@ impl PieceCollector {
Ok(parent) Ok(parent)
} }
join_set.spawn(sync_pieces( join_set.spawn(
task_id.clone(), sync_pieces(
parent.clone(), task_id.clone(),
parents.clone(), parent.clone(),
interested_pieces.clone(), parents.clone(),
collected_pieces.clone(), interested_pieces.clone(),
collected_piece_tx.clone(), collected_pieces.clone(),
collected_piece_timeout, collected_piece_tx.clone(),
)); collected_piece_timeout,
)
.in_current_span(),
);
} }
// Wait for all tasks to finish. // Wait for all tasks to finish.