dragonflyoss
/
client
mirror of https://github.com/dragonflyoss/client.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: optimize generater of the piece id (#839) 

Signed-off-by: Gaius <gaius.qi@gmail.com>
This commit is contained in:
Gaius 2024-11-12 23:53:54 +08:00 committed by GitHub
parent da033a8219
commit 6d27c92ac6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 208 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
dragonfly-client-storage/src/lib.rs
View File

@ -275,14 +275,14 @@ impl Storage {
#[instrument(skip_all)] #[instrument(skip_all)]
pub async fn download_piece_started( pub async fn download_piece_started(
&self, &self,
task_id: &str, piece_id: &str,
number: u32, number: u32,
) -> Result<metadata::Piece> { ) -> Result<metadata::Piece> {
// Wait for the piece to be finished. // Wait for the piece to be finished.
match self.wait_for_piece_finished(task_id, number).await { match self.wait_for_piece_finished(piece_id).await {
Ok(piece) => Ok(piece), Ok(piece) => Ok(piece),
// If piece is not found or wait timeout, create piece metadata. // If piece is not found or wait timeout, create piece metadata.
Err(_) => self.metadata.download_piece_started(task_id, number), Err(_) => self.metadata.download_piece_started(piece_id, number),
} }
} }
@ -290,8 +290,8 @@ impl Storage {
#[instrument(skip_all)] #[instrument(skip_all)]
pub async fn download_piece_from_source_finished<R: AsyncRead + Unpin + ?Sized>( pub async fn download_piece_from_source_finished<R: AsyncRead + Unpin + ?Sized>(
&self, &self,
piece_id: &str,
task_id: &str, task_id: &str,
number: u32,
offset: u64, offset: u64,
length: u64, length: u64,
reader: &mut R, reader: &mut R,
@ -300,8 +300,7 @@ impl Storage {
let digest = Digest::new(Algorithm::Crc32, response.hash); let digest = Digest::new(Algorithm::Crc32, response.hash);
self.metadata.download_piece_finished( self.metadata.download_piece_finished(
task_id, piece_id,
number,
offset, offset,
length, length,
digest.to_string().as_str(), digest.to_string().as_str(),
@ -313,8 +312,8 @@ impl Storage {
#[instrument(skip_all)] #[instrument(skip_all)]
pub async fn download_piece_from_remote_peer_finished<R: AsyncRead + Unpin + ?Sized>( pub async fn download_piece_from_remote_peer_finished<R: AsyncRead + Unpin + ?Sized>(
&self, &self,
piece_id: &str,
task_id: &str, task_id: &str,
number: u32,
offset: u64, offset: u64,
expected_digest: &str, expected_digest: &str,
parent_id: &str, parent_id: &str,
@ -333,8 +332,7 @@ impl Storage {
} }
self.metadata.download_piece_finished( self.metadata.download_piece_finished(
task_id, piece_id,
number,
offset, offset,
length, length,
digest.to_string().as_str(), digest.to_string().as_str(),
@ -344,8 +342,8 @@ impl Storage {
/// download_piece_failed updates the metadata of the piece when the piece downloads failed. /// download_piece_failed updates the metadata of the piece when the piece downloads failed.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn download_piece_failed(&self, task_id: &str, number: u32) -> Result<()> { pub fn download_piece_failed(&self, piece_id: &str) -> Result<()> {
self.metadata.download_piece_failed(task_id, number) self.metadata.download_piece_failed(piece_id)
} }
/// upload_piece updates the metadata of the piece and /// upload_piece updates the metadata of the piece and
@ -353,25 +351,25 @@ impl Storage {
#[instrument(skip_all)] #[instrument(skip_all)]
pub async fn upload_piece( pub async fn upload_piece(
&self, &self,
piece_id: &str,
task_id: &str, task_id: &str,
number: u32,
range: Option<Range>, range: Option<Range>,
) -> Result<impl AsyncRead> { ) -> Result<impl AsyncRead> {
// Wait for the piece to be finished. // Wait for the piece to be finished.
self.wait_for_piece_finished(task_id, number).await?; self.wait_for_piece_finished(piece_id).await?;
// Start uploading the task. // Start uploading the task.
self.metadata.upload_task_started(task_id)?; self.metadata.upload_task_started(task_id)?;
// Start uploading the piece. // Start uploading the piece.
if let Err(err) = self.metadata.upload_piece_started(task_id, number) { if let Err(err) = self.metadata.upload_piece_started(piece_id) {
// Failed uploading the task. // Failed uploading the task.
self.metadata.upload_task_failed(task_id)?; self.metadata.upload_task_failed(task_id)?;
return Err(err); return Err(err);
} }
// Get the piece metadata and return the content of the piece. // Get the piece metadata and return the content of the piece.
match self.metadata.get_piece(task_id, number) { match self.metadata.get_piece(piece_id) {
Ok(Some(piece)) => { Ok(Some(piece)) => {
match self match self
.content .content
@ -383,7 +381,7 @@ impl Storage {
self.metadata.upload_task_finished(task_id)?; self.metadata.upload_task_finished(task_id)?;
// Finish uploading the piece. // Finish uploading the piece.
self.metadata.upload_piece_finished(task_id, number)?; self.metadata.upload_piece_finished(piece_id)?;
Ok(reader) Ok(reader)
} }
Err(err) => { Err(err) => {
@ -391,7 +389,7 @@ impl Storage {
self.metadata.upload_task_failed(task_id)?; self.metadata.upload_task_failed(task_id)?;
// Failed uploading the piece. // Failed uploading the piece.
self.metadata.upload_piece_failed(task_id, number)?; self.metadata.upload_piece_failed(piece_id)?;
Err(err) Err(err)
} }
} }
@ -401,15 +399,15 @@ impl Storage {
self.metadata.upload_task_failed(task_id)?; self.metadata.upload_task_failed(task_id)?;
// Failed uploading the piece. // Failed uploading the piece.
self.metadata.upload_piece_failed(task_id, number)?; self.metadata.upload_piece_failed(piece_id)?;
Err(Error::PieceNotFound(self.piece_id(task_id, number))) Err(Error::PieceNotFound(piece_id.to_string()))
} }
Err(err) => { Err(err) => {
// Failed uploading the task. // Failed uploading the task.
self.metadata.upload_task_failed(task_id)?; self.metadata.upload_task_failed(task_id)?;
// Failed uploading the piece. // Failed uploading the piece.
self.metadata.upload_piece_failed(task_id, number)?; self.metadata.upload_piece_failed(piece_id)?;
Err(err) Err(err)
} }
} }
@ -417,8 +415,8 @@ impl Storage {
/// get_piece returns the piece metadata. /// get_piece returns the piece metadata.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn get_piece(&self, task_id: &str, number: u32) -> Result<Option<metadata::Piece>> { pub fn get_piece(&self, piece_id: &str) -> Result<Option<metadata::Piece>> {
self.metadata.get_piece(task_id, number) self.metadata.get_piece(piece_id)
} }
/// get_pieces returns the piece metadatas. /// get_pieces returns the piece metadatas.
@ -434,7 +432,7 @@ impl Storage {
/// wait_for_piece_finished waits for the piece to be finished. /// wait_for_piece_finished waits for the piece to be finished.
#[instrument(skip_all)] #[instrument(skip_all)]
async fn wait_for_piece_finished(&self, task_id: &str, number: u32) -> Result<metadata::Piece> { async fn wait_for_piece_finished(&self, piece_id: &str) -> Result<metadata::Piece> {
// Initialize the timeout of piece. // Initialize the timeout of piece.
let piece_timeout = tokio::time::sleep(self.config.download.piece_timeout); let piece_timeout = tokio::time::sleep(self.config.download.piece_timeout);
tokio::pin!(piece_timeout); tokio::pin!(piece_timeout);
@ -446,8 +444,8 @@ impl Storage {
tokio::select! { tokio::select! {
_ = interval.tick() => { _ = interval.tick() => {
let piece = self let piece = self
.get_piece(task_id, number)? .get_piece(piece_id)?
.ok_or_else(|| Error::PieceNotFound(self.piece_id(task_id, number)))?; .ok_or_else(|| Error::PieceNotFound(piece_id.to_string()))?;
// If the piece is finished, return. // If the piece is finished, return.
if piece.is_finished() { if piece.is_finished() {
@ -461,8 +459,8 @@ impl Storage {
wait_for_piece_count += 1; wait_for_piece_count += 1;
} }
_ = &mut piece_timeout => { _ = &mut piece_timeout => {
self.metadata.wait_for_piece_finished_failed(task_id, number).unwrap_or_else(|err| error!("delete piece metadata failed: {}", err)); self.metadata.wait_for_piece_finished_failed(piece_id).unwrap_or_else(|err| error!("delete piece metadata failed: {}", err));
return Err(Error::WaitForPieceFinishedTimeout(self.piece_id(task_id, number))); return Err(Error::WaitForPieceFinishedTimeout(piece_id.to_string()));
} }
} }
} }

140
dragonfly-client-storage/src/metadata.rs
View File

@ -699,7 +699,7 @@ impl<E: StorageEngineOwned> Metadata<E> {
/// download_piece_started updates the metadata of the piece when the piece downloads started. /// download_piece_started updates the metadata of the piece when the piece downloads started.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn download_piece_started(&self, task_id: &str, number: u32) -> Result<Piece> { pub fn download_piece_started(&self, piece_id: &str, number: u32) -> Result<Piece> {
// Construct the piece metadata. // Construct the piece metadata.
let piece = Piece { let piece = Piece {
number, number,
@ -709,8 +709,7 @@ impl<E: StorageEngineOwned> Metadata<E> {
}; };
// Put the piece metadata. // Put the piece metadata.
self.db self.db.put(piece_id.as_bytes(), &piece)?;
.put(self.piece_id(task_id, number).as_bytes(), &piece)?;
Ok(piece) Ok(piece)
} }
@ -718,16 +717,13 @@ impl<E: StorageEngineOwned> Metadata<E> {
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn download_piece_finished( pub fn download_piece_finished(
&self, &self,
task_id: &str, piece_id: &str,
number: u32,
offset: u64, offset: u64,
length: u64, length: u64,
digest: &str, digest: &str,
parent_id: Option<String>, parent_id: Option<String>,
) -> Result<Piece> { ) -> Result<Piece> {
// Get the piece id. let piece = match self.db.get::<Piece>(piece_id.as_bytes())? {
let id = self.piece_id(task_id, number);
let piece = match self.db.get::<Piece>(id.as_bytes())? {
Some(mut piece) => { Some(mut piece) => {
piece.offset = offset; piece.offset = offset;
piece.length = length; piece.length = length;
@ -737,84 +733,78 @@ impl<E: StorageEngineOwned> Metadata<E> {
piece.finished_at = Some(Utc::now().naive_utc()); piece.finished_at = Some(Utc::now().naive_utc());
piece piece
} }
None => return Err(Error::PieceNotFound(id)), None => return Err(Error::PieceNotFound(piece_id.to_string())),
}; };
self.db.put(id.as_bytes(), &piece)?; self.db.put(piece_id.as_bytes(), &piece)?;
Ok(piece) Ok(piece)
} }
/// download_piece_failed updates the metadata of the piece when the piece downloads failed. /// download_piece_failed updates the metadata of the piece when the piece downloads failed.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn download_piece_failed(&self, task_id: &str, number: u32) -> Result<()> { pub fn download_piece_failed(&self, piece_id: &str) -> Result<()> {
self.delete_piece(task_id, number) self.delete_piece(piece_id)
} }
/// wait_for_piece_finished_failed waits for the piece to be finished or failed. /// wait_for_piece_finished_failed waits for the piece to be finished or failed.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn wait_for_piece_finished_failed(&self, task_id: &str, number: u32) -> Result<()> { pub fn wait_for_piece_finished_failed(&self, piece_id: &str) -> Result<()> {
self.delete_piece(task_id, number) self.delete_piece(piece_id)
} }
/// upload_piece_started updates the metadata of the piece when piece uploads started. /// upload_piece_started updates the metadata of the piece when piece uploads started.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn upload_piece_started(&self, task_id: &str, number: u32) -> Result<Piece> { pub fn upload_piece_started(&self, piece_id: &str) -> Result<Piece> {
// Get the piece id. let piece = match self.db.get::<Piece>(piece_id.as_bytes())? {
let id = self.piece_id(task_id, number);
let piece = match self.db.get::<Piece>(id.as_bytes())? {
Some(mut piece) => { Some(mut piece) => {
piece.uploading_count += 1; piece.uploading_count += 1;
piece.updated_at = Utc::now().naive_utc(); piece.updated_at = Utc::now().naive_utc();
piece piece
} }
None => return Err(Error::PieceNotFound(id)), None => return Err(Error::PieceNotFound(piece_id.to_string())),
}; };
self.db.put(id.as_bytes(), &piece)?; self.db.put(piece_id.as_bytes(), &piece)?;
Ok(piece) Ok(piece)
} }
/// upload_piece_finished updates the metadata of the piece when piece uploads finished. /// upload_piece_finished updates the metadata of the piece when piece uploads finished.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn upload_piece_finished(&self, task_id: &str, number: u32) -> Result<Piece> { pub fn upload_piece_finished(&self, piece_id: &str) -> Result<Piece> {
// Get the piece id. let piece = match self.db.get::<Piece>(piece_id.as_bytes())? {
let id = self.piece_id(task_id, number);
let piece = match self.db.get::<Piece>(id.as_bytes())? {
Some(mut piece) => { Some(mut piece) => {
piece.uploading_count -= 1; piece.uploading_count -= 1;
piece.uploaded_count += 1; piece.uploaded_count += 1;
piece.updated_at = Utc::now().naive_utc(); piece.updated_at = Utc::now().naive_utc();
piece piece
} }
None => return Err(Error::PieceNotFound(id)), None => return Err(Error::PieceNotFound(piece_id.to_string())),
}; };
self.db.put(id.as_bytes(), &piece)?; self.db.put(piece_id.as_bytes(), &piece)?;
Ok(piece) Ok(piece)
} }
/// upload_piece_failed updates the metadata of the piece when the piece uploads failed. /// upload_piece_failed updates the metadata of the piece when the piece uploads failed.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn upload_piece_failed(&self, task_id: &str, number: u32) -> Result<Piece> { pub fn upload_piece_failed(&self, piece_id: &str) -> Result<Piece> {
// Get the piece id. let piece = match self.db.get::<Piece>(piece_id.as_bytes())? {
let id = self.piece_id(task_id, number);
let piece = match self.db.get::<Piece>(id.as_bytes())? {
Some(mut piece) => { Some(mut piece) => {
piece.uploading_count -= 1; piece.uploading_count -= 1;
piece.updated_at = Utc::now().naive_utc(); piece.updated_at = Utc::now().naive_utc();
piece piece
} }
None => return Err(Error::PieceNotFound(id)), None => return Err(Error::PieceNotFound(piece_id.to_string())),
}; };
self.db.put(id.as_bytes(), &piece)?; self.db.put(piece_id.as_bytes(), &piece)?;
Ok(piece) Ok(piece)
} }
/// get_piece gets the piece metadata. /// get_piece gets the piece metadata.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn get_piece(&self, task_id: &str, number: u32) -> Result<Option<Piece>> { pub fn get_piece(&self, piece_id: &str) -> Result<Option<Piece>> {
self.db.get(self.piece_id(task_id, number).as_bytes()) self.db.get(piece_id.as_bytes())
} }
/// get_pieces gets the piece metadatas. /// get_pieces gets the piece metadatas.
@ -836,10 +826,9 @@ impl<E: StorageEngineOwned> Metadata<E> {
/// delete_piece deletes the piece metadata. /// delete_piece deletes the piece metadata.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn delete_piece(&self, task_id: &str, number: u32) -> Result<()> { pub fn delete_piece(&self, piece_id: &str) -> Result<()> {
info!("delete piece metadata {}", self.piece_id(task_id, number)); info!("delete piece metadata {}", piece_id);
self.db self.db.delete::<Piece>(piece_id.as_bytes())
.delete::<Piece>(self.piece_id(task_id, number).as_bytes())
} }
/// delete_pieces deletes the piece metadatas. /// delete_pieces deletes the piece metadatas.
@ -950,20 +939,20 @@ mod tests {
// Test upload_task_started. // Test upload_task_started.
metadata.upload_task_started(task_id).unwrap(); metadata.upload_task_started(task_id).unwrap();
let task = metadata.get_task(task_id).unwrap().unwrap(); let task = metadata.get_task(task_id).unwrap().unwrap();
assert_eq!(task.uploading_count, 1,); assert_eq!(task.uploading_count, 1);
// Test upload_task_finished. // Test upload_task_finished.
metadata.upload_task_finished(task_id).unwrap(); metadata.upload_task_finished(task_id).unwrap();
let task = metadata.get_task(task_id).unwrap().unwrap(); let task = metadata.get_task(task_id).unwrap().unwrap();
assert_eq!(task.uploading_count, 0,); assert_eq!(task.uploading_count, 0);
assert_eq!(task.uploaded_count, 1,); assert_eq!(task.uploaded_count, 1);
// Test upload_task_failed. // Test upload_task_failed.
let task = metadata.upload_task_started(task_id).unwrap(); let task = metadata.upload_task_started(task_id).unwrap();
assert_eq!(task.uploading_count, 1); assert_eq!(task.uploading_count, 1);
let task = metadata.upload_task_failed(task_id).unwrap(); let task = metadata.upload_task_failed(task_id).unwrap();
assert_eq!(task.uploading_count, 0,); assert_eq!(task.uploading_count, 0);
assert_eq!(task.uploaded_count, 1,); assert_eq!(task.uploaded_count, 1);
// Test get_tasks. // Test get_tasks.
let task_id = "a535b115f18d96870f0422ac891f91dd162f2f391e4778fb84279701fcd02dd1"; let task_id = "a535b115f18d96870f0422ac891f91dd162f2f391e4778fb84279701fcd02dd1";
@ -985,49 +974,64 @@ mod tests {
let log_dir = dir.path().join("log"); let log_dir = dir.path().join("log");
let metadata = Metadata::new(Arc::new(Config::default()), dir.path(), &log_dir).unwrap(); let metadata = Metadata::new(Arc::new(Config::default()), dir.path(), &log_dir).unwrap();
let task_id = "d3c4e940ad06c47fc36ac67801e6f8e36cb400e2391708620bc7e865b102062c"; let task_id = "d3c4e940ad06c47fc36ac67801e6f8e36cb400e2391708620bc7e865b102062c";
let piece_id = metadata.piece_id(task_id, 1);
// Test download_piece_started. // Test download_piece_started.
metadata.download_piece_started(task_id, 1).unwrap(); metadata
let piece = metadata.get_piece(task_id, 1).unwrap().unwrap(); .download_piece_started(piece_id.as_str(), 1)
assert_eq!(piece.number, 1,); .unwrap();
let piece = metadata.get_piece(piece_id.as_str()).unwrap().unwrap();
assert_eq!(piece.number, 1);
// Test download_piece_finished. // Test download_piece_finished.
metadata metadata
.download_piece_finished(task_id, 1, 0, 1024, "digest1", None) .download_piece_finished(piece_id.as_str(), 0, 1024, "digest1", None)
.unwrap(); .unwrap();
let piece = metadata.get_piece(task_id, 1).unwrap().unwrap(); let piece = metadata.get_piece(piece_id.as_str()).unwrap().unwrap();
assert_eq!(piece.length, 1024,); assert_eq!(piece.length, 1024);
assert_eq!(piece.digest, "digest1",); assert_eq!(piece.digest, "digest1");
// Test get_pieces. // Test get_pieces.
metadata.download_piece_started(task_id, 2).unwrap(); metadata
metadata.download_piece_started(task_id, 3).unwrap(); .download_piece_started(metadata.piece_id(task_id, 2).as_str(), 2)
.unwrap();
metadata
.download_piece_started(metadata.piece_id(task_id, 3).as_str(), 3)
.unwrap();
let pieces = metadata.get_pieces(task_id).unwrap(); let pieces = metadata.get_pieces(task_id).unwrap();
assert_eq!(pieces.len(), 3, "should get 3 pieces in total"); assert_eq!(pieces.len(), 3);
// Test download_piece_failed. // Test download_piece_failed.
metadata.download_piece_started(task_id, 2).unwrap(); let piece_id = metadata.piece_id(task_id, 2);
metadata.download_piece_started(task_id, 3).unwrap(); metadata
metadata.download_piece_failed(task_id, 2).unwrap(); .download_piece_started(piece_id.as_str(), 2)
let piece = metadata.get_piece(task_id, 2).unwrap(); .unwrap();
metadata
.download_piece_started(metadata.piece_id(task_id, 3).as_str(), 3)
.unwrap();
metadata.download_piece_failed(piece_id.as_str()).unwrap();
let piece = metadata.get_piece(piece_id.as_str()).unwrap();
assert!(piece.is_none()); assert!(piece.is_none());
// Test upload_piece_started. // Test upload_piece_started.
metadata.upload_piece_started(task_id, 3).unwrap(); let piece_id = metadata.piece_id(task_id, 3);
let piece = metadata.get_piece(task_id, 3).unwrap().unwrap(); metadata.upload_piece_started(piece_id.as_str()).unwrap();
assert_eq!(piece.uploading_count, 1,); let piece = metadata.get_piece(piece_id.as_str()).unwrap().unwrap();
assert_eq!(piece.uploading_count, 1);
// Test upload_piece_finished. // Test upload_piece_finished.
metadata.upload_piece_finished(task_id, 3).unwrap(); let piece_id = metadata.piece_id(task_id, 3);
let piece = metadata.get_piece(task_id, 3).unwrap().unwrap(); metadata.upload_piece_finished(piece_id.as_str()).unwrap();
assert_eq!(piece.uploading_count, 0,); let piece = metadata.get_piece(piece_id.as_str()).unwrap().unwrap();
assert_eq!(piece.uploaded_count, 1,); assert_eq!(piece.uploading_count, 0);
assert_eq!(piece.uploaded_count, 1);
// Test upload_piece_failed. // Test upload_piece_failed.
metadata.upload_piece_started(task_id, 3).unwrap(); let piece_id = metadata.piece_id(task_id, 3);
metadata.upload_piece_failed(task_id, 3).unwrap(); metadata.upload_piece_started(piece_id.as_str()).unwrap();
let piece = metadata.get_piece(task_id, 3).unwrap().unwrap(); metadata.upload_piece_failed(piece_id.as_str()).unwrap();
assert_eq!(piece.uploading_count, 0,); let piece = metadata.get_piece(piece_id.as_str()).unwrap().unwrap();
assert_eq!(piece.uploading_count, 0);
// Test delete_pieces. // Test delete_pieces.
metadata.delete_pieces(task_id).unwrap(); metadata.delete_pieces(task_id).unwrap();

23
dragonfly-client-storage/src/storage_engine/rocksdb.rs
View File

@ -54,11 +54,11 @@ impl RocksdbStorageEngine {
// DEFAULT_MAX_BACKGROUND_JOBS is the default max background jobs for rocksdb, default is 2. // DEFAULT_MAX_BACKGROUND_JOBS is the default max background jobs for rocksdb, default is 2.
const DEFAULT_MAX_BACKGROUND_JOBS: i32 = 2; const DEFAULT_MAX_BACKGROUND_JOBS: i32 = 2;
/// DEFAULT_BLOCK_SIZE is the default block size for rocksdb, default is 128KB. /// DEFAULT_BLOCK_SIZE is the default block size for rocksdb, default is 64KB.
const DEFAULT_BLOCK_SIZE: usize = 128 * 1024; const DEFAULT_BLOCK_SIZE: usize = 64 * 1024;
/// DEFAULT_CACHE_SIZE is the default cache size for rocksdb, default is 512MB. /// DEFAULT_CACHE_SIZE is the default cache size for rocksdb, default is 1GB.
const DEFAULT_CACHE_SIZE: usize = 512 * 1024 * 1024; const DEFAULT_CACHE_SIZE: usize = 1024 * 1024 * 1024;
/// DEFAULT_LOG_MAX_SIZE is the default max log size for rocksdb, default is 64MB. /// DEFAULT_LOG_MAX_SIZE is the default max log size for rocksdb, default is 64MB.
const DEFAULT_LOG_MAX_SIZE: usize = 64 * 1024 * 1024; const DEFAULT_LOG_MAX_SIZE: usize = 64 * 1024 * 1024;
@ -74,16 +74,21 @@ impl RocksdbStorageEngine {
let mut options = rocksdb::Options::default(); let mut options = rocksdb::Options::default();
options.create_if_missing(true); options.create_if_missing(true);
options.create_missing_column_families(true); options.create_missing_column_families(true);
options.increase_parallelism(num_cpus::get() as i32);
// Optimize compression.
options.set_compression_type(rocksdb::DBCompressionType::Lz4); options.set_compression_type(rocksdb::DBCompressionType::Lz4);
options.set_bottommost_compression_type(rocksdb::DBCompressionType::Zstd);
// Improved parallelism.
options.increase_parallelism(num_cpus::get() as i32);
options.set_max_background_jobs(std::cmp::max( options.set_max_background_jobs(std::cmp::max(
num_cpus::get() as i32 / 2, num_cpus::get() as i32,
Self::DEFAULT_MAX_BACKGROUND_JOBS, Self::DEFAULT_MAX_BACKGROUND_JOBS,
)); ));
// Set rocksdb log options. // Set rocksdb log options.
options.set_db_log_dir(log_dir); options.set_db_log_dir(log_dir);
options.set_log_level(rocksdb::LogLevel::Debug); options.set_log_level(rocksdb::LogLevel::Info);
options.set_max_log_file_size(Self::DEFAULT_LOG_MAX_SIZE); options.set_max_log_file_size(Self::DEFAULT_LOG_MAX_SIZE);
options.set_keep_log_file_num(Self::DEFAULT_LOG_MAX_FILES); options.set_keep_log_file_num(Self::DEFAULT_LOG_MAX_FILES);
@ -91,12 +96,14 @@ impl RocksdbStorageEngine {
let mut block_options = rocksdb::BlockBasedOptions::default(); let mut block_options = rocksdb::BlockBasedOptions::default();
block_options.set_block_cache(&rocksdb::Cache::new_lru_cache(Self::DEFAULT_CACHE_SIZE)); block_options.set_block_cache(&rocksdb::Cache::new_lru_cache(Self::DEFAULT_CACHE_SIZE));
block_options.set_block_size(Self::DEFAULT_BLOCK_SIZE); block_options.set_block_size(Self::DEFAULT_BLOCK_SIZE);
block_options.set_cache_index_and_filter_blocks(true);
block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
options.set_block_based_table_factory(&block_options); options.set_block_based_table_factory(&block_options);
// Initialize column family options. // Initialize column family options.
let mut cf_options = rocksdb::Options::default(); let mut cf_options = rocksdb::Options::default();
cf_options.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(64)); cf_options.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(64));
cf_options.set_memtable_prefix_bloom_ratio(0.2); cf_options.set_memtable_prefix_bloom_ratio(0.25);
cf_options.optimize_level_style_compaction(Self::DEFAULT_MEMTABLE_MEMORY_BUDGET); cf_options.optimize_level_style_compaction(Self::DEFAULT_MEMTABLE_MEMORY_BUDGET);
// Initialize column families. // Initialize column families.

19
dragonfly-client/src/grpc/dfdaemon_upload.rs
View File

@ -652,10 +652,12 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
let mut has_started_piece = false; let mut has_started_piece = false;
let mut finished_piece_numbers = Vec::new(); let mut finished_piece_numbers = Vec::new();
for interested_piece_number in interested_piece_numbers.iter() { for interested_piece_number in interested_piece_numbers.iter() {
let piece = match task_manager let piece = match task_manager.piece.get(
.piece task_manager
.get(task_id.as_str(), *interested_piece_number) .piece
{ .id(task_id.as_str(), *interested_piece_number)
.as_str(),
) {
Ok(Some(piece)) => piece, Ok(Some(piece)) => piece,
Ok(None) => continue, Ok(None) => continue,
Err(err) => { Err(err) => {
@ -765,17 +767,20 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
// Get the interested piece number from the request. // Get the interested piece number from the request.
let piece_number = request.piece_number; let piece_number = request.piece_number;
// Generate the piece id.
let piece_id = self.task.piece.id(task_id.as_str(), piece_number);
// Span record the host id, task id and piece number. // Span record the host id, task id and piece number.
Span::current().record("host_id", host_id.as_str()); Span::current().record("host_id", host_id.as_str());
Span::current().record("remote_host_id", remote_host_id.as_str()); Span::current().record("remote_host_id", remote_host_id.as_str());
Span::current().record("task_id", task_id.as_str()); Span::current().record("task_id", task_id.as_str());
Span::current().record("piece_id", format!("{}-{}", task_id, piece_number).as_str()); Span::current().record("piece_id", piece_id.as_str());
// Get the piece metadata from the local storage. // Get the piece metadata from the local storage.
let piece = self let piece = self
.task .task
.piece .piece
.get(task_id.as_str(), piece_number) .get(piece_id.as_str())
.map_err(|err| { .map_err(|err| {
error!("upload piece metadata from local storage: {}", err); error!("upload piece metadata from local storage: {}", err);
Status::internal(err.to_string()) Status::internal(err.to_string())
@ -794,8 +799,8 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
.task .task
.piece .piece
.upload_from_local_peer_into_async_read( .upload_from_local_peer_into_async_read(
piece_id.as_str(),
task_id.as_str(), task_id.as_str(),
piece_number,
piece.length, piece.length,
None, None,
false, false,

4
dragonfly-client/src/proxy/mod.rs
View File

@ -740,8 +740,10 @@ async fn proxy_by_dfdaemon(
let piece_reader = match task let piece_reader = match task
.piece .piece
.download_from_local_peer_into_async_read( .download_from_local_peer_into_async_read(
task.piece
.id(message.task_id.as_str(), piece.number)
.as_str(),
message.task_id.as_str(), message.task_id.as_str(),
piece.number,
piece.length, piece.length,
download_task_started_response.range, download_task_started_response.range,
true, true,

33
dragonfly-client/src/resource/persistent_cache_task.rs
View File

@ -841,14 +841,16 @@ impl PersistentCacheTask {
// Limit the concurrent download count. // Limit the concurrent download count.
let _permit = semaphore.acquire().await.unwrap(); let _permit = semaphore.acquire().await.unwrap();
let piece_id = storage.piece_id(task_id.as_str(), number);
info!( info!(
"start to download piece {} from remote peer {:?}", "start to download piece {} from remote peer {:?}",
storage.piece_id(task_id.as_str(), number), piece_id,
parent.id.clone() parent.id.clone()
); );
let metadata = piece_manager let metadata = piece_manager
.download_from_remote_peer( .download_from_remote_peer(
peer_id.as_str(),
host_id.as_str(), host_id.as_str(),
task_id.as_str(), task_id.as_str(),
number, number,
@ -859,7 +861,7 @@ impl PersistentCacheTask {
.map_err(|err| { .map_err(|err| {
error!( error!(
"download piece {} from remote peer {:?} error: {:?}", "download piece {} from remote peer {:?} error: {:?}",
storage.piece_id(task_id.as_str(), number), piece_id,
parent.id.clone(), parent.id.clone(),
err err
); );
@ -930,8 +932,7 @@ impl PersistentCacheTask {
info!( info!(
"finished piece {} from remote peer {:?}", "finished piece {} from remote peer {:?}",
storage.piece_id(task_id.as_str(), metadata.number), piece_id, metadata.parent_id
metadata.parent_id
); );
Ok(metadata) Ok(metadata)
@ -1029,24 +1030,19 @@ impl PersistentCacheTask {
// Download the piece from the local peer. // Download the piece from the local peer.
for interested_piece in interested_pieces { for interested_piece in interested_pieces {
let piece_id = self
.storage
.piece_id(task.id.as_str(), interested_piece.number);
// Get the piece metadata from the local storage. // Get the piece metadata from the local storage.
let piece = match self.piece.get(task.id.as_str(), interested_piece.number) { let piece = match self.piece.get(piece_id.as_str()) {
Ok(Some(piece)) => piece, Ok(Some(piece)) => piece,
Ok(None) => { Ok(None) => {
info!( info!("piece {} not found in local storage", piece_id);
"piece {} not found in local storage",
self.storage
.piece_id(task.id.as_str(), interested_piece.number)
);
continue; continue;
} }
Err(err) => { Err(err) => {
error!( error!("get piece {} from local storage error: {:?}", piece_id, err);
"get piece {} from local storage error: {:?}",
self.storage
.piece_id(task.id.as_str(), interested_piece.number),
err
);
continue; continue;
} }
}; };
@ -1054,10 +1050,7 @@ impl PersistentCacheTask {
// Fake the download from the local peer. // Fake the download from the local peer.
self.piece self.piece
.download_from_local_peer(task.id.as_str(), piece.length); .download_from_local_peer(task.id.as_str(), piece.length);
info!( info!("finished piece {} from local peer", piece_id);
"finished piece {} from local peer",
self.storage.piece_id(task.id.as_str(), piece.number)
);
// Construct the piece. // Construct the piece.
let piece = Piece { let piece = Piece {

68
dragonfly-client/src/resource/piece.rs
View File

@ -110,10 +110,16 @@ impl Piece {
} }
} }
/// id generates a new piece id.
#[instrument(skip_all)]
pub fn id(&self, task_id: &str, number: u32) -> String {
self.storage.piece_id(task_id, number)
}
/// get gets a piece from the local storage. /// get gets a piece from the local storage.
#[instrument(skip_all)] #[instrument(skip_all)]
pub fn get(&self, task_id: &str, number: u32) -> Result<Option<metadata::Piece>> { pub fn get(&self, piece_id: &str) -> Result<Option<metadata::Piece>> {
self.storage.get_piece(task_id, number) self.storage.get_piece(piece_id)
} }
/// calculate_interested calculates the interested pieces by content_length and range. /// calculate_interested calculates the interested pieces by content_length and range.
@ -302,14 +308,14 @@ impl Piece {
#[instrument(skip_all, fields(piece_id))] #[instrument(skip_all, fields(piece_id))]
pub async fn upload_from_local_peer_into_async_read( pub async fn upload_from_local_peer_into_async_read(
&self, &self,
piece_id: &str,
task_id: &str, task_id: &str,
number: u32,
length: u64, length: u64,
range: Option<Range>, range: Option<Range>,
disable_rate_limit: bool, disable_rate_limit: bool,
) -> Result<impl AsyncRead> { ) -> Result<impl AsyncRead> {
// Span record the piece_id. // Span record the piece_id.
Span::current().record("piece_id", self.storage.piece_id(task_id, number)); Span::current().record("piece_id", piece_id);
// Acquire the upload rate limiter. // Acquire the upload rate limiter.
if !disable_rate_limit { if !disable_rate_limit {
@ -318,7 +324,7 @@ impl Piece {
// Upload the piece content. // Upload the piece content.
self.storage self.storage
.upload_piece(task_id, number, range) .upload_piece(piece_id, task_id, range)
.await .await
.map(|reader| { .map(|reader| {
collect_upload_piece_traffic_metrics( collect_upload_piece_traffic_metrics(
@ -333,14 +339,14 @@ impl Piece {
#[instrument(skip_all, fields(piece_id))] #[instrument(skip_all, fields(piece_id))]
pub async fn download_from_local_peer_into_async_read( pub async fn download_from_local_peer_into_async_read(
&self, &self,
piece_id: &str,
task_id: &str, task_id: &str,
number: u32,
length: u64, length: u64,
range: Option<Range>, range: Option<Range>,
disable_rate_limit: bool, disable_rate_limit: bool,
) -> Result<impl AsyncRead> { ) -> Result<impl AsyncRead> {
// Span record the piece_id. // Span record the piece_id.
Span::current().record("piece_id", self.storage.piece_id(task_id, number)); Span::current().record("piece_id", piece_id);
// Acquire the download rate limiter. // Acquire the download rate limiter.
if !disable_rate_limit { if !disable_rate_limit {
@ -348,7 +354,7 @@ impl Piece {
} }
// Upload the piece content. // Upload the piece content.
self.storage.upload_piece(task_id, number, range).await self.storage.upload_piece(piece_id, task_id, range).await
} }
/// download_from_local_peer downloads a single piece from a local peer. Fake the download piece /// download_from_local_peer downloads a single piece from a local peer. Fake the download piece
@ -366,6 +372,7 @@ impl Piece {
#[instrument(skip_all, fields(piece_id))] #[instrument(skip_all, fields(piece_id))]
pub async fn download_from_remote_peer( pub async fn download_from_remote_peer(
&self, &self,
piece_id: &str,
host_id: &str, host_id: &str,
task_id: &str, task_id: &str,
number: u32, number: u32,
@ -373,13 +380,16 @@ impl Piece {
parent: piece_collector::CollectedParent, parent: piece_collector::CollectedParent,
) -> Result<metadata::Piece> { ) -> Result<metadata::Piece> {
// Span record the piece_id. // Span record the piece_id.
Span::current().record("piece_id", self.storage.piece_id(task_id, number)); Span::current().record("piece_id", piece_id);
// Acquire the download rate limiter. // Acquire the download rate limiter.
self.download_rate_limiter.acquire(length as usize).await; self.download_rate_limiter.acquire(length as usize).await;
// Record the start of downloading piece. // Record the start of downloading piece.
let piece = self.storage.download_piece_started(task_id, number).await?; let piece = self
.storage
.download_piece_started(piece_id, number)
.await?;
// If the piece is downloaded by the other thread, // If the piece is downloaded by the other thread,
// return the piece directly. // return the piece directly.
@ -390,7 +400,7 @@ impl Piece {
// Create a dfdaemon client. // Create a dfdaemon client.
let host = parent.host.clone().ok_or_else(|| { let host = parent.host.clone().ok_or_else(|| {
error!("peer host is empty"); error!("peer host is empty");
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -406,7 +416,7 @@ impl Piece {
"create dfdaemon upload client from {}:{} failed: {}", "create dfdaemon upload client from {}:{} failed: {}",
host.ip, host.port, err host.ip, host.port, err
); );
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -426,7 +436,7 @@ impl Piece {
.await .await
.map_err(|err| { .map_err(|err| {
error!("download piece failed: {}", err); error!("download piece failed: {}", err);
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -435,7 +445,7 @@ impl Piece {
let piece = response.piece.ok_or_else(|| { let piece = response.piece.ok_or_else(|| {
error!("piece is empty"); error!("piece is empty");
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -445,7 +455,7 @@ impl Piece {
// Get the piece content. // Get the piece content.
let content = piece.content.ok_or_else(|| { let content = piece.content.ok_or_else(|| {
error!("piece content is empty"); error!("piece content is empty");
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -455,8 +465,8 @@ impl Piece {
// Record the finish of downloading piece. // Record the finish of downloading piece.
self.storage self.storage
.download_piece_from_remote_peer_finished( .download_piece_from_remote_peer_finished(
piece_id,
task_id, task_id,
number,
piece.offset, piece.offset,
piece.digest.as_str(), piece.digest.as_str(),
parent.id.as_str(), parent.id.as_str(),
@ -467,7 +477,7 @@ impl Piece {
// Record the failure of downloading piece, // Record the failure of downloading piece,
// If storage fails to record piece. // If storage fails to record piece.
error!("download piece finished: {}", err); error!("download piece finished: {}", err);
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -475,7 +485,7 @@ impl Piece {
})?; })?;
self.storage self.storage
.get_piece(task_id, number)? .get_piece(piece_id)?
.ok_or_else(|| { .ok_or_else(|| {
error!("piece not found"); error!("piece not found");
Error::PieceNotFound(number.to_string()) Error::PieceNotFound(number.to_string())
@ -495,6 +505,7 @@ impl Piece {
#[instrument(skip_all, fields(piece_id))] #[instrument(skip_all, fields(piece_id))]
pub async fn download_from_source( pub async fn download_from_source(
&self, &self,
piece_id: &str,
task_id: &str, task_id: &str,
number: u32, number: u32,
url: &str, url: &str,
@ -504,13 +515,16 @@ impl Piece {
object_storage: Option<ObjectStorage>, object_storage: Option<ObjectStorage>,
) -> Result<metadata::Piece> { ) -> Result<metadata::Piece> {
// Span record the piece_id. // Span record the piece_id.
Span::current().record("piece_id", self.storage.piece_id(task_id, number)); Span::current().record("piece_id", piece_id);
// Acquire the download rate limiter. // Acquire the download rate limiter.
self.download_rate_limiter.acquire(length as usize).await; self.download_rate_limiter.acquire(length as usize).await;
// Record the start of downloading piece. // Record the start of downloading piece.
let piece = self.storage.download_piece_started(task_id, number).await?; let piece = self
.storage
.download_piece_started(piece_id, number)
.await?;
// If the piece is downloaded by the other thread, // If the piece is downloaded by the other thread,
// return the piece directly. // return the piece directly.
@ -530,7 +544,7 @@ impl Piece {
// Download the piece from the source. // Download the piece from the source.
let backend = self.backend_factory.build(url).map_err(|err| { let backend = self.backend_factory.build(url).map_err(|err| {
error!("build backend failed: {}", err); error!("build backend failed: {}", err);
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -548,7 +562,7 @@ impl Piece {
let mut response = backend let mut response = backend
.get(GetRequest { .get(GetRequest {
task_id: task_id.to_string(), task_id: task_id.to_string(),
piece_id: self.storage.piece_id(task_id, number), piece_id: piece_id.to_string(),
url: url.to_string(), url: url.to_string(),
range: Some(Range { range: Some(Range {
start: offset, start: offset,
@ -569,7 +583,7 @@ impl Piece {
// if the request is failed. // if the request is failed.
error!("backend get failed: {}", err); error!("backend get failed: {}", err);
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -594,7 +608,7 @@ impl Piece {
let error_message = response.error_message.unwrap_or_default(); let error_message = response.error_message.unwrap_or_default();
error!("backend get failed: {} {}", error_message, buffer.as_str()); error!("backend get failed: {} {}", error_message, buffer.as_str());
self.storage.download_piece_failed(task_id, number)?; self.storage.download_piece_failed(piece_id)?;
return Err(Error::BackendError(BackendError { return Err(Error::BackendError(BackendError {
message: error_message, message: error_message,
status_code: Some(response.http_status_code.unwrap_or_default()), status_code: Some(response.http_status_code.unwrap_or_default()),
@ -612,8 +626,8 @@ impl Piece {
// Record the finish of downloading piece. // Record the finish of downloading piece.
self.storage self.storage
.download_piece_from_source_finished( .download_piece_from_source_finished(
piece_id,
task_id, task_id,
number,
offset, offset,
length, length,
&mut response.reader, &mut response.reader,
@ -623,7 +637,7 @@ impl Piece {
// Record the failure of downloading piece, // Record the failure of downloading piece,
// If storage fails to record piece. // If storage fails to record piece.
error!("download piece finished: {}", err); error!("download piece finished: {}", err);
if let Some(err) = self.storage.download_piece_failed(task_id, number).err() { if let Some(err) = self.storage.download_piece_failed(piece_id).err() {
error!("set piece metadata failed: {}", err) error!("set piece metadata failed: {}", err)
}; };
@ -631,7 +645,7 @@ impl Piece {
})?; })?;
self.storage self.storage
.get_piece(task_id, number)? .get_piece(piece_id)?
.ok_or_else(|| { .ok_or_else(|| {
error!("piece not found"); error!("piece not found");
Error::PieceNotFound(number.to_string()) Error::PieceNotFound(number.to_string())

75
dragonfly-client/src/resource/task.rs
View File

@ -964,14 +964,16 @@ impl Task {
// Limit the concurrent piece count. // Limit the concurrent piece count.
let _permit = semaphore.acquire().await.unwrap(); let _permit = semaphore.acquire().await.unwrap();
let piece_id = storage.piece_id(task_id.as_str(), number);
info!( info!(
"start to download piece {} from remote peer {:?}", "start to download piece {} from remote peer {:?}",
storage.piece_id(task_id.as_str(), number), piece_id,
parent.id.clone() parent.id.clone()
); );
let metadata = piece_manager let metadata = piece_manager
.download_from_remote_peer( .download_from_remote_peer(
piece_id.as_str(),
host_id.as_str(), host_id.as_str(),
task_id.as_str(), task_id.as_str(),
number, number,
@ -982,7 +984,7 @@ impl Task {
.map_err(|err| { .map_err(|err| {
error!( error!(
"download piece {} from remote peer {:?} error: {:?}", "download piece {} from remote peer {:?} error: {:?}",
storage.piece_id(task_id.as_str(), number), piece_id,
parent.id.clone(), parent.id.clone(),
err err
); );
@ -1026,8 +1028,7 @@ impl Task {
.map_err(|err| { .map_err(|err| {
error!( error!(
"send DownloadPieceFinishedRequest for piece {} failed: {:?}", "send DownloadPieceFinishedRequest for piece {} failed: {:?}",
storage.piece_id(task_id.as_str(), number), piece_id, err
err
); );
interrupt.store(true, Ordering::SeqCst); interrupt.store(true, Ordering::SeqCst);
err err
@ -1054,8 +1055,7 @@ impl Task {
.map_err(|err| { .map_err(|err| {
error!( error!(
"send DownloadPieceFinishedResponse for piece {} failed: {:?}", "send DownloadPieceFinishedResponse for piece {} failed: {:?}",
storage.piece_id(task_id.as_str(), number), piece_id, err
err
); );
interrupt.store(true, Ordering::SeqCst); interrupt.store(true, Ordering::SeqCst);
err err
@ -1063,8 +1063,7 @@ impl Task {
info!( info!(
"finished piece {} from remote peer {:?}", "finished piece {} from remote peer {:?}",
storage.piece_id(task_id.as_str(), metadata.number), piece_id, metadata.parent_id
metadata.parent_id
); );
let mut finished_pieces = finished_pieces.lock().unwrap(); let mut finished_pieces = finished_pieces.lock().unwrap();
@ -1206,13 +1205,12 @@ impl Task {
// Limit the concurrent download count. // Limit the concurrent download count.
let _permit = semaphore.acquire().await.unwrap(); let _permit = semaphore.acquire().await.unwrap();
info!( let piece_id = storage.piece_id(task_id.as_str(), number);
"start to download piece {} from source", info!("start to download piece {} from source", piece_id);
storage.piece_id(task_id.as_str(), number)
);
let metadata = piece_manager let metadata = piece_manager
.download_from_source( .download_from_source(
piece_id.as_str(),
task_id.as_str(), task_id.as_str(),
number, number,
url.as_str(), url.as_str(),
@ -1254,7 +1252,7 @@ impl Task {
REQUEST_TIMEOUT, REQUEST_TIMEOUT,
) )
.await.map_err(|err| { .await.map_err(|err| {
error!("send DownloadPieceBackToSourceFinishedRequest for piece {} failed: {:?}", storage.piece_id(task_id.as_str(), number), err); error!("send DownloadPieceBackToSourceFinishedRequest for piece {} failed: {:?}", piece_id, err);
err err
})?; })?;
@ -1279,17 +1277,12 @@ impl Task {
.map_err(|err| { .map_err(|err| {
error!( error!(
"send DownloadPieceFinishedResponse for piece {} failed: {:?}", "send DownloadPieceFinishedResponse for piece {} failed: {:?}",
storage.piece_id(task_id.as_str(), number), piece_id, err
err
); );
err err
})?; })?;
info!( info!("finished piece {} from source", piece_id);
"finished piece {} from source",
storage.piece_id(task_id.as_str(), piece.number)
);
Ok(metadata) Ok(metadata)
} }
@ -1423,24 +1416,19 @@ impl Task {
// Download the piece from the local peer. // Download the piece from the local peer.
for interested_piece in interested_pieces { for interested_piece in interested_pieces {
let piece_id = self
.storage
.piece_id(task.id.as_str(), interested_piece.number);
// Get the piece metadata from the local storage. // Get the piece metadata from the local storage.
let piece = match self.piece.get(task.id.as_str(), interested_piece.number) { let piece = match self.piece.get(piece_id.as_str()) {
Ok(Some(piece)) => piece, Ok(Some(piece)) => piece,
Ok(None) => { Ok(None) => {
info!( info!("piece {} not found in local storage", piece_id);
"piece {} not found in local storage",
self.storage
.piece_id(task.id.as_str(), interested_piece.number)
);
continue; continue;
} }
Err(err) => { Err(err) => {
error!( error!("get piece {} from local storage error: {:?}", piece_id, err);
"get piece {} from local storage error: {:?}",
self.storage
.piece_id(task.id.as_str(), interested_piece.number),
err
);
continue; continue;
} }
}; };
@ -1448,10 +1436,7 @@ impl Task {
// Fake the download from the local peer. // Fake the download from the local peer.
self.piece self.piece
.download_from_local_peer(task.id.as_str(), piece.length); .download_from_local_peer(task.id.as_str(), piece.length);
info!( info!("finished piece {} from local peer", piece_id,);
"finished piece {} from local peer",
self.storage.piece_id(task.id.as_str(), piece.number)
);
// Construct the piece. // Construct the piece.
let piece = Piece { let piece = Piece {
@ -1487,8 +1472,7 @@ impl Task {
.map_err(|err| { .map_err(|err| {
error!( error!(
"send DownloadPieceFinishedResponse for piece {} failed: {:?}", "send DownloadPieceFinishedResponse for piece {} failed: {:?}",
self.storage.piece_id(task.id.as_str(), piece.number), piece_id, err
err
); );
err err
})?; })?;
@ -1545,13 +1529,12 @@ impl Task {
// Limit the concurrent download count. // Limit the concurrent download count.
let _permit = semaphore.acquire().await.unwrap(); let _permit = semaphore.acquire().await.unwrap();
info!( let piece_id = storage.piece_id(task_id.as_str(), number);
"start to download piece {} from source", info!("start to download piece {} from source", piece_id);
storage.piece_id(task_id.as_str(), number)
);
let metadata = piece_manager let metadata = piece_manager
.download_from_source( .download_from_source(
piece_id.as_str(),
task_id.as_str(), task_id.as_str(),
number, number,
url.as_str(), url.as_str(),
@ -1596,16 +1579,12 @@ impl Task {
.map_err(|err| { .map_err(|err| {
error!( error!(
"send DownloadPieceFinishedResponse for piece {} failed: {:?}", "send DownloadPieceFinishedResponse for piece {} failed: {:?}",
storage.piece_id(task_id.as_str(), metadata.number), piece_id, err
err
); );
err err
})?; })?;
info!( info!("finished piece {} from source", piece_id);
"finished piece {} from source",
storage.piece_id(task_id.as_str(), metadata.number)
);
Ok(metadata) Ok(metadata)
} }