containerd
/
rust-extensions
mirror of https://github.com/containerd/rust-extensions.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #61 from abel-von/add-update-stats 

[async] pass all integration tests
This commit is contained in:
Maksym Pavlenko 2022-03-22 13:52:22 -07:00 committed by GitHub
parent b3ac82d9cc 72bdc9b9d4
commit d27767af57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 365 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.github/workflows/ci.yml vendored
View File

@ -63,11 +63,6 @@ jobs:
- name: Checkout extensions
uses: actions/checkout@v2
- name: Install shim
run: |
cargo build --release --bin containerd-shim-runc-v2-rs
sudo install -D ./target/release/containerd-shim-runc-v2-rs /usr/local/bin/
- uses: actions/setup-go@v2
with:
go-version: '1.17.5'
@ -101,9 +96,26 @@ jobs:
sudo -E PATH=$PATH install bin/containerd /usr/local/bin/
working-directory: src/github.com/containerd/containerd
- name: Install shim
run: |
cargo build --release --bin containerd-shim-runc-v2-rs
sudo install -D ./target/release/containerd-shim-runc-v2-rs /usr/local/bin/
- name: Integration
env:
GOPROXY: direct
TEST_RUNTIME: "io.containerd.runc.v2-rs"
run: sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration TESTFLAGS_RACE=-race EXTRA_TESTFLAGS=-no-criu
working-directory: src/github.com/containerd/containerd
- name: Install async shim
run: |
cargo build --release --all-features --bin containerd-shim-runc-v2-rs
sudo install -D ./target/release/containerd-shim-runc-v2-rs /usr/local/bin/
- name: Integration for async shim
env:
GOPROXY: direct
TEST_RUNTIME: "io.containerd.runc.v2-rs"
run: sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration TESTFLAGS_RACE=-race EXTRA_TESTFLAGS=-no-criu
working-directory: src/github.com/containerd/containerd

5
crates/runc-shim/Cargo.toml
View File

@ -36,7 +36,4 @@ tokio = { version = "1.17.0", features = ["full"], optional = true }
futures = { version = "0.3.21", optional = true }
containerd-shim = { path = "../shim", version = "0.3.0" }
runc = { path = "../runc", version = "0.2.0" }
[target.'cfg(target_os = "linux")'.dependencies]
cgroups-rs = "0.2.9"
runc = { path = "../runc", version = "0.2.0" }

17
crates/runc-shim/src/asynchronous/mod.rs
View File

@ -39,7 +39,7 @@ use containerd_shim::util::{read_options, read_runtime, read_spec, write_str_to_
use containerd_shim::{io_error, Config, Context, DeleteResponse, Error, StartOpts};
use crate::asynchronous::runc::{RuncContainer, RuncFactory};
use crate::common::create_runc;
use crate::common::{create_runc, has_shared_pid_namespace};
use crate::common::{ShimExecutor, GROUP_LABELS};
mod runc;
@ -142,7 +142,7 @@ async fn process_exits(
// pid belongs to container init process
if cont.init.pid == pid {
// kill all children process if the container has a private PID namespace
if cont.init.lifecycle.should_kill_all_on_exit(&bundle).await {
if should_kill_all_on_exit(&bundle).await {
cont.kill(None, 9, true).await.unwrap_or_else(|e| {
error!("failed to kill init's children: {}", e)
});
@ -203,3 +203,16 @@ async fn forward(
}
});
}
async fn should_kill_all_on_exit(bundle_path: &str) -> bool {
match read_spec(bundle_path).await {
Ok(spec) => has_shared_pid_namespace(&spec),
Err(e) => {
error!(
"failed to read spec when call should_kill_all_on_exit: {}",
e
);
false
}
}
}

192
crates/runc-shim/src/asynchronous/runc.rs
View File

@ -25,7 +25,7 @@ use async_trait::async_trait;
use log::{debug, error};
use nix::sys::signal::kill;
use nix::unistd::Pid;
use oci_spec::runtime::{LinuxNamespaceType, Process};
use oci_spec::runtime::{LinuxResources, Process};
use tokio::fs::{File, OpenOptions};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
@ -40,13 +40,13 @@ use containerd_shim::asynchronous::monitor::{
use containerd_shim::asynchronous::processes::{ProcessLifecycle, ProcessTemplate};
use containerd_shim::io::Stdio;
use containerd_shim::monitor::{ExitEvent, Subject, Topic};
use containerd_shim::protos::api::ProcessInfo;
use containerd_shim::protos::cgroups::metrics::Metrics;
use containerd_shim::protos::protobuf::{CodedInputStream, Message};
use containerd_shim::util::{
asyncify, mkdir, mount_rootfs, read_file_to_str, read_spec, write_options, write_runtime,
asyncify, mkdir, mount_rootfs, read_file_to_str, write_options, write_runtime,
};
use containerd_shim::Console;
use containerd_shim::{io_error, other, Error};
use containerd_shim::{other_error, Result};
use containerd_shim::{io_error, other, other_error, Console, Error, ExitSignal, Result};
use runc::{Command, Runc, Spawner};
use crate::common::receive_socket;
@ -116,7 +116,7 @@ impl ContainerFactory<RuncContainer> for RuncFactory {
let mut init = InitProcess::new(
id,
stdio,
RuncInitLifecycle::new(runc.clone(), opts.clone(), rootfs, bundle),
RuncInitLifecycle::new(runc.clone(), opts.clone(), bundle),
);
let config = CreateConfig::default();
@ -149,7 +149,7 @@ impl RuncFactory {
let bundle = &init.lifecycle.bundle;
let pid_path = Path::new(bundle).join(INIT_PID_FILE);
let mut create_opts = runc::options::CreateOpts::new()
.pid_file(pid_path.to_owned())
.pid_file(&pid_path)
.no_pivot(opts.no_pivot_root)
.no_new_keyring(opts.no_new_keyring)
.detach(false);
@ -174,7 +174,7 @@ impl RuncFactory {
}
return Err(other!("failed to create runc container: {}", e));
}
copy_io_or_console(init, socket, pio).await?;
copy_io_or_console(init, socket, pio, init.lifecycle.exit_signal.clone()).await?;
let pid = read_file_to_str(pid_path).await?.parse::<i32>()?;
init.pid = pid;
Ok(())
@ -206,25 +206,24 @@ impl ProcessFactory<ExecProcess> for RuncExecFactory {
exited_at: None,
wait_chan_tx: vec![],
console: None,
lifecycle: RuncExecLifecycle {
lifecycle: Arc::from(RuncExecLifecycle {
runtime: self.runtime.clone(),
bundle: self.bundle.to_string(),
container_id: req.id.to_string(),
io_uid: self.io_uid,
io_gid: self.io_gid,
spec: p,
},
exit_signal: Default::default(),
}),
})
}
}
#[derive(Clone)]
pub struct RuncInitLifecycle {
runtime: Runc,
opts: Options,
bundle: String,
rootfs: PathBuf,
work_dir: PathBuf,
exit_signal: Arc<ExitSignal>,
}
#[async_trait]
@ -268,12 +267,61 @@ impl ProcessLifecycle<InitProcess> for RuncInitLifecycle {
Ok(())
}
})
.map_err(other_error!(e, "failed delete"))
.map_err(other_error!(e, "failed delete"))?;
self.exit_signal.signal();
Ok(())
}
#[cfg(target_os = "linux")]
async fn update(&self, p: &mut InitProcess, resources: &LinuxResources) -> Result<()> {
if p.pid <= 0 {
return Err(other!(
"failed to update resources because init process is {}",
p.pid
));
}
containerd_shim::cgroup::update_resources(p.pid as u32, resources)
}
#[cfg(not(target_os = "linux"))]
async fn update(&self, _p: &mut InitProcess, _resources: &LinuxResources) -> Result<()> {
Err(Error::Unimplemented("update resource".to_string()))
}
#[cfg(target_os = "linux")]
async fn stats(&self, p: &InitProcess) -> Result<Metrics> {
if p.pid <= 0 {
return Err(other!(
"failed to collect metrics because init process is {}",
p.pid
));
}
containerd_shim::cgroup::collect_metrics(p.pid as u32)
}
#[cfg(not(target_os = "linux"))]
async fn stats(&self, _p: &InitProcess) -> Result<Metrics> {
Err(Error::Unimplemented("process stats".to_string()))
}
async fn ps(&self, p: &InitProcess) -> Result<Vec<ProcessInfo>> {
let pids = self
.runtime
.ps(&*p.id)
.await
.map_err(other_error!(e, "failed to execute runc ps"))?;
Ok(pids
.iter()
.map(|&x| ProcessInfo {
pid: x as u32,
..Default::default()
})
.collect())
}
}
impl RuncInitLifecycle {
pub fn new(runtime: Runc, opts: Options, rootfs: PathBuf, bundle: &str) -> Self {
pub fn new(runtime: Runc, opts: Options, bundle: &str) -> Self {
let work_dir = Path::new(bundle).join("work");
let mut opts = opts;
if opts.get_criu_path().is_empty() {
@ -283,39 +331,11 @@ impl RuncInitLifecycle {
runtime,
opts,
bundle: bundle.to_string(),
rootfs,
work_dir,
}
}
pub(crate) async fn should_kill_all_on_exit(&mut self, bundle_path: &str) -> bool {
match read_spec(bundle_path).await {
Ok(spec) => match spec.linux() {
None => true,
Some(linux) => match linux.namespaces() {
None => true,
Some(namespaces) => {
for ns in namespaces {
if ns.typ() == LinuxNamespaceType::Pid && ns.path().is_none() {
return false;
}
}
true
}
},
},
Err(e) => {
error!(
"failed to read spec when call should_kill_all_on_exit: {}",
e
);
false
}
exit_signal: Default::default(),
}
}
}
#[derive(Clone)]
pub struct RuncExecLifecycle {
runtime: Runc,
bundle: String,
@ -323,6 +343,7 @@ pub struct RuncExecLifecycle {
io_uid: u32,
io_gid: u32,
spec: Process,
exit_signal: Arc<ExitSignal>,
}
#[async_trait]
@ -355,7 +376,7 @@ impl ProcessLifecycle<ExecProcess> for RuncExecLifecycle {
}
return Err(other!("failed to start runc exec: {}", e));
}
copy_io_or_console(p, socket, pio).await?;
copy_io_or_console(p, socket, pio, p.lifecycle.exit_signal.clone()).await?;
let pid = read_file_to_str(pid_path).await?.parse::<i32>()?;
p.pid = pid;
p.state = Status::RUNNING;
@ -385,11 +406,28 @@ impl ProcessLifecycle<ExecProcess> for RuncExecLifecycle {
}
async fn delete(&self, _p: &mut ExecProcess) -> containerd_shim::Result<()> {
self.exit_signal.signal();
Ok(())
}
async fn update(&self, _p: &mut ExecProcess, _resources: &LinuxResources) -> Result<()> {
Err(Error::Unimplemented("exec update".to_string()))
}
async fn stats(&self, _p: &ExecProcess) -> Result<Metrics> {
Err(Error::Unimplemented("exec stats".to_string()))
}
async fn ps(&self, _p: &ExecProcess) -> Result<Vec<ProcessInfo>> {
Err(Error::Unimplemented("exec ps".to_string()))
}
}
async fn copy_console(console_socket: &ConsoleSocket, stdio: &Stdio) -> Result<Console> {
async fn copy_console(
console_socket: &ConsoleSocket,
stdio: &Stdio,
exit_signal: Arc<ExitSignal>,
) -> Result<Console> {
debug!("copy_console: waiting for runtime to send console fd");
let stream = console_socket.accept().await?;
let fd = asyncify(move || -> Result<RawFd> { receive_socket(stream.as_raw_fd()) }).await?;
@ -400,13 +438,28 @@ async fn copy_console(console_socket: &ConsoleSocket, stdio: &Stdio) -> Result<C
.try_clone()
.await
.map_err(io_error!(e, "failed to clone console file"))?;
let stdin = OpenOptions::new()
.read(true)
.write(true)
.open(stdio.stdin.as_str())
.await
.map_err(io_error!(e, "open stdin"))?;
spawn_copy(stdin, console_stdin, None::<fn()>);
let stdin_fut = async {
OpenOptions::new()
.read(true)
.open(stdio.stdin.as_str())
.await
};
let stdin_w_fut = async {
OpenOptions::new()
.write(true)
.open(stdio.stdin.as_str())
.await
};
let (stdin, stdin_w) =
tokio::try_join!(stdin_fut, stdin_w_fut).map_err(io_error!(e, "open stdin"))?;
spawn_copy(
stdin,
console_stdin,
exit_signal.clone(),
Some(move || {
drop(stdin_w);
}),
);
}
if !stdio.stdout.is_empty() {
@ -430,6 +483,7 @@ async fn copy_console(console_socket: &ConsoleSocket, stdio: &Stdio) -> Result<C
spawn_copy(
console_stdout,
stdout,
exit_signal,
Some(move || {
drop(stdout_r);
}),
@ -441,7 +495,7 @@ async fn copy_console(console_socket: &ConsoleSocket, stdio: &Stdio) -> Result<C
Ok(console)
}
pub async fn copy_io(pio: &ProcessIO, stdio: &Stdio) -> Result<()> {
pub async fn copy_io(pio: &ProcessIO, stdio: &Stdio, exit_signal: Arc<ExitSignal>) -> Result<()> {
if !pio.copy {
return Ok(());
};
@ -454,8 +508,7 @@ pub async fn copy_io(pio: &ProcessIO, stdio: &Stdio) -> Result<()> {
.open(stdio.stdin.as_str())
.await
.map_err(io_error!(e, "open stdin"))?;
//let closer = io.stdin().unwrap();
spawn_copy(stdin, w, None::<fn()>);
spawn_copy(stdin, w, exit_signal.clone(), None::<fn()>);
}
}
@ -477,6 +530,7 @@ pub async fn copy_io(pio: &ProcessIO, stdio: &Stdio) -> Result<()> {
spawn_copy(
r,
stdout,
exit_signal.clone(),
Some(move || {
drop(stdout_r);
}),
@ -502,6 +556,7 @@ pub async fn copy_io(pio: &ProcessIO, stdio: &Stdio) -> Result<()> {
spawn_copy(
r,
stderr,
exit_signal,
Some(move || {
drop(stderr_r);
}),
@ -513,7 +568,7 @@ pub async fn copy_io(pio: &ProcessIO, stdio: &Stdio) -> Result<()> {
Ok(())
}
fn spawn_copy<R, W, F>(from: R, to: W, on_close: Option<F>)
fn spawn_copy<R, W, F>(from: R, to: W, exit_signal: Arc<ExitSignal>, on_close: Option<F>)
where
R: AsyncRead + Send + Unpin + 'static,
W: AsyncWrite + Send + Unpin + 'static,
@ -522,12 +577,16 @@ where
let mut src = from;
let mut dst = to;
tokio::spawn(async move {
tokio::io::copy(&mut src, &mut dst)
.await
.unwrap_or_else(|e| {
error!("copy io failed {}", e);
0
});
tokio::select! {
_ = exit_signal.wait() => {
debug!("container exit, copy task should exit too");
},
res = tokio::io::copy(&mut src, &mut dst) => {
if let Err(e) = res {
error!("copy io failed {}", e);
}
}
}
if let Some(f) = on_close {
f();
}
@ -538,22 +597,23 @@ async fn copy_io_or_console<P>(
p: &mut ProcessTemplate<P>,
socket: Option<ConsoleSocket>,
pio: Option<ProcessIO>,
exit_signal: Arc<ExitSignal>,
) -> Result<()> {
if p.stdio.terminal {
if let Some(console_socket) = socket {
let console_result = copy_console(&console_socket, &p.stdio).await;
let console_result = copy_console(&console_socket, &p.stdio, exit_signal).await;
console_socket.clean().await;
match console_result {
Ok(c) => {
p.console = Some(c);
}
Err(e) => {
console_socket.clean().await;
return Err(e);
}
}
}
} else if let Some(pio) = pio {
copy_io(&pio, &p.stdio).await?;
copy_io(&pio, &p.stdio, exit_signal).await?;
}
Ok(())
}

18
crates/runc-shim/src/common.rs
View File

@ -23,6 +23,7 @@ use nix::cmsg_space;
use nix::sys::socket::{recvmsg, ControlMessageOwned, MsgFlags};
use nix::sys::termios::tcgetattr;
use nix::sys::uio::IoVec;
use oci_spec::runtime::{LinuxNamespaceType, Spec};
use containerd_shim::api::{ExecProcessRequest, Options};
use containerd_shim::io::Stdio;
@ -200,3 +201,20 @@ pub fn receive_socket(stream_fd: RawFd) -> containerd_shim::Result<RawFd> {
tcgetattr(fds[0])?;
Ok(fds[0])
}
pub fn has_shared_pid_namespace(spec: &Spec) -> bool {
match spec.linux() {
None => true,
Some(linux) => match linux.namespaces() {
None => true,
Some(namespaces) => {
for ns in namespaces {
if ns.typ() == LinuxNamespaceType::Pid && ns.path().is_none() {
return false;
}
}
true
}
},
}
}

1
crates/runc-shim/src/synchronous/mod.rs
View File

@ -18,7 +18,6 @@ use std::sync::Arc;
use containerd_shim::ExitSignal;
mod cgroup;
mod container;
mod io;
mod runc;

25
crates/runc-shim/src/synchronous/runc.rs
View File

@ -47,7 +47,9 @@ use shim::Console;
use shim::{other, other_error};
use crate::common;
use crate::common::{create_io, CreateConfig, ShimExecutor, INIT_PID_FILE};
use crate::common::{
create_io, has_shared_pid_namespace, CreateConfig, ShimExecutor, INIT_PID_FILE,
};
use crate::synchronous::container::{
CommonContainer, CommonProcess, Container, ContainerFactory, Process,
};
@ -294,7 +296,7 @@ impl Container for RuncContainer {
#[cfg(target_os = "linux")]
fn stats(&self) -> Result<Metrics> {
let pid = self.common.init.pid() as u32;
crate::synchronous::cgroup::collect_metrics(pid)
containerd_shim::cgroup::collect_metrics(pid)
}
#[cfg(not(target_os = "linux"))]
@ -305,7 +307,7 @@ impl Container for RuncContainer {
#[cfg(target_os = "linux")]
fn update(&mut self, resources: &LinuxResources) -> Result<()> {
let pid = self.common.init.pid() as u32;
crate::synchronous::cgroup::update_metrics(pid, resources)
containerd_shim::cgroup::update_resources(pid, resources)
}
#[cfg(not(target_os = "linux"))]
@ -353,20 +355,7 @@ impl Container for RuncContainer {
impl RuncContainer {
pub(crate) fn should_kill_all_on_exit(&mut self, bundle_path: &str) -> bool {
match read_spec_from_file(bundle_path) {
Ok(spec) => match spec.linux() {
None => true,
Some(linux) => match linux.namespaces() {
None => true,
Some(namespaces) => {
for ns in namespaces {
if ns.typ() == LinuxNamespaceType::Pid && ns.path().is_none() {
return false;
}
}
true
}
},
},
Ok(spec) => has_shared_pid_namespace(&spec),
Err(e) => {
error!("should_kill_all_on_exit: {}", e);
false
@ -437,7 +426,7 @@ impl InitProcess {
let bundle = self.bundle.to_string();
let pid_path = Path::new(&bundle).join(INIT_PID_FILE);
let mut create_opts = runc::options::CreateOpts::new()
.pid_file(pid_path.to_owned())
.pid_file(&pid_path)
.no_pivot(self.no_pivot_root)
.no_new_keyring(self.no_new_key_ring)
.detach(false);

6
crates/runc-shim/src/synchronous/service.rs
View File

@ -21,9 +21,9 @@ use std::path::Path;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::Arc;
use containerd_shim as shim;
use log::{debug, error};
use containerd_shim as shim;
use runc::options::{DeleteOpts, GlobalOpts, DEFAULT_COMMAND};
use shim::api::*;
use shim::error::{Error, Result};
@ -73,7 +73,7 @@ impl Shim for Service {
let (child_id, address) = spawn(opts, &grouping, Vec::new())?;
#[cfg(target_os = "linux")]
crate::synchronous::cgroup::set_cgroup_and_oom_score(child_id)?;
containerd_shim::cgroup::set_cgroup_and_oom_score(child_id)?;
write_address(&address)?;
Ok(address)

3
crates/shim/Cargo.toml
View File

@ -40,5 +40,8 @@ tokio = { version = "1.17.0", features = ["full"], optional = true }
futures = {version = "0.3.21", optional = true}
signal-hook-tokio = {version = "0.3.1", optional = true, features = ["futures-v0_3"]}
[target.'cfg(target_os = "linux")'.dependencies]
cgroups-rs = "0.2.9"
[dev-dependencies]
tempfile = "3.0"

2
crates/shim/src/asynchronous/console.rs
View File

@ -57,7 +57,7 @@ impl ConsoleSocket {
}
// async drop is not supported yet, we can only call clean manually after socket received
pub async fn clean(&self) {
pub async fn clean(self) {
if self.rmdir {
if let Some(tmp_socket_dir) = self.path.parent() {
tokio::fs::remove_dir_all(tmp_socket_dir)

33
crates/shim/src/asynchronous/container.rs
View File

@ -18,10 +18,14 @@ use std::collections::HashMap;
use async_trait::async_trait;
use log::debug;
use oci_spec::runtime::LinuxResources;
use time::OffsetDateTime;
use tokio::sync::oneshot::Receiver;
use containerd_shim_protos::api::{CreateTaskRequest, ExecProcessRequest, StateResponse};
use containerd_shim_protos::api::{
CreateTaskRequest, ExecProcessRequest, ProcessInfo, StateResponse,
};
use containerd_shim_protos::cgroups::metrics::Metrics;
use crate::asynchronous::processes::Process;
use crate::error::Result;
@ -45,6 +49,9 @@ pub trait Container {
async fn resize_pty(&mut self, exec_id: Option<&str>, height: u32, width: u32) -> Result<()>;
async fn pid(&self) -> i32;
async fn id(&self) -> String;
async fn update(&mut self, resources: &LinuxResources) -> Result<()>;
async fn stats(&self) -> Result<Metrics>;
async fn all_processes(&self) -> Result<Vec<ProcessInfo>>;
}
#[async_trait]
@ -154,6 +161,30 @@ where
async fn id(&self) -> String {
self.id.to_string()
}
#[cfg(target_os = "linux")]
async fn update(&mut self, resources: &LinuxResources) -> Result<()> {
self.init.update(resources).await
}
#[cfg(not(target_os = "linux"))]
async fn update(&mut self, _resources: &LinuxResources) -> Result<()> {
Err(Error::Unimplemented("update".to_string()))
}
#[cfg(target_os = "linux")]
async fn stats(&self) -> Result<Metrics> {
self.init.stats().await
}
#[cfg(not(target_os = "linux"))]
async fn stats(&self) -> Result<Metrics> {
Err(Error::Unimplemented("stats".to_string()))
}
async fn all_processes(&self) -> Result<Vec<ProcessInfo>> {
self.init.ps().await
}
}
impl<T, E, P> ContainerTemplate<T, E, P>

58
crates/shim/src/asynchronous/mod.rs
View File

@ -98,16 +98,6 @@ where
{
if let Some(err) = bootstrap::<T>(runtime_id, opts).await.err() {
eprintln!("{}: {:?}", runtime_id, err);
let mut f = tokio::fs::OpenOptions::new()
.create(true)
.write(true)
.append(true)
.open("/tmp/containerd-shim.log")
.await
.unwrap();
f.write_all(format!("error bootstrap {}....\n", err).as_bytes())
.await
.unwrap();
process::exit(1);
}
}
@ -122,7 +112,7 @@ where
let ttrpc_address = env::var(TTRPC_ADDRESS)?;
// Create shim instance
let mut config = opts.unwrap_or_else(Config::default);
let mut config = opts.unwrap_or_default();
// Setup signals
let signals = setup_signals_tokio(&config);
@ -145,12 +135,13 @@ where
};
let address = shim.start_shim(args).await?;
tokio::io::stdout()
let mut stdout = tokio::io::stdout();
stdout
.write_all(address.as_bytes())
.await
.map_err(io_error!(e, "write stdout"))?;
// containerd occasionally read an empty string without flushing the stdout
stdout.flush().await.map_err(io_error!(e, "flush stdout"))?;
Ok(())
}
"delete" => {
@ -290,14 +281,11 @@ pub async fn spawn(opts: StartOpts, grouping: &str, vars: Vec<(&str, &str)>) ->
}
command.envs(vars);
command
.spawn()
.map_err(io_error!(e, "spawn shim"))
.map(|_| {
// Ownership of `listener` has been passed to child.
std::mem::forget(listener);
address
})
let _child = command.spawn().map_err(io_error!(e, "spawn shim"))?;
#[cfg(target_os = "linux")]
crate::cgroup::set_cgroup_and_oom_score(_child.id())?;
std::mem::forget(listener);
Ok(address)
}
fn setup_signals_tokio(config: &Config) -> Signals {
@ -314,25 +302,29 @@ async fn handle_signals(signals: Signals) {
match sig {
SIGTERM | SIGINT => {
debug!("received {}", sig);
return;
}
SIGCHLD => {
let mut status: c_int = 0;
let options: c_int = libc::WNOHANG;
let res_pid = asyncify(move || -> Result<pid_t> {
Ok(unsafe { libc::waitpid(-1, &mut status, options) })
SIGCHLD => loop {
let result = asyncify(move || -> Result<(pid_t, c_int)> {
let mut status: c_int = -1;
let pid = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) };
if pid <= 0 {
return Err(other!("wait finished"));
}
let status = libc::WEXITSTATUS(status);
Ok((pid, status))
})
.await
.unwrap_or(-1);
let status = libc::WEXITSTATUS(status);
if res_pid > 0 {
.await;
if let Ok((res_pid, status)) = result {
monitor_notify_by_pid(res_pid, status)
.await
.unwrap_or_else(|e| {
error!("failed to send pid exit event {}", e);
})
} else {
break;
}
}
},
_ => {}
}
}

72
crates/shim/src/asynchronous/processes.rs
View File

@ -15,17 +15,19 @@
*/
use std::os::unix::io::AsRawFd;
use std::sync::Arc;
use async_trait::async_trait;
use oci_spec::runtime::LinuxResources;
use time::OffsetDateTime;
use tokio::sync::oneshot::{channel, Receiver, Sender};
use containerd_shim_protos::api::{StateResponse, Status};
use containerd_shim_protos::api::{ProcessInfo, StateResponse, Status};
use containerd_shim_protos::cgroups::metrics::Metrics;
use containerd_shim_protos::protobuf::well_known_types::Timestamp;
use crate::io::Stdio;
use crate::util::asyncify;
use crate::Error;
use crate::{ioctl_set_winsz, Console};
#[async_trait]
@ -40,6 +42,9 @@ pub trait Process {
async fn exit_code(&self) -> i32;
async fn exited_at(&self) -> Option<OffsetDateTime>;
async fn resize_pty(&mut self, height: u32, width: u32) -> crate::Result<()>;
async fn update(&mut self, resources: &LinuxResources) -> crate::Result<()>;
async fn stats(&self) -> crate::Result<Metrics>;
async fn ps(&self) -> crate::Result<Vec<ProcessInfo>>;
}
#[async_trait]
@ -47,6 +52,9 @@ pub trait ProcessLifecycle<P: Process> {
async fn start(&self, p: &mut P) -> crate::Result<()>;
async fn kill(&self, p: &mut P, signal: u32, all: bool) -> crate::Result<()>;
async fn delete(&self, p: &mut P) -> crate::Result<()>;
async fn update(&self, p: &mut P, resources: &LinuxResources) -> crate::Result<()>;
async fn stats(&self, p: &P) -> crate::Result<Metrics>;
async fn ps(&self, p: &P) -> crate::Result<Vec<ProcessInfo>>;
}
pub struct ProcessTemplate<S> {
@ -58,16 +66,13 @@ pub struct ProcessTemplate<S> {
pub exited_at: Option<OffsetDateTime>,
pub wait_chan_tx: Vec<Sender<()>>,
pub console: Option<Console>,
pub lifecycle: S,
pub lifecycle: Arc<S>,
}
impl<S> ProcessTemplate<S>
where
S: Clone,
{
impl<S> ProcessTemplate<S> {
pub fn new(id: &str, stdio: Stdio, lifecycle: S) -> Self {
Self {
state: Default::default(),
state: Status::CREATED,
id: id.to_string(),
stdio,
pid: 0,
@ -75,7 +80,7 @@ where
exited_at: None,
wait_chan_tx: vec![],
console: None,
lifecycle,
lifecycle: Arc::new(lifecycle),
}
}
}
@ -83,10 +88,10 @@ where
#[async_trait]
impl<S> Process for ProcessTemplate<S>
where
S: ProcessLifecycle<Self> + Clone + Sync + Send,
S: ProcessLifecycle<Self> + Sync + Send,
{
async fn start(&mut self) -> crate::Result<()> {
self.lifecycle.clone().start(&mut self).await?;
self.lifecycle.clone().start(self).await?;
Ok(())
}
@ -122,11 +127,11 @@ where
}
async fn kill(&mut self, signal: u32, all: bool) -> crate::Result<()> {
self.lifecycle.clone().kill(&mut self, signal, all).await
self.lifecycle.clone().kill(self, signal, all).await
}
async fn delete(&mut self) -> crate::Result<()> {
self.lifecycle.clone().delete(&mut self).await
self.lifecycle.clone().delete(self).await
}
async fn wait_channel(&mut self) -> crate::Result<Receiver<()>> {
@ -146,22 +151,31 @@ where
}
async fn resize_pty(&mut self, height: u32, width: u32) -> crate::Result<()> {
match self.console.as_ref() {
Some(console) => unsafe {
let w = libc::winsize {
ws_row: height as u16,
ws_col: width as u16,
ws_xpixel: 0,
ws_ypixel: 0,
};
let fd = console.file.as_raw_fd();
asyncify(move || -> crate::Result<()> {
ioctl_set_winsz(fd, &w).map(|_x| ()).map_err(Into::into)
})
.await?;
Ok(())
},
None => Err(other!("there is no console")),
if let Some(console) = self.console.as_ref() {
let w = libc::winsize {
ws_row: height as u16,
ws_col: width as u16,
ws_xpixel: 0,
ws_ypixel: 0,
};
let fd = console.file.as_raw_fd();
asyncify(move || -> crate::Result<()> {
unsafe { ioctl_set_winsz(fd, &w).map(|_x| ()).map_err(Into::into) }
})
.await?;
}
Ok(())
}
async fn update(&mut self, resources: &LinuxResources) -> crate::Result<()> {
self.lifecycle.clone().update(self, resources).await
}
async fn stats(&self) -> crate::Result<Metrics> {
self.lifecycle.stats(self).await
}
async fn ps(&self) -> crate::Result<Vec<ProcessInfo>> {
self.lifecycle.ps(self).await
}
}

63
crates/shim/src/asynchronous/task.rs
View File

@ -19,10 +19,14 @@ use std::sync::Arc;
use async_trait::async_trait;
use log::{debug, info, warn};
use oci_spec::runtime::LinuxResources;
use tokio::sync::mpsc::Sender;
use tokio::sync::{MappedMutexGuard, Mutex, MutexGuard};
use containerd_shim_protos::api::DeleteResponse;
use containerd_shim_protos::api::{
CloseIORequest, ConnectRequest, ConnectResponse, DeleteResponse, PidsRequest, PidsResponse,
StatsRequest, StatsResponse, UpdateTaskRequest,
};
use containerd_shim_protos::events::task::{
TaskCreate, TaskDelete, TaskExecAdded, TaskExecStarted, TaskIO, TaskStart,
};
@ -39,7 +43,7 @@ use crate::api::{
use crate::asynchronous::container::{Container, ContainerFactory};
use crate::asynchronous::ExitSignal;
use crate::event::Event;
use crate::util::{convert_to_timestamp, AsOption};
use crate::util::{convert_to_any, convert_to_timestamp, AsOption};
use crate::TtrpcResult;
type EventSender = Sender<(String, Box<dyn Message>)>;
@ -215,6 +219,17 @@ where
Ok(resp)
}
async fn pids(&self, _ctx: &TtrpcContext, req: PidsRequest) -> TtrpcResult<PidsResponse> {
debug!("Pids request for {:?}", req);
let container = self.get_container(req.get_id()).await?;
let procs = container.all_processes().await?;
debug!("Pids request for {:?} returns successfully", req);
Ok(PidsResponse {
processes: procs.into(),
..Default::default()
})
}
async fn kill(&self, _ctx: &TtrpcContext, req: KillRequest) -> TtrpcResult<Empty> {
info!("Kill request for {:?}", req);
let mut container = self.get_container(req.get_id()).await?;
@ -253,6 +268,25 @@ where
Ok(Empty::new())
}
async fn close_io(&self, _ctx: &TtrpcContext, _req: CloseIORequest) -> TtrpcResult<Empty> {
// TODO call close_io of container
Ok(Empty::new())
}
async fn update(&self, _ctx: &TtrpcContext, req: UpdateTaskRequest) -> TtrpcResult<Empty> {
debug!("Update request for {:?}", req);
let resources: LinuxResources = serde_json::from_slice(req.get_resources().get_value())
.map_err(|e| {
ttrpc::Error::RpcStatus(ttrpc::get_status(
ttrpc::Code::INVALID_ARGUMENT,
format!("failed to parse resource spec: {}", e),
))
})?;
let mut container = self.get_container(req.get_id()).await?;
container.update(&resources).await?;
Ok(Empty::new())
}
async fn wait(&self, _ctx: &TtrpcContext, req: WaitRequest) -> TtrpcResult<WaitResponse> {
info!("Wait request for {:?}", req);
let exec_id = req.exec_id.as_str().as_option();
@ -283,6 +317,31 @@ where
Ok(resp)
}
async fn stats(&self, _ctx: &TtrpcContext, req: StatsRequest) -> TtrpcResult<StatsResponse> {
debug!("Stats request for {:?}", req);
let container = self.get_container(req.get_id()).await?;
let stats = container.stats().await?;
let mut resp = StatsResponse::new();
resp.set_stats(convert_to_any(Box::new(stats))?);
Ok(resp)
}
async fn connect(
&self,
_ctx: &TtrpcContext,
req: ConnectRequest,
) -> TtrpcResult<ConnectResponse> {
info!("Connect request for {:?}", req);
let container = self.get_container(req.get_id()).await?;
Ok(ConnectResponse {
shim_pid: std::process::id() as u32,
task_pid: container.pid().await as u32,
..Default::default()
})
}
async fn shutdown(&self, _ctx: &TtrpcContext, _req: ShutdownRequest) -> TtrpcResult<Empty> {
debug!("Shutdown request");
let containers = self.containers.lock().await;

17
crates/runc-shim/src/synchronous/cgroup.rs → crates/shim/src/cgroup.rs
View File

@ -24,15 +24,14 @@ use cgroups_rs::cgroup::get_cgroups_relative_paths_by_pid;
use cgroups_rs::{hierarchies, Cgroup, CgroupPid, MaxValue, Subsystem};
use oci_spec::runtime::LinuxResources;
use containerd_shim as shim;
use containerd_shim::api::Options;
use containerd_shim::protos::cgroups::metrics::{
use containerd_shim_protos::cgroups::metrics::{
CPUStat, CPUUsage, MemoryEntry, MemoryStat, Metrics,
};
use containerd_shim::protos::protobuf::well_known_types::Any;
use containerd_shim::protos::protobuf::Message;
use shim::error::{Error, Result};
use shim::{io_error, other_error};
use containerd_shim_protos::protobuf::well_known_types::Any;
use containerd_shim_protos::protobuf::Message;
use containerd_shim_protos::shim::oci::Options;
use crate::error::{Error, Result};
// OOM_SCORE_ADJ_MAX is from https://github.com/torvalds/linux/blob/master/include/uapi/linux/oom.h#L10
const OOM_SCORE_ADJ_MAX: i64 = 1000;
@ -131,7 +130,7 @@ pub fn collect_metrics(pid: u32) -> Result<Metrics> {
}
/// Update process cgroup limits
pub fn update_metrics(pid: u32, resources: &LinuxResources) -> Result<()> {
pub fn update_resources(pid: u32, resources: &LinuxResources) -> Result<()> {
// get container main process cgroup
let path =
get_cgroups_relative_paths_by_pid(pid).map_err(other_error!(e, "get process cgroup"))?;
@ -225,7 +224,7 @@ pub fn update_metrics(pid: u32, resources: &LinuxResources) -> Result<()> {
mod tests {
use cgroups_rs::{hierarchies, Cgroup, CgroupPid};
use crate::synchronous::cgroup::{
use crate::cgroup::{
add_task_to_cgroup, adjust_oom_score, read_process_oom_score, OOM_SCORE_ADJ_MAX,
};

4
crates/shim/src/lib.rs
View File

@ -39,8 +39,9 @@ use std::os::unix::io::RawFd;
use std::os::unix::net::UnixListener;
use std::path::{Path, PathBuf};
pub use containerd_shim_protos as protos;
use nix::ioctl_write_ptr_bad;
pub use containerd_shim_protos as protos;
pub use protos::shim::shim::DeleteResponse;
pub use protos::ttrpc::{context::Context, Result as TtrpcResult};
@ -56,6 +57,7 @@ pub mod error;
mod args;
#[cfg(feature = "async")]
pub mod asynchronous;
pub mod cgroup;
pub mod event;
pub mod io;
mod logger;

3
crates/shim/src/synchronous/mod.rs
View File

@ -146,7 +146,7 @@ where
let ttrpc_address = env::var(TTRPC_ADDRESS)?;
// Create shim instance
let mut config = opts.unwrap_or_else(Config::default);
let mut config = opts.unwrap_or_default();
// Setup signals
let signals = setup_signals(&config);
@ -228,7 +228,6 @@ fn handle_signals(mut signals: Signals) {
match sig {
SIGTERM | SIGINT => {
debug!("received {}", sig);
return;
}
SIGCHLD => loop {
unsafe {

2
rust-toolchain.toml
View File

@ -1,3 +1,3 @@
[toolchain]
channel = "1.54"
channel = "1.59"
components = ["rustfmt", "clippy"]