!84 KubeOS: update to 1.0.8
From: @weiy6 Reviewed-by: @li-yuanrong Signed-off-by: @li-yuanrong
This commit is contained in:
openeuler-ci-bot
Gitee
commit
2af2d363d2
@ -1,61 +0,0 @@
|
|||||||
From e44e963748c8f2c19bd95383074c49fe35110ac2 Mon Sep 17 00:00:00 2001
|
|
||||||
From: weihao <weihao_yewu@cmss.chinamobile.com>
|
|
||||||
Date: Wed, 27 Nov 2024 09:45:45 +0800
|
|
||||||
Subject: [PATCH 01/11] fix: Fix some spelling errors in docs
|
|
||||||
|
|
||||||
---
|
|
||||||
docs/quick-start.md | 10 +++++-----
|
|
||||||
1 file changed, 5 insertions(+), 5 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/docs/quick-start.md b/docs/quick-start.md
|
|
||||||
index 1d59048e..da379062 100644
|
|
||||||
--- a/docs/quick-start.md
|
|
||||||
+++ b/docs/quick-start.md
|
|
||||||
@@ -163,7 +163,7 @@
|
|
||||||
* 使用kubernetes的声明式API进行配置,部署CRD（CustomResourceDefinition），operator，proxy以及rbac机制的YAML需要用户自行编写
|
|
||||||
* YAML举例说明模板参见本目录下example文件夹下的文件夹，你也可以将config文件夹拷贝到docs上一级目录，并进行简单的修改使用
|
|
||||||
* 这些YAML配置文件，由K8s集群管理员加载，如果恶意在yaml文件里面写了病毒，K8s集群管理员如果放行，传到我们的处理模块我们也是没有办法校验的，此处有风险
|
|
||||||
- * operator和proxy部署在kubernets集群中，operator应部署为deployment，proxy应部署为damonset
|
|
||||||
+ * operator和proxy部署在kubernetes集群中，operator应部署为deployment，proxy应部署为damonset
|
|
||||||
* 尽量部署好k8s的安全措施，如rbac机制，pod的service account和security policy配置等。**注意**：operator所在容器仅需要普通用户权限运行，proxy所在容器需要root权限运行以访问worker节点上的os-agent.sock，但是可以drop全部的capabilities，如：
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
@@ -345,7 +345,7 @@
|
|
||||||
version: edit.os.version
|
|
||||||
configs:
|
|
||||||
- model: kernel.sysctl
|
|
||||||
- contents:
|
|
||||||
+ contents:
|
|
||||||
- key: kernel param key1
|
|
||||||
value: kernel param value1
|
|
||||||
- key: kernel param key2
|
|
||||||
@@ -361,7 +361,7 @@
|
|
||||||
version: 1.0.0
|
|
||||||
configs:
|
|
||||||
- model: kernel.sysctl
|
|
||||||
- contents:
|
|
||||||
+ contents:
|
|
||||||
- key: kernel param key4
|
|
||||||
value: kernel param value4
|
|
||||||
```
|
|
||||||
@@ -536,7 +536,7 @@
|
|
||||||
apiVersion: upgrade.openeuler.org/v1alpha1
|
|
||||||
kind: OS
|
|
||||||
metadata:
|
|
||||||
- name: os-sample
|
|
||||||
+ name: os-sample
|
|
||||||
spec:
|
|
||||||
imagetype: ""
|
|
||||||
opstype: rollback
|
|
||||||
@@ -728,7 +728,7 @@ hostshell
|
|
||||||
|
|
||||||
#### kernel Settings
|
|
||||||
|
|
||||||
-* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下:
|
|
||||||
+* kernel.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
configs:
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,191 +0,0 @@
|
|||||||
From 59de6f644def5555c41e2ef14317e9178c40259f Mon Sep 17 00:00:00 2001
|
|
||||||
From: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
Date: Wed, 27 Nov 2024 18:34:14 +0000
|
|
||||||
Subject: [PATCH 03/11] fix(os-agent): use findmnt to find out which device
|
|
||||||
mounted on /
|
|
||||||
|
|
||||||
use "findmnt -no SOURCE --mountpoint /" to get the device name mounted on /
|
|
||||||
use "lsblk -blno FSTYPE,SIZE" to get the fstype and partition size
|
|
||||||
This patch fixes the problem if there are multiple mountpoints on a device
|
|
||||||
|
|
||||||
Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
---
|
|
||||||
KubeOS-Rust/manager/src/sys_mgmt/config.rs | 21 ++----
|
|
||||||
KubeOS-Rust/manager/src/utils/partition.rs | 86 +++++++++-------------
|
|
||||||
2 files changed, 41 insertions(+), 66 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
index 8e42fa18..a491a429 100644
|
|
||||||
--- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
@@ -826,13 +826,10 @@ mod tests {
|
|
||||||
let mut executor = MockCommandExec::new();
|
|
||||||
|
|
||||||
// the output shows that current root menuentry is A
|
|
||||||
- let command_output1 = r#"vda 23622320128
|
|
||||||
-vda1 /boot/efi vfat 61865984 BOOT
|
|
||||||
-vda2 / ext4 3145728000 ROOT-A
|
|
||||||
-vda3 ext4 2621440000 ROOT-B
|
|
||||||
-vda4 /persist ext4 17791188992 PERSIST
|
|
||||||
-"#;
|
|
||||||
- executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
|
|
||||||
+ let findmnt_output1 = "/dev/vda2";
|
|
||||||
+ let lsblk_output1 = "ext4 3145728000\n";
|
|
||||||
+ executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(findmnt_output1.to_string()));
|
|
||||||
+ executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(lsblk_output1.to_string()));
|
|
||||||
|
|
||||||
let result = grub_cmdline.get_config_partition(executor).unwrap();
|
|
||||||
// it should return false because the current root menuentry is A and we want to configure current partition
|
|
||||||
@@ -840,14 +837,8 @@ vda4 /persist ext4 17791188992 PERSIST
|
|
||||||
|
|
||||||
let mut executor = MockCommandExec::new();
|
|
||||||
|
|
||||||
- // the output shows that current root menuentry is A
|
|
||||||
- let command_output1 = r#"vda 23622320128
|
|
||||||
-vda1 /boot/efi vfat 61865984 BOOT
|
|
||||||
-vda2 / ext4 3145728000 ROOT-A
|
|
||||||
-vda3 ext4 2621440000 ROOT-B
|
|
||||||
-vda4 /persist ext4 17791188992 PERSIST
|
|
||||||
-"#;
|
|
||||||
- executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
|
|
||||||
+ executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(findmnt_output1.to_string()));
|
|
||||||
+ executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(lsblk_output1.to_string()));
|
|
||||||
grub_cmdline.is_cur_partition = false;
|
|
||||||
let result = grub_cmdline.get_config_partition(executor).unwrap();
|
|
||||||
// it should return true because the current root menuentry is A and we want to configure next partition
|
|
||||||
diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs
|
|
||||||
index 4941ee9d..b2c095c6 100644
|
|
||||||
--- a/KubeOS-Rust/manager/src/utils/partition.rs
|
|
||||||
+++ b/KubeOS-Rust/manager/src/utils/partition.rs
|
|
||||||
@@ -25,43 +25,33 @@ pub struct PartitionInfo {
|
|
||||||
|
|
||||||
/// get_partition_info returns the current partition info and the next partition info.
|
|
||||||
pub fn get_partition_info<T: CommandExecutor>(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> {
|
|
||||||
- let lsblk = executor.run_command_with_output("lsblk", &["-blno", "NAME,MOUNTPOINT,FSTYPE,SIZE,LABEL"])?;
|
|
||||||
let mut cur_partition = PartitionInfo::default();
|
|
||||||
let mut next_partition = PartitionInfo::default();
|
|
||||||
- let mut found_boot = 0;
|
|
||||||
- trace!("get_partition_info lsblk command output:\n{}", lsblk);
|
|
||||||
- for line in lsblk.lines() {
|
|
||||||
- let res: Vec<&str> = line.split_whitespace().collect();
|
|
||||||
- if res.len() == 5 && res[4] == "BOOT" {
|
|
||||||
- trace!("Found boot partition:\n{:?}", res);
|
|
||||||
- found_boot = 2;
|
|
||||||
- continue;
|
|
||||||
- }
|
|
||||||
- if found_boot > 0 {
|
|
||||||
- trace!("Handling two root partitions:\n{:?}", res);
|
|
||||||
- if res[1] == "/" {
|
|
||||||
- // current partition
|
|
||||||
- cur_partition.device = format!("/dev/{}", res[0]).to_string();
|
|
||||||
- cur_partition.fs_type = res[2].to_string();
|
|
||||||
- cur_partition.size = res[3]
|
|
||||||
- .parse()
|
|
||||||
- .with_context(|| format!("Failed to parse current partition size to i64: \"{}\"", res[3]))?;
|
|
||||||
- cur_partition.menuentry = if res[0].contains("2") { String::from("A") } else { String::from("B") };
|
|
||||||
- } else {
|
|
||||||
- // next partition
|
|
||||||
- next_partition.device = format!("/dev/{}", res[0]).to_string();
|
|
||||||
- next_partition.fs_type = res[1].to_string();
|
|
||||||
- next_partition.size = res[2]
|
|
||||||
- .parse()
|
|
||||||
- .with_context(|| format!("Failed to parse next partition size to i64: \"{}\"", res[2]))?;
|
|
||||||
- next_partition.menuentry = if res[0].contains("2") { String::from("A") } else { String::from("B") };
|
|
||||||
- }
|
|
||||||
- found_boot -= 1;
|
|
||||||
- }
|
|
||||||
+ cur_partition.device = executor.run_command_with_output("findmnt", &["-no", "SOURCE", "--mountpoint", "/"])?;
|
|
||||||
+ trace!("{} is mounted on /", cur_partition.device);
|
|
||||||
+ if cur_partition.device.contains('2') {
|
|
||||||
+ cur_partition.menuentry = String::from("A");
|
|
||||||
+ next_partition.menuentry = String::from("B");
|
|
||||||
+ next_partition.device = cur_partition.device.replace("2", "3");
|
|
||||||
+ } else if cur_partition.device.contains('3') {
|
|
||||||
+ cur_partition.menuentry = String::from("B");
|
|
||||||
+ next_partition.menuentry = String::from("A");
|
|
||||||
+ next_partition.device = cur_partition.device.replace("3", "2");
|
|
||||||
+ } else {
|
|
||||||
+ bail!("Failed to get partition info, / is not mounted on the second or the third partition");
|
|
||||||
}
|
|
||||||
- if cur_partition.menuentry.is_empty() || next_partition.menuentry.is_empty() {
|
|
||||||
- bail!("Failed to get partition info, lsblk output: {}", lsblk);
|
|
||||||
+ let lsblk = executor.run_command_with_output("lsblk", &["-blno", "FSTYPE,SIZE", &cur_partition.device])?;
|
|
||||||
+ trace!("get_partition_info lsblk command output:\n{}", lsblk);
|
|
||||||
+ let elements: Vec<&str> = lsblk.split_whitespace().collect();
|
|
||||||
+ if elements.len() != 2 {
|
|
||||||
+ bail!("Failed to get partition info of FSTYPE and SIZE, lsblk output: {}", lsblk);
|
|
||||||
}
|
|
||||||
+ cur_partition.fs_type = elements[0].to_string();
|
|
||||||
+ next_partition.fs_type = elements[0].to_string();
|
|
||||||
+ cur_partition.size = elements[1]
|
|
||||||
+ .parse()
|
|
||||||
+ .with_context(|| format!("Failed to parse current partition size to i64: \"{}\"", elements[1]))?;
|
|
||||||
+ next_partition.size = cur_partition.size;
|
|
||||||
Ok((cur_partition, next_partition))
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -94,14 +84,11 @@ mod tests {
|
|
||||||
#[test]
|
|
||||||
fn test_get_partition_info() {
|
|
||||||
init();
|
|
||||||
- let command_output1 = r#"vda 23622320128
|
|
||||||
-vda1 /boot/efi vfat 61865984 BOOT
|
|
||||||
-vda2 / ext4 3145728000 ROOT-A
|
|
||||||
-vda3 ext4 2621440000 ROOT-B
|
|
||||||
-vda4 /persist ext4 17791188992 PERSIST
|
|
||||||
-"#;
|
|
||||||
+ let findmnt_output1 = "/dev/vda2";
|
|
||||||
+ let lsblk_output1 = "ext4 3145728000\n";
|
|
||||||
let mut mock = MockCommandExec::new();
|
|
||||||
- mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
|
|
||||||
+ mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(findmnt_output1.to_string()));
|
|
||||||
+ mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(lsblk_output1.to_string()));
|
|
||||||
let res = get_partition_info(&mock).unwrap();
|
|
||||||
let expect_res = (
|
|
||||||
PartitionInfo {
|
|
||||||
@@ -114,25 +101,22 @@ vda4 /persist ext4 17791188992 PERSIST
|
|
||||||
device: "/dev/vda3".to_string(),
|
|
||||||
menuentry: "B".to_string(),
|
|
||||||
fs_type: "ext4".to_string(),
|
|
||||||
- size: 2621440000,
|
|
||||||
+ size: 3145728000,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
assert_eq!(res, expect_res);
|
|
||||||
|
|
||||||
- let command_output2 = r#"vda 23622320128
|
|
||||||
-vda1 /boot/efi vfat 61865984 BOOT
|
|
||||||
-vda2 ext4 3145728000 ROOT-A
|
|
||||||
-vda3 / ext4 2621440000 ROOT-B
|
|
||||||
-vda4 /persist ext4 17791188992 PERSIST
|
|
||||||
-"#;
|
|
||||||
- mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string()));
|
|
||||||
+ let findmnt_output2 = "/dev/vda3";
|
|
||||||
+ let lsblk_output2 = "ext4 3145728000\n";
|
|
||||||
+ mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(findmnt_output2.to_string()));
|
|
||||||
+ mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(lsblk_output2.to_string()));
|
|
||||||
let res = get_partition_info(&mock).unwrap();
|
|
||||||
let expect_res = (
|
|
||||||
PartitionInfo {
|
|
||||||
device: "/dev/vda3".to_string(),
|
|
||||||
menuentry: "B".to_string(),
|
|
||||||
fs_type: "ext4".to_string(),
|
|
||||||
- size: 2621440000,
|
|
||||||
+ size: 3145728000,
|
|
||||||
},
|
|
||||||
PartitionInfo {
|
|
||||||
device: "/dev/vda2".to_string(),
|
|
||||||
@@ -148,8 +132,8 @@ vda4 /persist ext4 17791188992 PERSIST
|
|
||||||
let res = get_partition_info(&mock);
|
|
||||||
assert!(res.is_err());
|
|
||||||
|
|
||||||
- let command_output4 = "sda4 / ext4 13000245248";
|
|
||||||
- mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output4.to_string()));
|
|
||||||
+ let findmnt_output3 = "/dev/vda4";
|
|
||||||
+ mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(findmnt_output3.to_string()));
|
|
||||||
let res = get_partition_info(&mock);
|
|
||||||
assert!(res.is_err());
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,70 +0,0 @@
|
|||||||
From 9cbf66603c5459be84467ba9de1f9248d082cbdf Mon Sep 17 00:00:00 2001
|
|
||||||
From: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
Date: Wed, 27 Nov 2024 21:30:17 +0000
|
|
||||||
Subject: [PATCH 04/11] style: format code
|
|
||||||
|
|
||||||
Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
---
|
|
||||||
KubeOS-Rust/manager/src/sys_mgmt/config.rs | 2 +-
|
|
||||||
KubeOS-Rust/manager/src/sys_mgmt/values.rs | 12 ++++++------
|
|
||||||
KubeOS-Rust/proxy/src/controller/apiserver_mock.rs | 2 +-
|
|
||||||
3 files changed, 8 insertions(+), 8 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
index a491a429..787d5bbd 100644
|
|
||||||
--- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
@@ -59,7 +59,7 @@ lazy_static! {
|
|
||||||
);
|
|
||||||
config_map.insert(
|
|
||||||
values::PAM_LIMTS.to_string(),
|
|
||||||
- Box::new(PamLimits{config_path: values::DEFAULT_PAM_LIMITS_PATH.to_string()})
|
|
||||||
+ Box::new(PamLimits { config_path: values::DEFAULT_PAM_LIMITS_PATH.to_string() })
|
|
||||||
as Box<dyn Configuration + Sync>,
|
|
||||||
);
|
|
||||||
config_map
|
|
||||||
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/values.rs b/KubeOS-Rust/manager/src/sys_mgmt/values.rs
|
|
||||||
index 3236caf3..e1addfa4 100644
|
|
||||||
--- a/KubeOS-Rust/manager/src/sys_mgmt/values.rs
|
|
||||||
+++ b/KubeOS-Rust/manager/src/sys_mgmt/values.rs
|
|
||||||
@@ -14,17 +14,17 @@ pub const KERNEL_SYSCTL: &str = "kernel.sysctl";
|
|
||||||
pub const KERNEL_SYSCTL_PERSIST: &str = "kernel.sysctl.persist";
|
|
||||||
pub const GRUB_CMDLINE_CURRENT: &str = "grub.cmdline.current";
|
|
||||||
pub const GRUB_CMDLINE_NEXT: &str = "grub.cmdline.next";
|
|
||||||
-pub const KUBERNETES_KUBELET:&str = "kubernetes.kubelet";
|
|
||||||
-pub const CONTAINER_CONTAINERD:&str = "container.containerd";
|
|
||||||
-pub const PAM_LIMTS:&str = "pam.limits";
|
|
||||||
+pub const KUBERNETES_KUBELET: &str = "kubernetes.kubelet";
|
|
||||||
+pub const CONTAINER_CONTAINERD: &str = "container.containerd";
|
|
||||||
+pub const PAM_LIMTS: &str = "pam.limits";
|
|
||||||
|
|
||||||
pub const DEFAULT_PROC_PATH: &str = "/proc/sys/";
|
|
||||||
pub const DEFAULT_KERNEL_CONFIG_PATH: &str = "/etc/sysctl.conf";
|
|
||||||
pub const DEFAULT_GRUB_CFG_PATH: &str = "/boot/efi/EFI/openEuler/grub.cfg";
|
|
||||||
pub const DEFAULT_GRUBENV_PATH: &str = "/boot/efi/EFI/openEuler/grubenv";
|
|
||||||
-pub const DEFAULT_KUBELET_CONFIG_PATH: &str = "/var/lib/kubelet/config.yaml";
|
|
||||||
-pub const DEFAULT_CONTAINERD_CONFIG_PATH: &str = "/etc/containerd/config.toml";
|
|
||||||
-pub const DEFAULT_PAM_LIMITS_PATH:&str = "/etc/security/limits.conf";
|
|
||||||
+pub const DEFAULT_KUBELET_CONFIG_PATH: &str = "/var/lib/kubelet/config.yaml";
|
|
||||||
+pub const DEFAULT_CONTAINERD_CONFIG_PATH: &str = "/etc/containerd/config.toml";
|
|
||||||
+pub const DEFAULT_PAM_LIMITS_PATH: &str = "/etc/security/limits.conf";
|
|
||||||
|
|
||||||
pub const PERSIST_DIR: &str = "/persist";
|
|
||||||
pub const ROOTFS_ARCHIVE: &str = "os.tar";
|
|
||||||
diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
|
|
||||||
index a96043b6..45949600 100644
|
|
||||||
--- a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
|
|
||||||
+++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
|
|
||||||
@@ -28,8 +28,8 @@ use kube::{
|
|
||||||
core::{ListMeta, ObjectList},
|
|
||||||
Client as KubeClient, Resource, ResourceExt,
|
|
||||||
};
|
|
||||||
-use serde_json::json;
|
|
||||||
use mockall::mock;
|
|
||||||
+use serde_json::json;
|
|
||||||
|
|
||||||
use self::mock_error::Error;
|
|
||||||
use super::{
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,110 +0,0 @@
|
|||||||
From c53a851af1460b037c1dd22e434318963717eabc Mon Sep 17 00:00:00 2001
|
|
||||||
From: liyuanr <liyuanrong1@huawei.com>
|
|
||||||
Date: Wed, 27 Nov 2024 15:46:31 +0800
|
|
||||||
Subject: [PATCH 05/11] docs: add description of the configuration
|
|
||||||
|
|
||||||
add description of the configuration of kubelet,
|
|
||||||
containerd and pam limts
|
|
||||||
|
|
||||||
Signed-off-by: liyuanr <liyuanrong1@huawei.com>
|
|
||||||
---
|
|
||||||
.../figures/kubeos-architecture_2024.png | Bin 0 -> 241051 bytes
|
|
||||||
.../config/crd/upgrade.openeuler.org_os.yaml | 2 +-
|
|
||||||
docs/quick-start.md | 65 ++++++++++++++++++
|
|
||||||
3 files changed, 66 insertions(+), 1 deletion(-)
|
|
||||||
create mode 100644 docs/design/figures/kubeos-architecture_2024.png
|
|
||||||
|
|
||||||
diff --git a/docs/design/figures/kubeos-architecture_2024.png b/docs/design/figures/kubeos-architecture_2024.png
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..1862a1c4
|
|
||||||
Binary files /dev/null and b/docs/design/figures/kubeos-architecture_2024.png differ
|
|
||||||
diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml
|
|
||||||
index 97d0d27b..0ec657b2 100644
|
|
||||||
--- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml
|
|
||||||
+++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml
|
|
||||||
@@ -147,7 +147,7 @@ spec:
|
|
||||||
operation:
|
|
||||||
type: string
|
|
||||||
value:
|
|
||||||
- type: string
|
|
||||||
+ x-kubernetes-preserve-unknown-fields: true
|
|
||||||
type: object
|
|
||||||
type: array
|
|
||||||
model:
|
|
||||||
diff --git a/docs/quick-start.md b/docs/quick-start.md
|
|
||||||
index 6c5f47a2..5730c164 100644
|
|
||||||
--- a/docs/quick-start.md
|
|
||||||
+++ b/docs/quick-start.md
|
|
||||||
@@ -767,3 +767,68 @@ hostshell
|
|
||||||
operation: delete
|
|
||||||
- key: crash_kexec_post_notifiers
|
|
||||||
```
|
|
||||||
+### kubelet配置
|
|
||||||
+* kuberntes.kubelet: 配置节点kubelet的配置文件中的参数，参数说明和约束如下：
|
|
||||||
+ * 仅支持```KubeletConfiguration```中的配置参数。
|
|
||||||
+ * 节点kubelet配置文件需要为yaml格式的文件。
|
|
||||||
+ * 如不指定configpath，默认配置文件路径为```/var/lib/kubelet/config.yaml```，并且需要注意的是配置文件的路径需要与kubelet启动时的```-- config```参数指定的路径一致才能生效。
|
|
||||||
+ * 如配置存在嵌套，则通过```'.'```连接嵌套的key值，例如如果修改如下yaml示例中```cacheAuthorizedTTL```参数为1s。
|
|
||||||
+ ```
|
|
||||||
+ authorization:
|
|
||||||
+ mode: Webhook
|
|
||||||
+ webhook:
|
|
||||||
+ cacheAuthorizedTTL: 0s
|
|
||||||
+ ```
|
|
||||||
+ 参数配置示例如下：
|
|
||||||
+ ```
|
|
||||||
+ configs:
|
|
||||||
+ - model: kuberntes.kubelet
|
|
||||||
+ configpath: /etc/test.yaml
|
|
||||||
+ contents:
|
|
||||||
+ - key: authorization.webhook.cacheAuthorizedTTL
|
|
||||||
+ value: 1s
|
|
||||||
+ ```
|
|
||||||
+ * kubernetes.kubelet进行删除时，不对value与配置文件中的值进行比较
|
|
||||||
+### containerd配置
|
|
||||||
+* container.containerd: 配置节点上containerd的配置文件中的参数，参数说明和约束如下：
|
|
||||||
+ * containerd需要配置文件为toml格式，所以key为toml中该参数的表头.键名，例如希望修改如下toml示例中```no_shim```为true。
|
|
||||||
+ ```
|
|
||||||
+ [plugins."io.containerd.runtime.v1.linux"]
|
|
||||||
+ no_shim=false
|
|
||||||
+ runtime="runc"
|
|
||||||
+ runtime_root="
|
|
||||||
+ ```
|
|
||||||
+ 参数配置示例如下：
|
|
||||||
+ ```
|
|
||||||
+ configs:
|
|
||||||
+ - model: container.containerd
|
|
||||||
+ configpath: /etc/test.toml
|
|
||||||
+ contents:
|
|
||||||
+ - key: plugins."io.containerd.runtime.v1.linux".no_shim
|
|
||||||
+ value: true
|
|
||||||
+ ```
|
|
||||||
+ * toml使用```"."```分割键，os-agent识别时与toml保持一致，所以当键名中包含```"."```时，该键名需要使用```""```，例如上例中的```"io.containerd.runtime.v1.linux"```为一个键
|
|
||||||
+ * 如不指定configpath，默认配置文件路径为```/etc/containerd/config.toml```
|
|
||||||
+ * container.conatainerd配置的key和value均不能为空
|
|
||||||
+ * container.containerd进行删除时，不对value与配置文件中的值进行比较
|
|
||||||
+### Pam Limits配置
|
|
||||||
+* pam.limits：配置节点上/etc/security/limits.conf文件
|
|
||||||
+ * key为domain值，value的格式需要为type.item.value（limits.conf文件要求每行格式为：\<domain\> \<type\> \<item\> \<value\>），例如：
|
|
||||||
+ ```
|
|
||||||
+ configs:
|
|
||||||
+ - model: pam.limits
|
|
||||||
+ contents:
|
|
||||||
+ - key: ftp
|
|
||||||
+ value: soft.core.0
|
|
||||||
+ ```
|
|
||||||
+ * 更新时，如不需要对type/item/value更新时，可以使用```"_"```，忽略对此参数的更新，但value必须为点隔的三段式，例如：
|
|
||||||
+ ```
|
|
||||||
+ configs:
|
|
||||||
+ - model: pam.limits
|
|
||||||
+ contents:
|
|
||||||
+ - key: ftp
|
|
||||||
+ value: hard._.1
|
|
||||||
+ ```
|
|
||||||
+ * pam.limits新增时，value中不允许包含```"_"```
|
|
||||||
+ * pam.limits删除时，会对value进行校验，当value与配置文件中的值不同时，删除失败
|
|
||||||
+ * pam.limits配置的key和value均不能为空
|
|
||||||
\ No newline at end of file
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,77 +0,0 @@
|
|||||||
From 4acf9ad93decfcd50e1b10671fdc866e7766ac85 Mon Sep 17 00:00:00 2001
|
|
||||||
From: liyuanr <liyuanrong1@huawei.com>
|
|
||||||
Date: Fri, 29 Nov 2024 16:31:11 +0800
|
|
||||||
Subject: [PATCH 06/11] fix(os-agent): fix settings of kubelet and add log
|
|
||||||
|
|
||||||
1.Fix the error where kernel.sysctl value cannot be configured when set to
|
|
||||||
string
|
|
||||||
2.Add logs when adding configurations to kubelet and pam.limts
|
|
||||||
3.Kubelet configuration reports error when reading a file in the wrong
|
|
||||||
format and unable to convert it to a map
|
|
||||||
|
|
||||||
Signed-off-by: liyuanr <liyuanrong1@huawei.com>
|
|
||||||
---
|
|
||||||
KubeOS-Rust/manager/src/sys_mgmt/config.rs | 22 +++++++++++++++++-----
|
|
||||||
1 file changed, 17 insertions(+), 5 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
index 787d5bbd..ebf04d39 100644
|
|
||||||
--- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
|
|
||||||
@@ -102,13 +102,13 @@ impl Configuration for KernelSysctl {
|
|
||||||
if key_info.operation == "delete" {
|
|
||||||
warn!("Failed to delete kernel.sysctl config with key \"{}\"", key);
|
|
||||||
} else if !key_info_value.is_empty() && key_info.operation.is_empty() {
|
|
||||||
- fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes())
|
|
||||||
+ fs::write(&proc_path, format!("{}\n", &key_info_value).as_bytes())
|
|
||||||
.with_context(|| format!("Failed to write kernel.sysctl with key: \"{}\"", key))?;
|
|
||||||
- info!("Configured kernel.sysctl {}={}", key, key_info.value);
|
|
||||||
+ info!("Configured kernel.sysctl {}={}", key, key_info_value);
|
|
||||||
} else {
|
|
||||||
warn!(
|
|
||||||
"Failed to parse kernel.sysctl, key: \"{}\", value: \"{}\", operation: \"{}\"",
|
|
||||||
- key, key_info.value, key_info.operation
|
|
||||||
+ key, key_info_value, key_info.operation
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -529,7 +529,18 @@ impl Configuration for KubernetesKubelet {
|
|
||||||
if value_iter.is_null() {
|
|
||||||
*value_iter = serde_yaml::Value::Mapping(serde_yaml::Mapping::new());
|
|
||||||
}
|
|
||||||
- let value_mapping = value_iter.as_mapping_mut().unwrap();
|
|
||||||
+ let value_mapping = match value_iter.as_mapping_mut() {
|
|
||||||
+ Some(m) => m,
|
|
||||||
+ None => {
|
|
||||||
+ warn!(
|
|
||||||
+ "Failed to convert yaml value to mapping, maybe read the file in the wrong format,
|
|
||||||
+ or write wrong value when handle the configuration of key {}",
|
|
||||||
+ key
|
|
||||||
+ );
|
|
||||||
+ break;
|
|
||||||
+ },
|
|
||||||
+ };
|
|
||||||
+ info!("Add configuration \"{}: {}\"", key, key_info.value.clone());
|
|
||||||
value_mapping.insert(Value::String(k.to_string()).into(), config_value);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
@@ -651,7 +662,7 @@ impl Configuration for ContainerContainerd {
|
|
||||||
config_value = toml::Value::Table(value_tmp);
|
|
||||||
key_index = key_index - 1;
|
|
||||||
}
|
|
||||||
- debug!("Add key is {}, value is {:?}", key_list[i..].join("."), config_value);
|
|
||||||
+ info!("Add configuration \"{}: {}\"", key, key_info.value.clone());
|
|
||||||
value_iter.insert(k.to_string(), config_value);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
@@ -782,6 +793,7 @@ fn handle_add_key_pam_limits(new_configs: &HashMap<String, KeyInfo>) -> Vec<Stri
|
|
||||||
continue 'configs;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+ info!("Add configuration \"{} {}\"", key, new_value_list.join(" "));
|
|
||||||
configs_write.push(format!("{} {}", key, new_value_list.join(" ")));
|
|
||||||
}
|
|
||||||
configs_write
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,207 +0,0 @@
|
|||||||
From 50ac494ebf14fefbb0ad9d985e5497b90cd19821 Mon Sep 17 00:00:00 2001
|
|
||||||
From: whisky-ma <majie61@huawei.com>
|
|
||||||
Date: Wed, 27 Nov 2024 16:45:27 +0800
|
|
||||||
Subject: [PATCH 07/11] docs: add dm-verity user-guide
|
|
||||||
|
|
||||||
---
|
|
||||||
docs/user_guide/dm-verity.md | 187 +++++++++++++++++++++++++++++++++++
|
|
||||||
1 file changed, 187 insertions(+)
|
|
||||||
create mode 100644 docs/user_guide/dm-verity.md
|
|
||||||
|
|
||||||
diff --git a/docs/user_guide/dm-verity.md b/docs/user_guide/dm-verity.md
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..224c8650
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/docs/user_guide/dm-verity.md
|
|
||||||
@@ -0,0 +1,187 @@
|
|
||||||
+## dm-verity功能介绍
|
|
||||||
+
|
|
||||||
+KubeOS基于[dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html)提供对根文件系统rootfs的完整性保护。Dm-verity对目标设备rootfs分成固定大小(4096)的块，每块分别计算hash得到第一层hash。第一层hash按照固定大小的块再次计算hash形成第二层。如此迭代，形成一棵hash树，最终得到roothash。示意如下
|
|
||||||
+```
|
|
||||||
+0层：rootfs block0 ... blockx ... blockx ..... blockn
|
|
||||||
+ | | | | |
|
|
||||||
+1层：对0层计算hash h1.0 ... h1.x ... h1.x ..... h1.n
|
|
||||||
+ \______________/ \__________________/ \__________/
|
|
||||||
+ | | |
|
|
||||||
+2层：对1层计算hash h2.0 h2.x h2.n
|
|
||||||
+ \ ............. ............/
|
|
||||||
+
|
|
||||||
+逐层计算hash \ ....... ..... /
|
|
||||||
+ \... .../
|
|
||||||
+ \ /
|
|
||||||
+root层 roothash
|
|
||||||
+
|
|
||||||
+```
|
|
||||||
+hash树除roothash外的中间节点hash作为元数据验证rootfs的完整性，验证时重新计算roothash，并与存储的初始roothash进行比对，一致则rootfs完整。因此，dm-verity的关键在于保证roothash的完整性。
|
|
||||||
+
|
|
||||||
+## dm-verity当前实现
|
|
||||||
+
|
|
||||||
+KubeOS当前基于dm-verity+安全启动实现对rootfs的完整性保护，安全启动用于保护roothash完整性。由于安全启动只支持UEFI模式启动，因此当前只支持UEFI启动的场景，不支持legacy启动场景。
|
|
||||||
+
|
|
||||||
+安全启动基于密码学签名机制实现信任传递。通过在BIOS中引入可信证书，BIOS验证shim的签名，shim验证grub签名，grub验证kernel签名，签名验证失败则启动失败，由此完成系统启动，保证启动内核完整性。为保护dm-verity的roothash的完整，此处扩展安全启动功能，通过在grub中导入可信公钥，实现grub对initramfs、grub.cfg的签名验证，roothash在制作镜像时写入grub.cfg，系统启动时从grub.cfg获取roothash作为对比基线，借助dm-verity实现对rootfs的完整性校验。制作镜像时，安全启动的根信任证书保存在boot分区EFI目录下，名称为`rsa4BIOS.der`，首次启动时需要将该证书导入UEFI固件中**PK Options**和**DB Options**，参考如下安全启动设置。
|
|
||||||
+
|
|
||||||
+安全启动需要用户自行生成证书及相关签名密钥，并设置口令保护签名私钥。此处不对口令做复杂性校验，建议包含大小写字母、数字等。主要涉及的口令有以下三个：
|
|
||||||
+* BIOS签名私钥口令（pesign签名数据库口令）：明文口令，用于安全启动中保护签名私钥的安全性。此签名私钥存于镜像制作服务器上，用于对shim、grub进行签名。每次制作镜像需要输入该口令，否则无法签名。
|
|
||||||
+* grub配置文件签名私钥口令：明文口令，用于保护grub配置文件签名私钥。grub中导入公钥，对应私钥存于镜像制作服务器上，用于对配置文件grub.cfg签名，签名私钥由该口令保护，每次制作镜像需要输入。
|
|
||||||
+* grub shell口令：明文口令，如果在镜像启动时要进入grub shell则需要输入该口令。
|
|
||||||
+
|
|
||||||
+dm-verity+安全启动采用双boot+root，此时磁盘分区对应如下：
|
|
||||||
+```
|
|
||||||
+ ----------------
|
|
||||||
+part1 | boot1 |
|
|
||||||
+ ----------------
|
|
||||||
+part2 | root1 | ---- Grub menuentry 'A'
|
|
||||||
+ ----------------
|
|
||||||
+part3 | hash1 |
|
|
||||||
+ ----------------
|
|
||||||
+part4 | boot2 |
|
|
||||||
+ ----------------
|
|
||||||
+part5 | root2 | ---- Grub menuentry 'B'
|
|
||||||
+ ----------------
|
|
||||||
+part6 | hash2 |
|
|
||||||
+ ----------------
|
|
||||||
+part7 | persist |
|
|
||||||
+ ----------------
|
|
||||||
+```
|
|
||||||
+其中boot分区包含启动文件，ROOT分区存放rootfs，hash分区存放hash元数据，用于验证root分区的完整性。
|
|
||||||
+
|
|
||||||
+当前实现说明如下：
|
|
||||||
+
|
|
||||||
+* dm-verity开启后不支持grub阶段加载mod，因此grub.cfg中`insmod xx`命令删除，使用dm-verity/grub.cfg文件替换
|
|
||||||
+* dm-verity+安全启动模式下，grub.cfg文件不支持修改。如需修改需要在镜像制作服务器上修改并重新签名
|
|
||||||
+* 当前dm-verity/grub.cfg中，menuentry选项默认设置第一个disk, virtio类型设备：`set root='hd0,gpt2'`, `root=/dev/vda2` or `root=/dev/vda5`
|
|
||||||
+* 开启dm-verity，用户可以配置`ESP/EFI/openEuler/grubenv`文件，实现部分配置grub环境变量（白名单形式），如启动项（从哪一个root启动）
|
|
||||||
+* 开启dm-verity，rootfs以只读方式挂载`/dev/mapper/kubeos-root`。当前dm-verity通过veritysetup工具实现，可以通过如下命令查看rootfs完整性状态
|
|
||||||
+```
|
|
||||||
+veritysetup status kubeos-root # 显示状态（verified）、目标数据设备、hash元数据设备、roothash
|
|
||||||
+
|
|
||||||
+veritysetup verify /dev/vda2 /dev/vda3 roothash --debug # status显示的数据设备、hash设备、roothash，验证成功Command successful
|
|
||||||
+```
|
|
||||||
+* 如果当前rootfs（如root1）验证失败，尝试从另一个rootfs（root2）启动，若均验证失败，则系统启动失败
|
|
||||||
+* 安全启动证书基于rsa签名，制作镜像时通过openssl生成自签名证书`rsa4BIOS.der`
|
|
||||||
+* 安全启动可以通过mokutil工具查看，`mokutil --sb`
|
|
||||||
+* 镜像制作服务器需要安装如下包
|
|
||||||
+```
|
|
||||||
+yum install -y pesign nss openssl veritysetup crypto-policies
|
|
||||||
+```
|
|
||||||
+* **密钥管理** 当前第一次开启dm-verity功能，镜像制作服务器会自动生成相关证书和密钥文件，默认位置为`my/path/to/KubeOS/scripts/dm-verity/keys`，密钥均由口令保护。不建议频繁更换密钥、证书文件，否则系统可能无法正常启动（签名验证失败）。密钥、证书生成过程见**附录**。
|
|
||||||
+ - 安全启动依赖证书`rsa4BIOS.der`，每次更新需要进入BIOS重新导入证书（导入DB Options），否则系统无法启动，不建议频繁更新此证书。可选地，安全启动可以通过BIOS直接关闭
|
|
||||||
+ - roothash完整性验证依赖`grub配置文件签名公钥`，该公钥在镜像制作时导入grub，对应私钥用于grub.cfg签名，为防止系统启动失败，不建议频繁更换。此处验签功能可以通过进入grub-shell（需要上述grub shell 口令）进行关闭，输入`set check_signatures=no`, `configfile (hd0,1 or 4)/EFI/openEuler/grub.cfg`进入系统
|
|
||||||
+
|
|
||||||
+## 安全启动配置
|
|
||||||
+
|
|
||||||
+
|
|
||||||
+KubeOS支持在**虚拟机**镜像制作时开启dm-verity+安全启动配置。以下介绍以HOST侧操作系统为openEuler系统为例，介绍KubeOS虚机安全启动配置步骤，参考[openEuler虚机安全启动介绍](https://docs.openeuler.org/zh/docs/24.03_LTS/docs/Virtualization/%E7%AE%A1%E7%90%86%E8%99%9A%E6%8B%9F%E6%9C%BA.html)。对于其他操作系统，参考相应安全启动配置（配置文件名称、路径、依赖等有差异）。
|
|
||||||
+
|
|
||||||
+**xml文件修改**
|
|
||||||
+
|
|
||||||
+虚拟机安全启动依赖于UEFI BIOS的实现，HOST侧需要安装edk2。 以aarch64为例，需安装`yum install -y edk2-aarch64`， edk2 rpm包中的组件安装于/usr/share/edk2/aarch64目录下，包括QEMU_EFI-pflash.raw和vars-template-pflash.raw。虚拟机启动UEFI BIOS部分xml配置如下：
|
|
||||||
+```
|
|
||||||
+<os>
|
|
||||||
+ <type arch='aarch64' machine='virt'>hvm</type>
|
|
||||||
+ <loader readonly='yes' type='pflash'>/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw</loader>
|
|
||||||
+ <nvram template='/usr/share/edk2/aarch64/vars-template-pflash.raw'>/path/to/QEMU-VARS.fd</nvram>
|
|
||||||
+</os>
|
|
||||||
+```
|
|
||||||
+其中/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw为UEFI BIOS镜像路径。/usr/share/edk2/aarch64/vars-template-pflash.raw为nvram镜像模板路径，/path/to/QEMU-VARS.fd为当前虚拟机nvram镜像文件路径，用于保存UEFI BIOS系统中的环境变量。
|
|
||||||
+
|
|
||||||
+X86架构略有差异，需安装`yum install edk2-ovmf`，xml示例如下
|
|
||||||
+```
|
|
||||||
+<os>
|
|
||||||
+ <type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
|
|
||||||
+ <loader type='pflash'>/usr/share/edk2/ovmf/OVMF_CODE.fd</loader>
|
|
||||||
+ <nvram template='/usr/share/edk2/ovmf/OVMF_VARS.fd'>/path/to/OVMF_VARS.fd</nvram>
|
|
||||||
+</os>
|
|
||||||
+```
|
|
||||||
+
|
|
||||||
+**BIOS导入证书文件**
|
|
||||||
+
|
|
||||||
+当前实现中，制作KubeOS镜像时通过openssl生成自签名证书`rsa4BIOS.der`，证书文件存在BOOT分区EFI目录下。
|
|
||||||
+虚拟机启动后，点击`F2`进入BIOS界面，配置路径如下
|
|
||||||
+```
|
|
||||||
+Device Manager
|
|
||||||
+ -> Secure Boot Configuration
|
|
||||||
+ -> Secure Boot Mode
|
|
||||||
+ -> Custom Mode
|
|
||||||
+ -> Custom Secure Boot Option
|
|
||||||
+ -> PK Options
|
|
||||||
+ -> Enroll PK
|
|
||||||
+ -> Enroll PK Using File
|
|
||||||
+ -> BOOT / EFI / rsa4BIOS.der
|
|
||||||
+ -> DB Options
|
|
||||||
+ -> Enroll Signature
|
|
||||||
+ -> Enroll Signature Using File
|
|
||||||
+ -> BOOT / EFI / rsa4BIOS.der
|
|
||||||
+```
|
|
||||||
+证书导入完成点击`F10`保存修改，执行`reset`，完成系统重置。
|
|
||||||
+
|
|
||||||
+## dm-verity升级&回滚
|
|
||||||
+
|
|
||||||
+开启dm-verity功能，升级通过`dd`命令将升级镜像导入对应分区。注意事项如下：
|
|
||||||
+
|
|
||||||
+* 升级前后root分区的LABEL相同，均为`ROOT-A`
|
|
||||||
+* 升级失败回滚
|
|
||||||
+ - boot分区故障，无需手动操作，自动尝试另一个boot分区进行引导
|
|
||||||
+ - root分区故障，需要手动切换另一个root分区，假如升级到A失败需要回滚到B，则手动选择从B启动，系统会重启2次：第一次手动选取B，第二次无需手动操作，系统自动选取B完成回滚
|
|
||||||
+
|
|
||||||
+## 附录: 生成密钥/证书
|
|
||||||
+
|
|
||||||
+KubeOS提供密钥、证书生成脚本支持用户生成自定义密钥、证书文件。当前实现基于RSA密码算法，国密SM算法支持可参考[这里](https://docs.openeuler.org/zh/docs/23.03/docs/ShangMi/%E5%AE%89%E5%85%A8%E5%90%AF%E5%8A%A8.html)，密钥生成过程如下：
|
|
||||||
+
|
|
||||||
+```
|
|
||||||
+ # 准备密钥目录
|
|
||||||
+ KEYDIR="my/path/to/keys"
|
|
||||||
+ CERTDB="$KEYDIR/certdb"
|
|
||||||
+ BIOSkeyname="rsa4BIOS"
|
|
||||||
+ PIN_PASSWORD="foo"
|
|
||||||
+ keyname="$BIOSkeyname"
|
|
||||||
+
|
|
||||||
+ # 生成RSA密钥、证书，其中PIN_PASSWORD为BIOS签名私钥口令（pesign签名数据库口令）
|
|
||||||
+ mkdir -p "${CERTDB}"
|
|
||||||
+ cat > "${KEYDIR}/pinfile" << EOF
|
|
||||||
+$PIN_PASSWORD
|
|
||||||
+EOF
|
|
||||||
+
|
|
||||||
+ openssl genrsa -out "${KEYDIR}/${keyname}.key" 4096
|
|
||||||
+ openssl req -new -key "${KEYDIR}/${keyname}.key" -out "${KEYDIR}/${keyname}.csr" -subj '/C=AA/ST=BB/O=CC/OU=DD/CN=BIOS-cert-for-kubeos-secure-boot'
|
|
||||||
+ openssl x509 -req -days 365 -in "${KEYDIR}/${keyname}.csr" -signkey "${KEYDIR}/${keyname}.key" -out "${KEYDIR}/${keyname}.crt"
|
|
||||||
+ openssl x509 -in "${KEYDIR}/${keyname}.crt" -out "${KEYDIR}/${keyname}.der" -outform der
|
|
||||||
+
|
|
||||||
+ # 创建pesign签名数据库
|
|
||||||
+ certutil -N -d "${CERTDB}" -f "${KEYDIR}/pinfile"
|
|
||||||
+ certutil -A -n ${keyname} -d "${CERTDB}" -t CT,CT,CT -i "${KEYDIR}/${keyname}.crt" -f "${KEYDIR}/pinfile"
|
|
||||||
+ openssl pkcs12 -export -out "${KEYDIR}/${keyname}.p12" -inkey "${KEYDIR}/${keyname}.key" -in "${KEYDIR}/${keyname}.crt" -password pass:"${PIN_PASSWORD}"
|
|
||||||
+ pk12util -d "${CERTDB}" -i "${KEYDIR}/${keyname}.p12" -w "${KEYDIR}/pinfile" -k "${KEYDIR}/pinfile"
|
|
||||||
+
|
|
||||||
+ # 签名示例，对shimx64.efi签名
|
|
||||||
+ SHIM="my/path/to/shimx64.efi"
|
|
||||||
+ pesign -n "${CERTDB}" -c ${BIOSkeyname} --pinfile "${KEYDIR}/pinfile" -s -i "$SHIM" -o "${SHIM}.signed"
|
|
||||||
+
|
|
||||||
+
|
|
||||||
+ # 生成GPG签名密钥，用于对配置文件grub.cfg签名，其中GPG_PASSWORD为grub配置文件签名密钥保护口令
|
|
||||||
+ GPG_PASSWORD="foo"
|
|
||||||
+ GPGkeyid="gpgKey4kubeos"
|
|
||||||
+ cat > "${KEYDIR}/gpg.batch.file" << EOF
|
|
||||||
+Key-Type: RSA
|
|
||||||
+Key-Length: 4096
|
|
||||||
+Subkey-Type: RSA
|
|
||||||
+Subkey-Length: 4096
|
|
||||||
+Name-Real: ${GPGkeyid}
|
|
||||||
+Expire-Date: 0
|
|
||||||
+Passphrase: ${GPG_PASSWORD}
|
|
||||||
+EOF
|
|
||||||
+
|
|
||||||
+ gpg --batch --gen-key "${KEYDIR}/gpg.batch.file"
|
|
||||||
+ gpg --list-keys --keyid-format LONG ${GPGkeyid} | grep pub > "${KEYDIR}/gpg.log"
|
|
||||||
+ GPG_KEY=$(gpg --list-keys --keyid-format LONG ${GPGkeyid} | grep pub | awk -F 'rsa4096/' '{print $2}' | cut -b 1-16)
|
|
||||||
+ gpg --export "$GPG_KEY" > "${KEYDIR}/gpg.key"
|
|
||||||
+
|
|
||||||
+ # 签名示例
|
|
||||||
+ GRUB_CFG="my/path/to/grub.cfg"
|
|
||||||
+ gpg --pinentry-mode=loopback --passphrase "${GPG_PASSWORD}" --default-key "$GPG_KEY" --detach-sign "${GRUB_CFG}"
|
|
||||||
+```
|
|
||||||
+注意：密钥/证书文件生成后应及时删除口令和私钥文件
|
|
||||||
\ No newline at end of file
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,140 +0,0 @@
|
|||||||
From 2c31929f3f32cfcd7db422fdbab66c75b5ad9175 Mon Sep 17 00:00:00 2001
|
|
||||||
From: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
Date: Thu, 28 Nov 2024 10:46:03 +0800
|
|
||||||
Subject: [PATCH 08/11] docs: update readme
|
|
||||||
|
|
||||||
Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
---
|
|
||||||
README.md | 117 +++++++++++++++++++++++++++++++++++++++++++++++++-----
|
|
||||||
1 file changed, 108 insertions(+), 9 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/README.md b/README.md
|
|
||||||
index 7f325623..cb17d89e 100644
|
|
||||||
--- a/README.md
|
|
||||||
+++ b/README.md
|
|
||||||
@@ -1,14 +1,113 @@
|
|
||||||
# KubeOS
|
|
||||||
-## Introduction
|
|
||||||
-KubeOS 是针对业务以容器的形式运行的场景，专门设计的一种轻量级操作系统。KubeOS 通过 kubernetes CRD + operator 扩展机制将将 OS 作为组件接入 kubernetes，使 OS 和业务处于同等地位，用户通过 kubernetes 集群统一管理节点上的容器和节点 OS，实现一套系统管理容器和 OS。
|
|
||||||
-## Architecture
|
|
||||||
-KubeOS架构的介绍请见： [architecture](docs/design/architecture.md)
|
|
||||||
-## Getting Started
|
|
||||||
-### Build from source and deploy
|
|
||||||
-从源码构建指南请见： [quick-start.md](docs/quick-start.md).
|
|
||||||
-### User Guide
|
|
||||||
-用户指南请见：[user guide](https://docs.openeuler.org/zh/docs/22.03_LTS_SP1/docs/KubeOS/overview.html)
|
|
||||||
+
|
|
||||||
+在云计算场景中，容器和 Kubernetes 的应用日益广泛。然而，当前容器和操作系统（OS）独立管理的方式常常面临功能冗余，以及两套调度系统难以协同的问题。此外，OS 的版本管理也存在诸多挑战：相同版本的 OS 在使用过程中会因安装、更新或删除不同的软件包而逐渐产生差异，导致版本分裂。同时，OS 与业务紧耦合，造成大版本升级难度较高，进一步增加了运维复杂性。
|
|
||||||
+
|
|
||||||
+**KubeOS** 是专为以容器形式运行业务的场景设计的一种轻量级操作系统。通过 Kubernetes 的 CRD（自定义资源定义）与 Operator 扩展机制，KubeOS 将 OS 作为 Kubernetes 的一个组件接入，使 OS 和业务处于同等地位。用户可通过 Kubernetes 集群统一管理节点上的容器和节点的 OS，从而实现一套系统同时管理容器与 OS。
|
|
||||||
+
|
|
||||||
+KubeOS 的组件利用 Kubernetes Operator 扩展机制来控制容器 OS 的升级流程，支持对 KubeOS 的整体原子化升级。这种升级方式在升级前会将业务迁移到未升级的节点上，从而最大程度减少升级和配置过程对业务的影响。同时，通过原子化升级保持 OS 始终与预期状态同步，确保集群内 OS 的版本一致性，有效避免版本分裂问题。
|
|
||||||
+
|
|
||||||
+
|
|
||||||
+
|
|
||||||
+以下链接可协助您使用KubeOS:
|
|
||||||
+
|
|
||||||
+* [KubeOS组件构建指南](docs/quick-start.md)涵盖了从编译、制作和部署KubeOS组件的全流程。
|
|
||||||
+* [镜像制作指导](docs/user_guide/KubeOS镜像制作指导-binary.md)包含了KubeOS镜像制作工具的详细使用方法。
|
|
||||||
+* [KubeOS架构](docs/design/architecture.md)展示了其文件系统的设计理念及核心组件的详细信息。
|
|
||||||
+* [用户指南](https://docs.openeuler.org/zh/docs/24.03_LTS/docs/KubeOS/overview.html)链接至openEuler容器OS文档。
|
|
||||||
+
|
|
||||||
+## 特性介绍
|
|
||||||
+
|
|
||||||
+### 升级
|
|
||||||
+
|
|
||||||
+与传统的包管理器逐一升级软件包不同，KubeOS 在升级时会通过预制的完整根分区文件系统镜像实现全量升级。升级流程包括从 HTTP 服务器或容器镜像仓库下载升级镜像，并将新的根分区文件系统覆盖到备用的 root 分区。随后，节点从备用 root 分区启动，完成操作系统的全量升级。
|
|
||||||
+
|
|
||||||
+此外，KubeOS 支持一键回滚至上一版本的操作系统。通过切换至备用分区启动，快速恢复节点的 OS 状态。
|
|
||||||
+
|
|
||||||
+* **os-operator**: 部署在 Master 节点的 OS 自定义资源控制器，负责管理集群内所有节点 OS 的升级、回滚以及配置请求的下发。
|
|
||||||
+* **os-proxy**: 部署在每个节点上的 OS 控制器，将针对该节点的升级、回滚和配置请求转发至 os-agent。
|
|
||||||
+* **os-agent**: 部署在每个节点 OS 的 systemd 服务，负责执行具体的升级、回滚和配置任务。
|
|
||||||
+
|
|
||||||
+更多详细信息，可见[升级指导](docs/quick-start.md#升级指导)。
|
|
||||||
+
|
|
||||||
+### 配置
|
|
||||||
+
|
|
||||||
+KubeOS 通过 Kubernetes 下发 OS 自定义资源，实现对集群内所有容器 OS 的统一配置管理。目前支持以下配置类型：
|
|
||||||
+
|
|
||||||
+* 内核参数（临时/持久化）
|
|
||||||
+* 内核启动参数
|
|
||||||
+* pam_limits
|
|
||||||
+* KubeletConfiguration
|
|
||||||
+* containerd
|
|
||||||
+
|
|
||||||
+更多详细信息，可见[配置指导](docs/quick-start.md#配置（Settings）指导)
|
|
||||||
+
|
|
||||||
+### Admin运维容器
|
|
||||||
+
|
|
||||||
+为了保持系统的轻量化，KubeOS 可不安装 SSH 服务（sshd）。在必要情况下，管理员可以通过部署 Admin 容器到目标节点，SSH 登录到容器内，再切换到节点主机命名空间，完成运维操作。
|
|
||||||
+
|
|
||||||
+Admin 容器内可以安装丰富的调试工具，从而在主机命名空间下调用容器内的命令完成调试和检测任务。
|
|
||||||
+
|
|
||||||
+更多详细信息，可见[Admin容器镜像制作部署和使用](docs/quick-start.md#admin容器镜像制作部署和使用)。
|
|
||||||
+
|
|
||||||
+### dm-verity静态完整性保护
|
|
||||||
+
|
|
||||||
+KubeOS当前基于[dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html)+安全启动实现对rootfs的完整性保护，安全启动用于保护roothash完整性。
|
|
||||||
+
|
|
||||||
+更多详细信息，可见[dm-verity介绍](docs/user_guide/dm-verity.md)。
|
|
||||||
+
|
|
||||||
+### 镜像制作
|
|
||||||
+
|
|
||||||
+KubeOS 支持制作多种类型的镜像，包括：
|
|
||||||
+
|
|
||||||
+* 普通虚拟机镜像
|
|
||||||
+* PXE 物理机镜像
|
|
||||||
+* 升级容器镜像
|
|
||||||
+* 启用 dm-verity 特性的虚拟机镜像
|
|
||||||
+
|
|
||||||
+目前支持 x86 和 aarch64 架构，默认使用UEFI引导模式，部分支持 legacy 引导模式。
|
|
||||||
+
|
|
||||||
+您可以通过以下命令创建 KubeOS 镜像：
|
|
||||||
+
|
|
||||||
+```bash
|
|
||||||
+make rust-agent
|
|
||||||
+cargo run --package kbimg -- create -f KubeOS-Rust/kbimg/kbimg.toml <IMG-TYPE>
|
|
||||||
+```
|
|
||||||
+
|
|
||||||
+更多详细信息，可见[镜像制作指导](docs/user_guide/KubeOS镜像制作指导-binary.md)
|
|
||||||
+
|
|
||||||
+## Roadmap
|
|
||||||
+
|
|
||||||
+### 即将到来
|
|
||||||
+
|
|
||||||
+* **2025**:
|
|
||||||
+ * [ ] Pod热迁移：提供用户无感、业务中断时间短的Pod热迁移解决方案
|
|
||||||
+
|
|
||||||
+### 当前进展
|
|
||||||
+
|
|
||||||
+* **2024**:
|
|
||||||
+ * [x] 灵活、多维度的运维策略：分组分批次升级，基于时间窗升级的升级策略
|
|
||||||
+ * [x] KubeOS 镜像定制化制作：支持用户根据需求制作定制化镜像
|
|
||||||
+ * [x] 更强安全能力：支持安全启动和dm-verity
|
|
||||||
+ * [x] 更丰富的配置管理：支持统一管理节点`containerd`和`kubelet`配置
|
|
||||||
+
|
|
||||||
+* **2023**:
|
|
||||||
+ * [x] 支持容器镜像(containerd)升级
|
|
||||||
+ * [x] 新增settings配置功能
|
|
||||||
+ * [x] 新增Admin容器功能
|
|
||||||
+ * [x] 内存底噪优化：降低os-proxy和os-agent内存底噪80%
|
|
||||||
+
|
|
||||||
+* **2022**:
|
|
||||||
+ * [x] 支持物理机安装、升级
|
|
||||||
+ * [x] 支持容器镜像(docker)升级
|
|
||||||
+
|
|
||||||
+* **2021**:
|
|
||||||
+ * [x] KubeOS发布
|
|
||||||
+ * [x] 支持ARM架构
|
|
||||||
+
|
|
||||||
## How to Contribute
|
|
||||||
+
|
|
||||||
我们非常欢迎新贡献者加入到项目中来，也非常高兴能为新加入贡献者提供指导和帮助。您可以通过issue或者合入PR来贡献
|
|
||||||
+
|
|
||||||
## Licensing
|
|
||||||
+
|
|
||||||
KubeOS 使用 Mulan PSL v2.
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,26 +0,0 @@
|
|||||||
From 6760d1adc236ccbe2d5bb1ae6a12087a8eb82d90 Mon Sep 17 00:00:00 2001
|
|
||||||
From: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
Date: Thu, 28 Nov 2024 10:08:25 +0000
|
|
||||||
Subject: [PATCH 09/11] fix(kbimg): exit with error code on image creation
|
|
||||||
failure
|
|
||||||
|
|
||||||
Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
---
|
|
||||||
KubeOS-Rust/kbimg/src/main.rs | 1 +
|
|
||||||
1 file changed, 1 insertion(+)
|
|
||||||
|
|
||||||
diff --git a/KubeOS-Rust/kbimg/src/main.rs b/KubeOS-Rust/kbimg/src/main.rs
|
|
||||||
index 3fc68cb1..8639f81f 100644
|
|
||||||
--- a/KubeOS-Rust/kbimg/src/main.rs
|
|
||||||
+++ b/KubeOS-Rust/kbimg/src/main.rs
|
|
||||||
@@ -130,6 +130,7 @@ fn main() {
|
|
||||||
},
|
|
||||||
Err(e) => {
|
|
||||||
error!("Failed to create image: {:?}", e);
|
|
||||||
+ exit(1);
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,26 +0,0 @@
|
|||||||
From 6f0357c10320202d62ecca286b39a9f7fc80baa2 Mon Sep 17 00:00:00 2001
|
|
||||||
From: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
Date: Tue, 3 Dec 2024 10:44:41 +0800
|
|
||||||
Subject: [PATCH 10/11] docs: update kbimg pxe guide
|
|
||||||
|
|
||||||
Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
---
|
|
||||||
...7\345\210\266\344\275\234\346\214\207\345\257\274-binary.md" | 2 ++
|
|
||||||
1 file changed, 2 insertions(+)
|
|
||||||
|
|
||||||
diff --git "a/docs/user_guide/KubeOS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274-binary.md" "b/docs/user_guide/KubeOS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274-binary.md"
|
|
||||||
index b951b388..d01a41ce 100644
|
|
||||||
--- "a/docs/user_guide/KubeOS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274-binary.md"
|
|
||||||
+++ "b/docs/user_guide/KubeOS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274-binary.md"
|
|
||||||
@@ -246,6 +246,8 @@ version = "v1"
|
|
||||||
|
|
||||||
* 支持CPU 架构为 x86 和 aarch64 的物理机场景，不支持legacy引导模式
|
|
||||||
* PXE物理机镜像制作不支持dm-verity功能
|
|
||||||
+* 使用默认的 rpmlist 进行镜像制作时，所需磁盘空间至少为 5GB。如果使用自定义的 rpmlist，可能需要超过 5GB 的磁盘空间。
|
|
||||||
+* 在 PXE 安装阶段，需要从 HTTP 服务器下载根分区 tar 包。请确保机器拥有足够的内存空间以存储根分区 tar 包及临时中间文件。
|
|
||||||
* 首先需要修改```kbimg.toml```中```pxe_config```的配置，对相关参数进行配置，详细参数可见[参数说明](#pxe_config)，ip目前仅支持ipv4，配置示例如下
|
|
||||||
|
|
||||||
```toml
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
@ -1,30 +0,0 @@
|
|||||||
From d882aa4202dec390cb8b9683e974b9705b4c19eb Mon Sep 17 00:00:00 2001
|
|
||||||
From: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
Date: Tue, 3 Dec 2024 11:01:56 +0800
|
|
||||||
Subject: [PATCH 11/11] fix(kbimg): prevent concurrent execution by checking a
|
|
||||||
lock file
|
|
||||||
|
|
||||||
Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
|
|
||||||
---
|
|
||||||
KubeOS-Rust/kbimg/src/main.rs | 5 +++++
|
|
||||||
1 file changed, 5 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/KubeOS-Rust/kbimg/src/main.rs b/KubeOS-Rust/kbimg/src/main.rs
|
|
||||||
index 8639f81f..43e9d422 100644
|
|
||||||
--- a/KubeOS-Rust/kbimg/src/main.rs
|
|
||||||
+++ b/KubeOS-Rust/kbimg/src/main.rs
|
|
||||||
@@ -40,6 +40,11 @@ trait CreateImage {
|
|
||||||
|
|
||||||
fn process(info: Box<dyn CreateImage>, mut config: Config, debug: bool) -> Result<()> {
|
|
||||||
let dir = PathBuf::from(SCRIPTS_DIR);
|
|
||||||
+ let lock = dir.join("test.lock");
|
|
||||||
+ if lock.exists() {
|
|
||||||
+ error!("It looks like another kbimg process is running. Please wait it to finish.");
|
|
||||||
+ exit(1);
|
|
||||||
+ }
|
|
||||||
if dir.exists() {
|
|
||||||
debug!("Removing existing scripts directory");
|
|
||||||
fs::remove_dir_all(&dir)?;
|
|
||||||
--
|
|
||||||
2.39.5 (Apple Git-154)
|
|
||||||
|
|
||||||
21
KubeOS.spec
21
KubeOS.spec
@ -1,22 +1,11 @@
|
|||||||
# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved.
|
# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved.
|
||||||
|
|
||||||
Name: KubeOS
|
Name: KubeOS
|
||||||
Version: 1.0.7
|
Version: 1.0.8
|
||||||
Release: 2
|
Release: 1
|
||||||
Summary: O&M platform used to update the whole OS as an entirety
|
Summary: O&M platform used to update the whole OS as an entirety
|
||||||
License: Mulan PSL v2
|
License: Mulan PSL v2
|
||||||
Source0: https://gitee.com/openeuler/KubeOS/repository/archive/v%{version}.tar.gz
|
Source0: https://gitee.com/openeuler/KubeOS/repository/archive/v%{version}.tar.gz
|
||||||
Patch1: 0001-fix-Fix-some-spelling-errors-in-docs.patch
|
|
||||||
Patch2: 0002-docs-update-kbimg-docs.patch
|
|
||||||
Patch3: 0003-fix-os-agent-use-findmnt-to-find-out-which-device-mo.patch
|
|
||||||
Patch4: 0004-style-format-code.patch
|
|
||||||
Patch5: 0005-docs-add-description-of-the-configuration.patch
|
|
||||||
Patch6: 0006-fix-os-agent-fix-settings-of-kubelet-and-add-log.patch
|
|
||||||
Patch7: 0007-docs-add-dm-verity-user-guide.patch
|
|
||||||
Patch8: 0008-docs-update-readme.patch
|
|
||||||
Patch9: 0009-fix-kbimg-exit-with-error-code-on-image-creation-fai.patch
|
|
||||||
Patch10: 0010-docs-update-kbimg-pxe-guide.patch
|
|
||||||
Patch11: 0011-fix-kbimg-prevent-concurrent-execution-by-checking-a.patch
|
|
||||||
|
|
||||||
BuildRoot: %{_tmppath}/%{name}-%{version}-build
|
BuildRoot: %{_tmppath}/%{name}-%{version}-build
|
||||||
BuildRequires: make rust cargo openssl-devel
|
BuildRequires: make rust cargo openssl-devel
|
||||||
@ -81,6 +70,12 @@ install -p -m 0600 ./KubeOS-Rust/kbimg/kbimg.toml %{buildroot}/opt/kubeOS/script
|
|||||||
rm -rfv %{buildroot}
|
rm -rfv %{buildroot}
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
|
* Tue Dec 10 2024 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.8-1
|
||||||
|
- Type:requirement
|
||||||
|
- CVE:NA
|
||||||
|
- SUG:restart
|
||||||
|
- DESC:update version to 1.0.8
|
||||||
|
|
||||||
* Tue Dec 03 2024 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.7-2
|
* Tue Dec 03 2024 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.7-2
|
||||||
- Type:requirement
|
- Type:requirement
|
||||||
- CVE:NA
|
- CVE:NA
|
||||||
|
|||||||
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user