20230623esxi自动创建虚机+安装ubuntu

0.背景

做实验需要反复创建虚机装系统，略麻烦。打算给家里云做个低配版的虚机自动化管理

1.方案调研

esxi本身有接口和SDK可以给各种语言做自动化对接，从官方github就能找到不少资料

ubuntu自动安装也比较成熟，常见的方案都是使用cloud-init来提供user-data，可以参考官方文档试试

2.虚机自动创建

使用govc命令行即可，脚本参考https://github.com/vmware/govmomi/blob/main/scripts/vcsa/create-esxi-vm.sh 去掉了对我没用的部分并增加了几个参数方便自己使用，完整文档请参考https://github.com/vmware/govmomi/tree/main/govc#usage


#!/bin/bash -e
# Create a VM and boot vm from cdrom/iso, modified from https://github.com/vmware/govmomi/blob/main/scripts/vcsa/create-esxi-vm.sh
# GOVC_* environment variables also apply, see https://github.com/vmware/govmomi/tree/main/govc#usage
# If GOVC_USERNAME is set, it is used to login ESXi
# If GOVC_PASSWORD is set, the account password will be set to this value
# 6.7 U3 https://docs.vmware.com/en/VMware-vSphere/6.7/rn/vsphere-esxi-vcenter-server-67-release-notes.html
export GOVC_URL=1.2.3.4 #esxi的IP
export GOVC_INSECURE=1
export GOVC_USERNAME=ESXI_USER #esxi用户名
export GOVC_PASSWORD=ESXI_PASSWORD #esxi密码
export GOVC_DATASTORE=datastore1
export GOVC_NETWORK=ESXI_NET#默认交换机网络，我用的是test
#. ~/.govcrc
set -o pipefail
usage() {
  cat <<'EOF'
Usage: $0 [-c CORE] [-m MEM_GB] [-d DISK_GB] [-t DISK_THICK] [-i ISO] [-p POWER_STATE] VM_NAME
Example1: $0 vmname
Example2: $0 -c 2 -m 2 -d 40 -t true -i ubuntu-22.04.1-live-server-amd64.iso -p on vmname
EOF
}

core=2
mem=2
disk=40
thick=true
iso=ubuntu-22.04.1-live-server-amd64.iso
power=off
while getopts c:m:d:t:i:p:h flag
do
  echo "flag=$flag"
  echo "val=$OPTARG"
  case $flag in
    c)
      core=$OPTARG
      ;;
    m)
      mem=$OPTARG
      ;;
    d)
      disk=$OPTARG
      ;;
    t)
      thick=$OPTARG
      ;;
    i)
      iso=$OPTARG
      ;;
    p)
      power=$OPTARG
      ;;
    h)
      usage
      exit
      ;;
    *)
      usage 1>&2
      exit 1
      ;;
  esac
done

shift $((OPTIND-1))

if [ $# -ne 1 ] ; then
  usage
  exit 1
fi

# if [[ "$iso" == *"-Installer-"* ]] ; then
#   echo "Invalid iso name (need stateless, not installer): $iso" 1>&2
#   exit 1
# fi
network=${GOVC_NETWORK:-"VM Network"}
username=$GOVC_USERNAME
password=$GOVC_PASSWORD
guest=${GUEST:-"ubuntu64Guest"}
name=$1
echo -n "Checking govc version..."
govc version -require 0.15.0
#vm
echo "Creating vm ${name}..."
govc vm.create -version 6.7 -on=false -net "$network" -m $((mem*1024)) -c $core -g "$guest" -net.adapter=vmxnet3 -disk.controller pvscsi "$name"
#cdrom
echo "Adding cdrom device to ${name}..."
id=$(govc device.cdrom.add -vm "$name")
boot="sysimg/$iso"
# upload iso
# if ! govc datastore.ls "$boot" > /dev/null 2>&1 ; then
#   govc datastore.upload "$iso" "$boot"
# fi
echo "Inserting $boot into $name cdrom device..."
govc device.cdrom.insert -vm "$name" -device "$id" "$boot"
#disk
echo "Creating disk for use by $name..."
diskname=$name
govc vm.disk.create -vm "$name" -name "$name"/"$diskname" -size "${disk}G" -thick=$thick
#change boot seq
echo "Change bios boot seq for $name..."
govc device.boot -vm "$name" -order disk,ethernet,cdrom
#get mac address
echo "Powering on $name VM..."
govc vm.power -on "$name"
mac=""
while [ x$mac = x"" ] || [ x$mac = x"null" ]
do
  mac=$(govc vm.info -json $name | jq -r ".VirtualMachines[0].Config.Hardware.Device[]|select(.Backing.DeviceName == \"$network\")|.MacAddress")
  sleep 1
done
echo "Get vm macaddress succ, $mac"
#power on
if [ x$power = x"on" ] ; then
    echo "Waiting for $name IP..."
    vm_ip=$(govc vm.ip "$name")
    ! govc events -n 100 "vm/$name" | grep -E 'warning|error'
    echo "$name IP get succ"
    echo $vm_ip
    echo "Create VM Done: $name, $vm_ip"
else
    govc vm.power -off "$name"
    echo "Create VM Done: $name"
    echo "You can power on it by:"
    echo govc vm.power -on "$name"
fi
exit 0

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

#!/bin/bash -e

# Create a VM and boot vm from cdrom/iso, modified from https://github.com/vmware/govmomi/blob/main/scripts/vcsa/create-esxi-vm.sh

# GOVC_* environment variables also apply, see https://github.com/vmware/govmomi/tree/main/govc#usage

# If GOVC_USERNAME is set, it is used to login ESXi

# If GOVC_PASSWORD is set, the account password will be set to this value

# 6.7 U3 https://docs.vmware.com/en/VMware-vSphere/6.7/rn/vsphere-esxi-vcenter-server-67-release-notes.html

export GOVC_URL=1.2.3.4 #esxi的IP

export GOVC_INSECURE=1

export GOVC_USERNAME=ESXI_USER #esxi用户名

export GOVC_PASSWORD=ESXI_PASSWORD #esxi密码

export GOVC_DATASTORE=datastore1

export GOVC_NETWORK=ESXI_NET#默认交换机网络，我用的是test

#. ~/.govcrc

set -o pipefail

usage() {

cat <<'EOF'

Usage: $0 [-c CORE] [-m MEM_GB] [-d DISK_GB] [-t DISK_THICK] [-i ISO] [-p POWER_STATE] VM_NAME

Example1: $0 vmname

Example2: $0 -c 2 -m 2 -d 40 -t true -i ubuntu-22.04.1-live-server-amd64.iso -p on vmname

EOF

}

core=2

mem=2

disk=40

thick=true

iso=ubuntu-22.04.1-live-server-amd64.iso

power=off

while getopts c:m:d:t:i:p:h flag

echo "flag=$flag"

echo "val=$OPTARG"

case $flag in

core=$OPTARG

;;

mem=$OPTARG

;;

disk=$OPTARG

;;

thick=$OPTARG

;;

iso=$OPTARG

;;

power=$OPTARG

;;

usage

exit

;;

usage 1>&2

exit 1

;;

esac

done

shift $((OPTIND-1))

if [ $# -ne 1 ] ; then

usage

exit 1

# if [[ "$iso" == *"-Installer-"* ]] ; then

# echo "Invalid iso name (need stateless, not installer): $iso" 1>&2

# exit 1

# fi

network=${GOVC_NETWORK:-"VM Network"}

username=$GOVC_USERNAME

password=$GOVC_PASSWORD

guest=${GUEST:-"ubuntu64Guest"}

name=$1

echo -n "Checking govc version..."

govc version -require 0.15.0

#vm

echo "Creating vm ${name}..."

govc vm.create -version 6.7 -on=false -net "$network" -m $((mem*1024)) -c $core -g "$guest" -net.adapter=vmxnet3 -disk.controller pvscsi "$name"

#cdrom

echo "Adding cdrom device to ${name}..."

id=$(govc device.cdrom.add -vm "$name")

boot="sysimg/$iso"

# upload iso

# if ! govc datastore.ls "$boot" > /dev/null 2>&1 ; then

# govc datastore.upload "$iso" "$boot"

# fi

echo "Inserting $boot into $name cdrom device..."

govc device.cdrom.insert -vm "$name" -device "$id" "$boot"

#disk

echo "Creating disk for use by $name..."

diskname=$name

govc vm.disk.create -vm "$name" -name "$name"/"$diskname" -size "${disk}G" -thick=$thick

#change boot seq

echo "Change bios boot seq for $name..."

govc device.boot -vm "$name" -order disk,ethernet,cdrom

#get mac address

echo "Powering on $name VM..."

govc vm.power -on "$name"

mac=""

while [ x$mac = x"" ] || [ x$mac = x"null" ]

mac=$(govc vm.info -json $name | jq -r ".VirtualMachines[0].Config.Hardware.Device[]|select(.Backing.DeviceName == \"$network\")|.MacAddress")

sleep 1

done

echo "Get vm macaddress succ, $mac"

#power on

if [ x$power = x"on" ] ; then

echo "Waiting for $name IP..."

vm_ip=$(govc vm.ip "$name")

! govc events -n 100 "vm/$name" | grep -E 'warning|error'

echo "$name IP get succ"

echo $vm_ip

echo "Create VM Done: $name, $vm_ip"

else

govc vm.power -off "$name"

echo "Create VM Done: $name"

echo "You can power on it by:"

echo govc vm.power -on "$name"

exit 0

3.ubuntu自动安装

官方的cloud-image.ova我测了好几次都都没法用govc正常传递-options参数，json里指定的user-data并没有生效，不知道为什么，因此先尝试半自动手动修改引导参数测试

3.1.user-data准备

首先按官方文档准备一个目录，放入user-data和user-meta后启动http服务器

user-data内容如下，有这么几点需要注意

开头的第一行注释不要删
在22.04.1这个版本的ubuntu下，如果不加refresh-installer会导致lvm的sizing-policy失效，40G系统盘默认只会用20G。
disable_suites如果不加security会导致本地实验老去联网升级安全补丁，为了加快速度我就关了，实际生产环境不要关
packages如果指定了内容，package_update无论是什么都会更新一次源


#cloud-config
autoinstall:
  version: 1
  refresh-installer:
    update: yes
  identity:
    hostname: ubuntu1
    username: ubuntu
    password: "$6$exDY1mhS4KUYCE/2$zmn9ToZwTKLhCw.b4/b.ZRTIZM30JZ4QrOQ2aOXJ8yk96xpcCof0kxKwuX1kqLG/ygbJ1f8wxED22bTL4F46P0"
  package_upgrade: false
  package_update: true
  packages:
    - net-tools
  apt:
    disable_suites: [security]
    primary:
      - arches: [default]
        uri: https://mirrors.tuna.tsinghua.edu.cn/ubuntu/
  ssh:
    install-server: true
    allow-pw: false
    authorized-keys:
      - ssh-rsa 放你的ssh公钥
  storage:
    layout:
      name: lvm
      sizing-policy: all
  runcmd:
    - echo test

#cloud-config

autoinstall:

version: 1

refresh-installer:

update: yes

identity:

hostname: ubuntu1

username: ubuntu

password: "$6$exDY1mhS4KUYCE/2$zmn9ToZwTKLhCw.b4/b.ZRTIZM30JZ4QrOQ2aOXJ8yk96xpcCof0kxKwuX1kqLG/ygbJ1f8wxED22bTL4F46P0"

package_upgrade: false

package_update: true

packages:

- net-tools

apt:

disable_suites: [security]

primary:

- arches: [default]

uri: https://mirrors.tuna.tsinghua.edu.cn/ubuntu/

ssh:

install-server: true

allow-pw: false

authorized-keys:

- ssh-rsa 放你的ssh公钥

storage:

layout:

name: lvm

sizing-policy: all

runcmd:

- echo test

3.2.远程加载user-data

20.04.5的图形界面按F6和esc后可以编辑启动菜单的指令，改成这样就能从远程加载user-data了。

话说不知道为啥vmware粘贴多了会导致有概率内容丢失


/casper/vmlinuz initrd=/casper/initrd autoinstall ds=nocloud-net;s=http://192.168.2.159/ubuntu1/

0 1	/casper/vmlinuz initrd=/casper/initrd autoinstall ds=nocloud-net;s=http://192.168.2.159/ubuntu1/

22.04.1的那个图形界面没了，可以按c进入grub命令行，手动执行启动参数如下，注意nocloud-net后面的分号之前必须有反斜杠转义

vmware在这个命令行下的粘贴完全没问题


set gfxpayload=keep;linux /casper/vmlinuz autoinstall ds=nocloud-net\;s=http://192.168.2.159/ubuntu1/ ---;initrd /casper/initrd;boot

0 1	set gfxpayload=keep;linux /casper/vmlinuz autoinstall ds=nocloud-net\;s=http://192.168.2.159/ubuntu1/ ---;initrd /casper/initrd;boot

3.3.从网络启动并自定义内核启动参数

1.设置openwrt当DHCP server指定 tftp服务

https://openwrt.org/docs/guide-user/services/tftp.pxe-server

https://forum.openwrt.org/t/solved-dhcp-config-pxe-boot-from-external-tftp-server/5880

配置如下图

2.tftp由op、jumpbox、群晖负责（群晖的tftp测试失败，疑似是因为我的群晖不在测试网段，暂时使用op自带的tftp）

tftp内容制作如下，参考openwrt的文档修改而成。我使用的是BIOS legency引导，示例如下。


cd autoinstall
mkdir tftp
cd tftp
mkdir syslinux
cd syslinux
wget --no-check-certificate https://www.kernel.org/pub/linux/utils/boot/syslinux/syslinux-6.03.tar.gz
tar -zxvf syslinux-6.03.tar.gz
cd syslinux-6.03/bios
cp core/pxelinux.0 com32/elflink/ldlinux/ldlinux.c32 com32/menu/vesamenu.c32 com32/lib/libcom32.c32 com32/libutil/libutil.c32 ../../../../tftp

DEFAULT vesamenu.c32
PROMPT 0
TIMEOUT 30
MENU TITLE Hyl PXE-Boot Menu
# LABEL install
# 	MENU LABEL Ubuntu Live 22.04 64-Bit
# 	KERNEL casper/vmlinuz
# 	INITRD casper/initrd
# 	APPEND root=/dev/ram0  cloud-config-url=/dev/null ramdisk_size=1500000 ip=dhcp url=http://192.168.2.159/iso/ubuntu-22.04.1-live-server-amd64.iso
#     TEXT HELP
#             Starts the Ubuntu 22.04 64-Bit
#     ENDTEXT
LABEL autoinstallcd
	MENU LABEL Ubuntu Live 22.04 64-Bit CD ROM
	KERNEL /casper/vmlinuz
    APPEND cloud-config-url=/dev/null autoinstall ds=nocloud-net;s=http://192.168.2.159/ubuntu1/
	INITRD /casper/initrd
    TEXT HELP
            Starts the Ubuntu 22.04 64-Bit autoinsttall from CD ROM
    ENDTEXT
LABEL autoinstall
	MENU LABEL Ubuntu Live 22.04 64-Bit autoinstall
    KERNEL casper/vmlinuz
    APPEND root=/dev/ram0 cloud-config-url=/dev/null ramdisk_size=1500000 ip=dhcp url=http://192.168.2.159/iso/ubuntu-22.04.1-live-server-amd64.iso autoinstall ds=nocloud-net;s=http://192.168.2.159/ubuntu1/
    INITRD casper/initrd
    TEXT HELP
            Starts the Ubuntu 22.04 64-Bit autoinstall from http
    ENDTEXT

cd autoinstall

mkdir tftp

cd tftp

mkdir syslinux

cd syslinux

wget --no-check-certificate https://www.kernel.org/pub/linux/utils/boot/syslinux/syslinux-6.03.tar.gz

tar -zxvf syslinux-6.03.tar.gz

cd syslinux-6.03/bios

cp core/pxelinux.0 com32/elflink/ldlinux/ldlinux.c32 com32/menu/vesamenu.c32 com32/lib/libcom32.c32 com32/libutil/libutil.c32 ../../../../tftp

DEFAULT vesamenu.c32

PROMPT 0

TIMEOUT 30

MENU TITLE Hyl PXE-Boot Menu

# LABEL install

# MENU LABEL Ubuntu Live 22.04 64-Bit

# KERNEL casper/vmlinuz

# INITRD casper/initrd

# APPEND root=/dev/ram0 cloud-config-url=/dev/null ramdisk_size=1500000 ip=dhcp url=http://192.168.2.159/iso/ubuntu-22.04.1-live-server-amd64.iso

# TEXT HELP

# Starts the Ubuntu 22.04 64-Bit

# ENDTEXT

LABEL autoinstallcd

MENU LABEL Ubuntu Live 22.04 64-Bit CD ROM

KERNEL /casper/vmlinuz

APPEND cloud-config-url=/dev/null autoinstall ds=nocloud-net;s=http://192.168.2.159/ubuntu1/

INITRD /casper/initrd

TEXT HELP

Starts the Ubuntu 22.04 64-Bit autoinsttall from CD ROM

ENDTEXT

LABEL autoinstall

MENU LABEL Ubuntu Live 22.04 64-Bit autoinstall

KERNEL casper/vmlinuz

APPEND root=/dev/ram0 cloud-config-url=/dev/null ramdisk_size=1500000 ip=dhcp url=http://192.168.2.159/iso/ubuntu-22.04.1-live-server-amd64.iso autoinstall ds=nocloud-net;s=http://192.168.2.159/ubuntu1/

INITRD casper/initrd

TEXT HELP

Starts the Ubuntu 22.04 64-Bit autoinstall from http

ENDTEXT

注意，根据https://askubuntu.com/questions/1235723/automated-20-04-server-installation-using-pxe-and-live-server-image中描述

使用cloud-config-url=/dev/null可以降低内存的使用要求

3.autoinstall的user-data和iso文件由jumpbox或omv负责提供

如上面参数所示临时用python启动了一个http服务器放在了http://192.168.2.159/iso/ubuntu-22.04.1-live-server-amd64.iso，后面考虑换成omv或群晖提供。理论上这里过了DHCP过程因此不受测试网段的限制才对

尝试使用cdrom挂载iso但使用pxe修改内核参数从光驱执行自动安装

https://www.cnblogs.com/xyshun/p/9427472.html

https://www.dddns.icu/posts/pxe/

https://github.com/pypxe/PyPXE

https://github.com/netbootxyz/netboot.xyz

4.全自动化

1.esxi自动化由govc搞定，创建后可拿到网卡mac地址

虚机创建见前面的脚本

从cdrom安装这种方式需要修改bios启动顺序才行，disk>net>cd-rom，disk放第一个是防止安装成功后还重复执行安装，net比cd-rom靠前是因为避免自动进光盘页面无法走自动化。


govc device.boot -vm "$name" -order disk,ethernet,cdrom

0 1	govc device.boot -vm "$name" -order disk,ethernet,cdrom

2.通过mac地址可控制openwrt的tftp按mac地址返回引导选项

自动获取mac如下


mac=""
while [ x$mac = x"" ] || [ x$mac = x"null" ]
do
  mac=$(govc vm.info -json $name | jq -r ".VirtualMachines[0].Config.Hardware.Device[]|select(.Backing.DeviceName == \"$network\")|.MacAddress")
  sleep 1
done
echo "Get vm macaddress succ, $mac"

mac=""

while [ x$mac = x"" ] || [ x$mac = x"null" ]

mac=$(govc vm.info -json $name | jq -r ".VirtualMachines[0].Config.Hardware.Device[]|select(.Backing.DeviceName == \"$network\")|.MacAddress")

sleep 1

done

echo "Get vm macaddress succ, $mac"

编写脚本按mac和hostname动态生成引导菜单并上传到tftp


pxecfgname=01-${mac//:/-}
cat <<-EOF | tee /tmp/$pxecfgname
DEFAULT vesamenu.c32
PROMPT 0
TIMEOUT 30
MENU TITLE Hyl PXE-Boot Menu
LABEL autoinstallcd
	MENU LABEL Ubuntu Live 22.04 64-Bit CD auto
	KERNEL /casper/vmlinuz
    APPEND cloud-config-url=/dev/null autoinstall ds=nocloud-net;s=http://192.168.0.6:5003/$name/
	INITRD /casper/initrd
    TEXT HELP
            Starts the Ubuntu 22.04 64-Bit autoinsttall from CD ROM
    ENDTEXT
EOF
echo "Generated menu on pxelinux.cfg/$pxecfgname:"
# cat /tmp/$pxecfgname
scp /tmp/$pxecfgname root@192.168.2.1:/root/tftp/pxelinux.cfg/
rm /tmp/$pxecfgname

pxecfgname=01-${mac//:/-}

cat <<-EOF | tee /tmp/$pxecfgname

DEFAULT vesamenu.c32

PROMPT 0

TIMEOUT 30

MENU TITLE Hyl PXE-Boot Menu

LABEL autoinstallcd

MENU LABEL Ubuntu Live 22.04 64-Bit CD auto

KERNEL /casper/vmlinuz

APPEND cloud-config-url=/dev/null autoinstall ds=nocloud-net;s=http://192.168.0.6:5003/$name/

INITRD /casper/initrd

TEXT HELP

Starts the Ubuntu 22.04 64-Bit autoinsttall from CD ROM

ENDTEXT

EOF

echo "Generated menu on pxelinux.cfg/$pxecfgname:"

# cat /tmp/$pxecfgname

scp /tmp/$pxecfgname root@192.168.2.1:/root/tftp/pxelinux.cfg/

rm /tmp/$pxecfgname

注意文件名应该以01-开头，详见https://wiki.syslinux.org/wiki/index.php?title=PXELINUX

还可以用16进制的ip地址，总规则是

After attempting the file as specified in the DHCP or hardcoded options, PXELINUX will probe the following paths, prefixed with “pxelinux.cfg/“, under the initial Working Directory.

The client UUID, if provided by the PXE stack.

Note that some BIOSes do not have a valid UUID, and it might end up reporting something like all 1’s.

This value is represented in the standard UUID format using lowercase hexadecimal digits, e.g. “b8945908-d6a6-41a9-611d-74a6ab80b83d“.

The hardware type (using its ARP “htype” code) and address, all in lowercase hexadecimal with dash separators.

For example, for an Ethernet (i.e. ARP hardware type “1“) with address “88:99:AA:BB:CC:DD“, it would search for the filename “01-88-99-aa-bb-cc-dd“.

The client’s own IPv4 address in uppercase hexadecimal, followed by removing hex characters, one at a time, from the end. For example, “192.168.2.91” → “C0A8025B“.

The included program, “gethostip“, can be used to compute the hexadecimal IP address for any host.

Lowercase “default“.

3.引导选项中的iso暂时由群晖webstaion提供

静态站点，无需多言

暂时不需要了，ramdisk有问题。使用ramdisk加载iso后安装很慢，应该是我给的内存太少了。

4.引导选项中的autoinstall配置文件也由webstation提供

群晖自带的webstation可以用PHP，打算尝试使用PHP测试伪静态

使/prefix/{hostname}/user-data可按我指定的模板返回，并动态替换hostname

找到webstation的主nginxconfg文件，/var/tmp/nginx/app.d/server.webstation-vhost.conf，可以看到有include


server {
............
    include /usr/local/etc/nginx/conf.d/blablablabla/user.conf*;

}

server {

............

include /usr/local/etc/nginx/conf.d/blablablabla/user.conf*;

}

按路径新建一个 /usr/local/etc/nginx/conf.d/blablablabla/user.conf

/usr/local/etc/nginx/conf.d/a9d1c5c8-082a-482c-8fe1-73afc670ff6c/user.conf

感谢eric提供的参考


location ~ ^/(.+)/user-data$ {
        try_files $uri $uri/ /index.php?hostname=$1;
}
location ~ ^/(.+)/meta-data$ {
        return 200 "";
}

location ~ ^/(.+)/user-data$ {

try_files $uri $uri/ /index.php?hostname=$1;

}

location ~ ^/(.+)/meta-data$ {

return 200 "";

}

编写index.php如下


<?php
$hostname = $_GET['hostname'];
$file = fopen("template/user-data", "r");
while(!feof($file)){
    $buffer = $buffer.fgets($file, 4096);
}
fclose($file);
$buffer = str_replace("\${hostname}", $hostname, $buffer);
// echo $hostname;
header('Content-Type: application/octet-stream');
echo $buffer;

<?php

$hostname = $_GET['hostname'];

$file = fopen("template/user-data", "r");

while(!feof($file)){

$buffer = $buffer.fgets($file, 4096);

}

fclose($file);

$buffer = str_replace("\${hostname}", $hostname, $buffer);

// echo $hostname;

header('Content-Type: application/octet-stream');

echo $buffer;

参考

https://ubuntu.com/server/docs/install/autoinstall

https://github.com/hedzr/pxe-server-and-focal

https://hedzr.com/devops/linux/build-pxe-server-and-autoinstall-ubuntu-server

https://github.com/vmware/govmomi/blob/main/govc/README.md

https://blog.amrom.tk/2022/2022-03-03-esxi-cloud-init/

https://cloudinit.readthedocs.io/en/latest/topics/examples.html

https://nickhowell.uk/2020/05/01/Automating-Ubuntu2004-Images/

https://discourse.ubuntu.com/t/automated-server-installer-config-file-reference/16613/42

http://hmli.ustc.edu.cn/doc/linux/ubuntu-autoinstall/ubuntu-autoinstall.html