sravan/linux-tkg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve customization of source and work folders (#672)

Browse Source 
* customization.cfg: add new entries for kernel paths

Two choices:
- Where to put git bare repository,
  - Will be implemented as a bare repository, the folder will have only
    compressed blobs and will be around ~400MB. Should be good to have
    even in HDDs
  - This is meant to be in a permanent storage like HDD or SSD.
    But some uses could have it on a tmpfs if the storage is very slow
    when compared to a full git (shallow) clone
- Where to have the work/build folder
  - This will be implemented through a git worktree,
    where the bare repo will be decompressed. This is best on a tmpfs folder
    but needs >=32GB for a full kernel build. Otherwise should be okay with
    modprobed-db

* customization.cfg: remove obsolete entries

the tmpfs related entries can be set with the previously added entries,
i.e. _kernel_work_folder and _kernel_source_folder

* prepare: check if obsoleted vars get set, warn if so and exit

Like this users will get notified about the change and act accordingly

* prepare: remove unused kernel subver patching code

this used to bump from kernel x.y to x.y.z

* prepare: add func to define kernel abs paths

We will use these absolute paths to refer to kernel folders (work and source)

* prepare: rewrite git fetching code

- Rename to "_setup_kernel_work_folder"
- Now we simply fetch directly the "commit hash" / "git tag"
  from the user chosen remote, no branches
- The source folder (the one that is preferably on disk) is only a bare repo
  with a small size (~400MB)
- The work folder is then (re)created / updated (as need) as a git worktree

* prepare: tkg_patcher: embed patch name into the message

Otherwise it doesn't work on the install.sh approach,
which reimplements a simple msg2 function that doesn't support %s
replacement

* PKGBUILD: use new path variable for work folder

* install.sh: use new path variable for work folder

* CI: setup source and build folder in /tmp

* prepare: update diffconfig with new kernel work folder var
This commit is contained in:
Adel Kara Slimane
2023-01-04 15:27:41 +01:00
committed by GitHub
parent 86578256b7
commit a38ef1643c
5 changed files with 104 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build-current-kernel-arch.yml vendored
View File

@@ -10,6 +10,8 @@ env:
_processor_opt: "generic"
PKGDEST: "/tmp/linux-tkg"
_debugdisable: "true"
_kernel_work_folder: /tmp/linux-src-git
_kernel_source_folder: /tmp/linux-kernel.git
# _modprobeddb: "true"
# _modprobeddb_db_path: ${{ github.workspace }}/modprobed.db

13
PKGBUILD
View File

@@ -84,15 +84,15 @@ prepare() {
ln -s "${_where}/customization.cfg" "${srcdir}" # workaround
ln -s "${_where}/linux-src-git" "${srcdir}" # workaround, this doesn't respect tmpfs choice
cd "${srcdir}/${_srcpath}"
source "${_where}/current_env"
_tkg_srcprep
}
build() {
cd "${srcdir}/${_srcpath}"
_define_kernel_abs_paths
cd "$_kernel_work_folder_abs"
# Use custom compiler paths if defined
if [ "$_compiler_name" = "-llvm" ] && [ -n "${CUSTOM_LLVM_PATH}" ]; then
@@ -143,7 +143,8 @@ hackbase() {
fi
replaces=(virtualbox-guest-modules-arch wireguard-arch)
cd "${srcdir}/${_srcpath}"
_define_kernel_abs_paths
cd "$_kernel_work_folder_abs"
# get kernel version
local _kernver="$(<version)"
@@ -202,7 +203,9 @@ hackheaders() {
;;
esac
cd "${srcdir}/${_srcpath}"
_define_kernel_abs_paths
cd "$_kernel_work_folder_abs"
local builddir="${pkgdir}/usr/lib/modules/$(<version)/build"
msg2 "Installing build files..."

23
customization.cfg
View File

@@ -23,6 +23,18 @@ _NUKR="true"
# Git mirror to use to get the kernel sources, possible values are "kernel.org", "googlesource.com", "github.com" and "torvalds"
_git_mirror="kernel.org"
# Folder where to checkout the kernel sources and build inside it
# Note: - Start with a '/' for an absolute path
# - This setting can be used to set the work/build folder to a tmpfs folder
# - Requires >= 32GB ram when building a full kernel, should work with less ram with modprobed-db
_kernel_work_folder="linux-src-git"
# Permanent folder where to keep the git clone and fetch new blobs
# Note: - Start with a '/' for an absolute path
# - If your internet is faster than your storage, it may be wise to put this folder
# in a tmpfs location (although it will reclone after each restart / tmpfs folder cleanup)
_kernel_source_folder="linux-kernel.git"
# Custom compiler root dirs - Leave empty to use system compilers
# Example: CUSTOM_GCC_PATH="/home/frog/PKGBUILDS/mostlyportable-gcc/gcc-mostlyportable-9.2.0"
CUSTOM_GCC_PATH=""
@@ -54,17 +66,6 @@ _diffconfig=""
# Set to the file name where the generated config fragment should be written to. Only used if _diffconfig is active.
_diffconfig_name=""
# [install.sh specific] Use tmpfs as a work directory, recommended when RAM >= 32GB to reduce HDD/SSD usage. For more information, see https://wiki.archlinux.org/title/Tmpfs
_use_tmpfs="false"
# Always make a fresh clone of the source in tmpfs to speed up compilation times
# ! This will take ~20GB of RAM by itself, so don't use on <32GB RAM systems !
_source_in_tmpfs="false"
# [install.sh specific] tmpfs folder path, only used when _use_tmpfs="true".
# Creates a linux-tkg work folder within that pathmake sure to have nothing important in "$_tmpfs_path/linux-tkg"
_tmpfs_path="/tmp"
# [install.sh: Generic and Gentoo specific] Dracut options when generating initramfs
_dracut_options="--lz4"

26
install.sh
View File

@@ -83,7 +83,7 @@ _install_dependencies() {
if [ "$1" != "install" ] && [ "$1" != "config" ] && [ "$1" != "uninstall-help" ]; then
msg2 "Argument not recognised, options are:
- config : interactive script that shallow clones the linux 5.x.y git tree into the folder linux-src-git, then applies extra patches and prepares the .config file
- config : interactive script that shallow clones the linux 5.x.y git tree into the folder \$_kernel_work_folder, then applies extra patches and prepares the .config file
by copying the one from the currently running linux system and updates it.
- install : does the config step, proceeds to compile, then prompts to install
- 'DEB' distros: it creates .deb packages that will be installed then stored in the DEBS folder.
@@ -136,24 +136,9 @@ if [ "$1" = "install" ] || [ "$1" = "config" ]; then
_distro=""
fi
# cd into the linux-src folder is important before calling _tkg_srcprep
cd "$_where/linux-src-git"
_tkg_srcprep
_build_dir="$_where"
if [ "$_use_tmpfs" = "true" ]; then
if [ -d "$_tmpfs_path/linux-tkg" ]; then
msg2 "Nuking linux-tkg tmpfs folder $_tmpfs_path/linux-tkg"
rm -rf "$_tmpfs_path/linux-tkg"
fi
mkdir "$_tmpfs_path/linux-tkg"
cp -r "$_where/linux-src-git" "$_tmpfs_path/linux-tkg/linux-src-git"
# cd into the linux-src folder is important before calling _tkg_srcprep
_build_dir="$_tmpfs_path/linux-tkg"
cd "$_tmpfs_path/linux-tkg/linux-src-git"
fi
_build_dir="$_kernel_work_folder_abs/.."
# Uppercase characters are not allowed in source package name for debian based distros
if [[ "$_distro" =~ ^(Debian|Ubuntu)$ && "$_cpusched" = "MuQSS" ]]; then
@@ -221,6 +206,8 @@ if [ "$1" = "install" ]; then
#_runtime=$( time ( schedtool -B -n 1 -e ionice -n 1 "$@" 2>&1 ) 3>&1 1>&2 2>&3 ) || _runtime=$( time ( "$@" 2>&1 ) 3>&1 1>&2 2>&3 ) - Bash 5.2 is broken https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1018727
#}
cd "$_kernel_work_folder_abs"
if [[ "$_distro" =~ ^(Ubuntu|Debian)$ ]]; then
msg2 "Building kernel DEB packages"
@@ -262,10 +249,7 @@ if [ "$1" = "install" ]; then
_extra_ver_str="_${_kernel_flavor}"
fi
_fedora_work_dir="${HOME}/.cache/linux-tkg-rpmbuild"
if [ "$_use_tmpfs" = "true" ]; then
_fedora_work_dir="$_tmpfs_path/linux-tkg/linux-tkg-rpmbuild"
fi
_fedora_work_dir="$_kernel_work_folder/linux-tkg-rpmbuild"
msg2 "Building kernel RPM packages"
RPMOPTS="--define '_topdir ${_fedora_work_dir}'" make ${llvm_opt} -j ${_thread_num} rpm-pkg EXTRAVERSION="${_extra_ver_str}"

169
linux-tkg-config/prepare
View File

@@ -382,8 +382,28 @@ _set_compiler(){
echo -e "_compiler_name='$_compiler_name'\nllvm_opt='$llvm_opt'" >> "$_where"/BIG_UGLY_FROGMINER
}
_define_kernel_abs_paths() {
_linux_git_branch_checkout() {
_kernel_work_folder_abs="$_kernel_work_folder"
if ! [[ "$_kernel_work_folder" == /* ]]; then
_kernel_work_folder_abs="$_where/$_kernel_work_folder"
fi
_kernel_source_folder_abs="$_kernel_source_folder"
if ! [[ "$_kernel_source_folder" == /* ]]; then
_kernel_source_folder_abs="$_where/$_kernel_source_folder"
fi
}
_setup_kernel_work_folder() {
_define_kernel_abs_paths
if [ "$_kernel_source_folder_abs" == "$_kernel_work_folder_abs" ]; then
warning "The kernel source folder cannot be the same as the kernel work folder"
exit 1
fi
if [ -z "$_kernel_git_tag" ]; then
warning "internal error: kernel version should be chosen before cloning kernel sources"
@@ -392,85 +412,44 @@ _linux_git_branch_checkout() {
cd "$_where"
if ! [ -d linux-src-git ] || ( [ "$_source_in_tmpfs" = "true" ] && ! [ -d /tmp/linux-src-git ] ); then
if ! [ -d "$_kernel_source_folder_abs" ]; then
msg2 "First initialization of the linux source code git folder"
if [ "$_source_in_tmpfs" = "true" ]; then
rm -rf "${_where}/linux-src-git"
mkdir "/tmp/linux-src-git"
ln -s "/tmp/linux-src-git" "${_where}"
else
mkdir linux-src-git
fi
cd linux-src-git
git init
mkdir -p "$_kernel_source_folder_abs"
cd "$_kernel_source_folder_abs"
git init --bare
fi
for remote in "${!_kernel_git_remotes[@]}"; do
cd "$_kernel_source_folder_abs"
# Add remotes if needed
for remote in "${!_kernel_git_remotes[@]}"; do
if ! git remote -v | grep -w "$remote" ; then
git remote add "$remote" "${_kernel_git_remotes[$remote]}"
done
else
if [ "$_source_in_tmpfs" = "true" ]; then
rm -rf "${_where}/linux-src-git"
ln -s "/tmp/linux-src-git" "${_where}"
fi
done
cd linux-src-git
msg2 "Fetching tag: $_kernel_git_tag from mirror $_git_mirror"
git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
# Remove "origin" remote if present
if git remote -v | grep -w "origin" ; then
git remote rm origin
fi
msg2 "Checking out tag: $_kernel_git_tag"
msg2 " in the work folder: $_kernel_work_folder_abs"
for remote in "${!_kernel_git_remotes[@]}"; do
if ! git remote -v | grep -w "$remote" ; then
git remote add "$remote" "${_kernel_git_remotes[$remote]}"
fi
done
msg2 "Current branch: $(git branch | grep "\*")"
msg2 "Reseting files to their original state"
git reset --hard HEAD
git clean -f -d -x
fi
if [[ "$_sub" = rc* ]]; then
msg2 "Switching to master branch for RC Kernel"
if ! git branch --list | grep "master-${_git_mirror}" ; then
msg2 "master branch doesn't locally exist, shallow cloning..."
git remote set-branches --add kernel.org master
git remote set-branches --add googlesource.com master
git remote set-branches --add torvalds master
git fetch --depth=1 $_git_mirror master
git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
git checkout -b master-${_git_mirror} ${_git_mirror}/master
else
msg2 "master branch exists locally, updating..."
git checkout master-${_git_mirror}
git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
git reset --hard ${_git_mirror}/master
fi
msg2 "Checking out latest RC tag: $_kernel_git_tag"
# The space ' ' in grep -w "$_kernel_work_folder_abs " is important
# to not match an existing folder with a longer name with the same prefix name
if [ -d "$_kernel_work_folder_abs" ] && \
( git worktree list | grep -w "$_kernel_work_folder_abs " ) && \
( cd "$_kernel_work_folder_abs" && git status > /dev/null 2>&1 ); then
# Worktree folder exists and is a valid worktree
cd "$_kernel_work_folder_abs"
git reset --hard
git clean -ffdx
git checkout "$_kernel_git_tag"
else
msg2 "Switching to linux-${_basekernel}.y"
if ! git branch --list | grep -w "linux-${_basekernel}-${_git_mirror}" ; then
msg2 "${_basekernel}.y branch doesn't locally exist, shallow cloning..."
git remote set-branches --add kernel.org linux-${_basekernel}.y
git remote set-branches --add googlesource.com linux-${_basekernel}.y
git remote set-branches --add torvalds linux-${_basekernel}.y
git fetch --depth=1 $_git_mirror linux-${_basekernel}.y
git fetch --depth=1 $_git_mirror tag "$_kernel_git_tag"
git checkout -b linux-${_basekernel}-${_git_mirror} ${_git_mirror}/linux-${_basekernel}.y
else
msg2 "${_basekernel}.y branch exists locally, updating..."
git checkout linux-${_basekernel}-${_git_mirror}
git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
git reset --hard ${_git_mirror}/linux-${_basekernel}.y
fi
msg2 "Checking out latest release: $_kernel_git_tag"
git checkout "$_kernel_git_tag"
# In all other cases, just force create the work tree
rm -rf "$_kernel_work_folder_abs"
git worktree add -f "$_kernel_work_folder_abs" "$_kernel_git_tag"
fi
}
@@ -488,6 +467,23 @@ _tkg_initscript() {
exit 1
fi
typeset -Ag _deprecated_config_var_set
# Check if user is defining deprecated config vars
_deprecated_config_var_set=(
["_use_tmpfs"]=`[ -n "$_use_tmpfs" ]; echo $?`
["_source_in_tmpfs"]=`[ -n "$_source_in_tmpfs" ]; echo $?`
["_tmpfs_path"]=`[ -n "$_tmpfs_path" ]; echo $?`
)
for _deprectated_config_var in "${!_deprecated_config_var_set[@]}"; do
if [ "${_deprecated_config_var_set[$_deprectated_config_var]}" == "0" ]; then
warning "The deprecated config var $_deprectated_config_var has been set"
warning "Please check the latest customization.cfg file to see what replaces it"
exit 1
fi
done
# Default to Arch
if [ -z "$_distro" ] || [ "$_ispkgbuild" = "true" ]; then
msg2 "Defaulting to Archlinux target\n"
@@ -526,7 +522,7 @@ _tkg_initscript() {
echo -e "_custom_pkgbase=\"$_custom_pkgbase\"" >> "$_where"/BIG_UGLY_FROGMINER
fi
_linux_git_branch_checkout
_setup_kernel_work_folder
}
user_patcher() {
@@ -593,12 +589,16 @@ _tkg_patcher() {
patch -Np1 -i "$tkgpatch" >> "$_where"/logs/prepare.log.txt || error "An error was encountered applying patches. It was logged to the prepare.log.txt file."
echo -e "\n" >> "$_where"/logs/prepare.log.txt
else
msg2 "Skipping patch %s...\n (unavailable for this kernel version)" "${tkgpatch##*/}" #"
msg2 "Skipping patch ${tkgpatch##*/}...\n (unavailable for this kernel version)"
fi
}
_tkg_srcprep() {
_define_kernel_abs_paths
cd "$_kernel_work_folder_abs"
msg2 "Setting version..."
scripts/setlocalversion --save-scmversion
@@ -610,10 +610,6 @@ _tkg_srcprep() {
if [ "${_distro}" = "Void" ]; then
pkgver="${version}"
fi
if [ "${_distro}" = "Arch" ] || [ "${_distro}" = "Void" ]; then
tkgpatch="$srcdir/patch-${pkgver}"
_msg="Patching from $_basekernel to $pkgver" && _tkg_patcher
fi
# Hardened Patches
if [ "${_configfile}" = "config_hardened.x86_64" ] && [ "${_cpusched}" = "cfs" ]; then
@@ -1770,16 +1766,11 @@ CONFIG_DEBUG_INFO_BTF_MODULES=y\r
fi
if [ -z "$_diffconfig_name" ]; then
echo 'No file name given, not generating config fragment.'
else (
if [ "$_distro" = "Arch" ] || [ "$_ispkgbuild" = "true" ]; then
prev_pwd="${PWD:-$(pwd)}/linux-src-git"
cd "$_where" || exit
else
prev_pwd="${PWD:-$(pwd)}"
cd "$_where" || exit
fi
"${prev_pwd}/scripts/diffconfig" -m "${prev_pwd}/.config.orig" "${prev_pwd}/.config" > "$_diffconfig_name"
) fi
else
pushd "$_kernel_work_folder_abs"
scripts/diffconfig -m .config.orig .config > "$_where/$_diffconfig_name"
popd
fi
fi
rm .config.orig
fi
@@ -1809,12 +1800,6 @@ exit_cleanup() {
# Remove winesync rules file
rm -f "$_where"/winesync.rules
# Remove RPM temporary files left
rm -rf ${HOME}/.cache/linux-tkg-rpmbuild
if [ "$_distro" != "Arch" ] && [ "$_use_tmpfs" = "true" ]; then
rm -rf "$_tmpfs_path/linux-tkg"
fi
# Community patches removal in case of failure
for _p in ${_community_patches[@]}; do
rm -f "$_where"/linux"$_basever"-tkg-userpatches/"$_p"