From a38ef1643cee7a4041494bbcc2474a9e1824f1ca Mon Sep 17 00:00:00 2001 From: Adel Kara Slimane Date: Wed, 4 Jan 2023 15:27:41 +0100 Subject: [PATCH] Improve customization of source and work folders (#672) * customization.cfg: add new entries for kernel paths Two choices: - Where to put git bare repository, - Will be implemented as a bare repository, the folder will have only compressed blobs and will be around ~400MB. Should be good to have even in HDDs - This is meant to be in a permanent storage like HDD or SSD. But some uses could have it on a tmpfs if the storage is very slow when compared to a full git (shallow) clone - Where to have the work/build folder - This will be implemented through a git worktree, where the bare repo will be decompressed. This is best on a tmpfs folder but needs >=32GB for a full kernel build. Otherwise should be okay with modprobed-db * customization.cfg: remove obsolete entries the tmpfs related entries can be set with the previously added entries, i.e. _kernel_work_folder and _kernel_source_folder * prepare: check if obsoleted vars get set, warn if so and exit Like this users will get notified about the change and act accordingly * prepare: remove unused kernel subver patching code this used to bump from kernel x.y to x.y.z * prepare: add func to define kernel abs paths We will use these absolute paths to refer to kernel folders (work and source) * prepare: rewrite git fetching code - Rename to "_setup_kernel_work_folder" - Now we simply fetch directly the "commit hash" / "git tag" from the user chosen remote, no branches - The source folder (the one that is preferably on disk) is only a bare repo with a small size (~400MB) - The work folder is then (re)created / updated (as need) as a git worktree * prepare: tkg_patcher: embed patch name into the message Otherwise it doesn't work on the install.sh approach, which reimplements a simple msg2 function that doesn't support %s replacement * PKGBUILD: use new path variable for work folder * install.sh: use new path variable for work folder * CI: setup source and build folder in /tmp * prepare: update diffconfig with new kernel work folder var --- .../workflows/build-current-kernel-arch.yml | 2 + PKGBUILD | 13 +- customization.cfg | 23 +-- install.sh | 26 +-- linux-tkg-config/prepare | 169 ++++++++---------- 5 files changed, 104 insertions(+), 129 deletions(-) diff --git a/.github/workflows/build-current-kernel-arch.yml b/.github/workflows/build-current-kernel-arch.yml index 6d87093..a03b15d 100644 --- a/.github/workflows/build-current-kernel-arch.yml +++ b/.github/workflows/build-current-kernel-arch.yml @@ -10,6 +10,8 @@ env: _processor_opt: "generic" PKGDEST: "/tmp/linux-tkg" _debugdisable: "true" + _kernel_work_folder: /tmp/linux-src-git + _kernel_source_folder: /tmp/linux-kernel.git # _modprobeddb: "true" # _modprobeddb_db_path: ${{ github.workspace }}/modprobed.db diff --git a/PKGBUILD b/PKGBUILD index 9733e4c..6ff8048 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -84,15 +84,15 @@ prepare() { ln -s "${_where}/customization.cfg" "${srcdir}" # workaround ln -s "${_where}/linux-src-git" "${srcdir}" # workaround, this doesn't respect tmpfs choice - cd "${srcdir}/${_srcpath}" - source "${_where}/current_env" _tkg_srcprep } build() { - cd "${srcdir}/${_srcpath}" + + _define_kernel_abs_paths + cd "$_kernel_work_folder_abs" # Use custom compiler paths if defined if [ "$_compiler_name" = "-llvm" ] && [ -n "${CUSTOM_LLVM_PATH}" ]; then @@ -143,7 +143,8 @@ hackbase() { fi replaces=(virtualbox-guest-modules-arch wireguard-arch) - cd "${srcdir}/${_srcpath}" + _define_kernel_abs_paths + cd "$_kernel_work_folder_abs" # get kernel version local _kernver="$(= 32GB ram when building a full kernel, should work with less ram with modprobed-db +_kernel_work_folder="linux-src-git" + +# Permanent folder where to keep the git clone and fetch new blobs +# Note: - Start with a '/' for an absolute path +# - If your internet is faster than your storage, it may be wise to put this folder +# in a tmpfs location (although it will reclone after each restart / tmpfs folder cleanup) +_kernel_source_folder="linux-kernel.git" + # Custom compiler root dirs - Leave empty to use system compilers # Example: CUSTOM_GCC_PATH="/home/frog/PKGBUILDS/mostlyportable-gcc/gcc-mostlyportable-9.2.0" CUSTOM_GCC_PATH="" @@ -54,17 +66,6 @@ _diffconfig="" # Set to the file name where the generated config fragment should be written to. Only used if _diffconfig is active. _diffconfig_name="" -# [install.sh specific] Use tmpfs as a work directory, recommended when RAM >= 32GB to reduce HDD/SSD usage. For more information, see https://wiki.archlinux.org/title/Tmpfs -_use_tmpfs="false" - -# Always make a fresh clone of the source in tmpfs to speed up compilation times -# ! This will take ~20GB of RAM by itself, so don't use on <32GB RAM systems ! -_source_in_tmpfs="false" - -# [install.sh specific] tmpfs folder path, only used when _use_tmpfs="true". -# Creates a linux-tkg work folder within that pathmake sure to have nothing important in "$_tmpfs_path/linux-tkg" -_tmpfs_path="/tmp" - # [install.sh: Generic and Gentoo specific] Dracut options when generating initramfs _dracut_options="--lz4" diff --git a/install.sh b/install.sh index 6c0c8cf..28134e8 100755 --- a/install.sh +++ b/install.sh @@ -83,7 +83,7 @@ _install_dependencies() { if [ "$1" != "install" ] && [ "$1" != "config" ] && [ "$1" != "uninstall-help" ]; then msg2 "Argument not recognised, options are: - - config : interactive script that shallow clones the linux 5.x.y git tree into the folder linux-src-git, then applies extra patches and prepares the .config file + - config : interactive script that shallow clones the linux 5.x.y git tree into the folder \$_kernel_work_folder, then applies extra patches and prepares the .config file by copying the one from the currently running linux system and updates it. - install : does the config step, proceeds to compile, then prompts to install - 'DEB' distros: it creates .deb packages that will be installed then stored in the DEBS folder. @@ -136,24 +136,9 @@ if [ "$1" = "install" ] || [ "$1" = "config" ]; then _distro="" fi - # cd into the linux-src folder is important before calling _tkg_srcprep - cd "$_where/linux-src-git" _tkg_srcprep - _build_dir="$_where" - if [ "$_use_tmpfs" = "true" ]; then - if [ -d "$_tmpfs_path/linux-tkg" ]; then - msg2 "Nuking linux-tkg tmpfs folder $_tmpfs_path/linux-tkg" - rm -rf "$_tmpfs_path/linux-tkg" - fi - mkdir "$_tmpfs_path/linux-tkg" - cp -r "$_where/linux-src-git" "$_tmpfs_path/linux-tkg/linux-src-git" - - # cd into the linux-src folder is important before calling _tkg_srcprep - _build_dir="$_tmpfs_path/linux-tkg" - cd "$_tmpfs_path/linux-tkg/linux-src-git" - fi - + _build_dir="$_kernel_work_folder_abs/.." # Uppercase characters are not allowed in source package name for debian based distros if [[ "$_distro" =~ ^(Debian|Ubuntu)$ && "$_cpusched" = "MuQSS" ]]; then @@ -221,6 +206,8 @@ if [ "$1" = "install" ]; then #_runtime=$( time ( schedtool -B -n 1 -e ionice -n 1 "$@" 2>&1 ) 3>&1 1>&2 2>&3 ) || _runtime=$( time ( "$@" 2>&1 ) 3>&1 1>&2 2>&3 ) - Bash 5.2 is broken https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1018727 #} + cd "$_kernel_work_folder_abs" + if [[ "$_distro" =~ ^(Ubuntu|Debian)$ ]]; then msg2 "Building kernel DEB packages" @@ -262,10 +249,7 @@ if [ "$1" = "install" ]; then _extra_ver_str="_${_kernel_flavor}" fi - _fedora_work_dir="${HOME}/.cache/linux-tkg-rpmbuild" - if [ "$_use_tmpfs" = "true" ]; then - _fedora_work_dir="$_tmpfs_path/linux-tkg/linux-tkg-rpmbuild" - fi + _fedora_work_dir="$_kernel_work_folder/linux-tkg-rpmbuild" msg2 "Building kernel RPM packages" RPMOPTS="--define '_topdir ${_fedora_work_dir}'" make ${llvm_opt} -j ${_thread_num} rpm-pkg EXTRAVERSION="${_extra_ver_str}" diff --git a/linux-tkg-config/prepare b/linux-tkg-config/prepare index c6d3d18..ebbc7a9 100644 --- a/linux-tkg-config/prepare +++ b/linux-tkg-config/prepare @@ -382,8 +382,28 @@ _set_compiler(){ echo -e "_compiler_name='$_compiler_name'\nllvm_opt='$llvm_opt'" >> "$_where"/BIG_UGLY_FROGMINER } +_define_kernel_abs_paths() { -_linux_git_branch_checkout() { + _kernel_work_folder_abs="$_kernel_work_folder" + if ! [[ "$_kernel_work_folder" == /* ]]; then + _kernel_work_folder_abs="$_where/$_kernel_work_folder" + fi + + _kernel_source_folder_abs="$_kernel_source_folder" + if ! [[ "$_kernel_source_folder" == /* ]]; then + _kernel_source_folder_abs="$_where/$_kernel_source_folder" + fi +} + + +_setup_kernel_work_folder() { + + _define_kernel_abs_paths + + if [ "$_kernel_source_folder_abs" == "$_kernel_work_folder_abs" ]; then + warning "The kernel source folder cannot be the same as the kernel work folder" + exit 1 + fi if [ -z "$_kernel_git_tag" ]; then warning "internal error: kernel version should be chosen before cloning kernel sources" @@ -392,85 +412,44 @@ _linux_git_branch_checkout() { cd "$_where" - if ! [ -d linux-src-git ] || ( [ "$_source_in_tmpfs" = "true" ] && ! [ -d /tmp/linux-src-git ] ); then + if ! [ -d "$_kernel_source_folder_abs" ]; then msg2 "First initialization of the linux source code git folder" - if [ "$_source_in_tmpfs" = "true" ]; then - rm -rf "${_where}/linux-src-git" - mkdir "/tmp/linux-src-git" - ln -s "/tmp/linux-src-git" "${_where}" - else - mkdir linux-src-git - fi - cd linux-src-git - git init + mkdir -p "$_kernel_source_folder_abs" + cd "$_kernel_source_folder_abs" + git init --bare + fi - for remote in "${!_kernel_git_remotes[@]}"; do + cd "$_kernel_source_folder_abs" + + # Add remotes if needed + for remote in "${!_kernel_git_remotes[@]}"; do + if ! git remote -v | grep -w "$remote" ; then git remote add "$remote" "${_kernel_git_remotes[$remote]}" - done - else - if [ "$_source_in_tmpfs" = "true" ]; then - rm -rf "${_where}/linux-src-git" - ln -s "/tmp/linux-src-git" "${_where}" fi + done - cd linux-src-git + msg2 "Fetching tag: $_kernel_git_tag from mirror $_git_mirror" + git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag" - # Remove "origin" remote if present - if git remote -v | grep -w "origin" ; then - git remote rm origin - fi + msg2 "Checking out tag: $_kernel_git_tag" + msg2 " in the work folder: $_kernel_work_folder_abs" - for remote in "${!_kernel_git_remotes[@]}"; do - if ! git remote -v | grep -w "$remote" ; then - git remote add "$remote" "${_kernel_git_remotes[$remote]}" - fi - done - - msg2 "Current branch: $(git branch | grep "\*")" - msg2 "Reseting files to their original state" - - git reset --hard HEAD - git clean -f -d -x - fi - - if [[ "$_sub" = rc* ]]; then - msg2 "Switching to master branch for RC Kernel" - - if ! git branch --list | grep "master-${_git_mirror}" ; then - msg2 "master branch doesn't locally exist, shallow cloning..." - git remote set-branches --add kernel.org master - git remote set-branches --add googlesource.com master - git remote set-branches --add torvalds master - git fetch --depth=1 $_git_mirror master - git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag" - git checkout -b master-${_git_mirror} ${_git_mirror}/master - else - msg2 "master branch exists locally, updating..." - git checkout master-${_git_mirror} - git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag" - git reset --hard ${_git_mirror}/master - fi - msg2 "Checking out latest RC tag: $_kernel_git_tag" + # The space ' ' in grep -w "$_kernel_work_folder_abs " is important + # to not match an existing folder with a longer name with the same prefix name + if [ -d "$_kernel_work_folder_abs" ] && \ + ( git worktree list | grep -w "$_kernel_work_folder_abs " ) && \ + ( cd "$_kernel_work_folder_abs" && git status > /dev/null 2>&1 ); then + # Worktree folder exists and is a valid worktree + cd "$_kernel_work_folder_abs" + git reset --hard + git clean -ffdx git checkout "$_kernel_git_tag" else - msg2 "Switching to linux-${_basekernel}.y" - if ! git branch --list | grep -w "linux-${_basekernel}-${_git_mirror}" ; then - msg2 "${_basekernel}.y branch doesn't locally exist, shallow cloning..." - git remote set-branches --add kernel.org linux-${_basekernel}.y - git remote set-branches --add googlesource.com linux-${_basekernel}.y - git remote set-branches --add torvalds linux-${_basekernel}.y - git fetch --depth=1 $_git_mirror linux-${_basekernel}.y - git fetch --depth=1 $_git_mirror tag "$_kernel_git_tag" - git checkout -b linux-${_basekernel}-${_git_mirror} ${_git_mirror}/linux-${_basekernel}.y - else - msg2 "${_basekernel}.y branch exists locally, updating..." - git checkout linux-${_basekernel}-${_git_mirror} - git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag" - git reset --hard ${_git_mirror}/linux-${_basekernel}.y - fi - msg2 "Checking out latest release: $_kernel_git_tag" - git checkout "$_kernel_git_tag" + # In all other cases, just force create the work tree + rm -rf "$_kernel_work_folder_abs" + git worktree add -f "$_kernel_work_folder_abs" "$_kernel_git_tag" fi + } @@ -488,6 +467,23 @@ _tkg_initscript() { exit 1 fi + typeset -Ag _deprecated_config_var_set + + # Check if user is defining deprecated config vars + _deprecated_config_var_set=( + ["_use_tmpfs"]=`[ -n "$_use_tmpfs" ]; echo $?` + ["_source_in_tmpfs"]=`[ -n "$_source_in_tmpfs" ]; echo $?` + ["_tmpfs_path"]=`[ -n "$_tmpfs_path" ]; echo $?` + ) + + for _deprectated_config_var in "${!_deprecated_config_var_set[@]}"; do + if [ "${_deprecated_config_var_set[$_deprectated_config_var]}" == "0" ]; then + warning "The deprecated config var $_deprectated_config_var has been set" + warning "Please check the latest customization.cfg file to see what replaces it" + exit 1 + fi + done + # Default to Arch if [ -z "$_distro" ] || [ "$_ispkgbuild" = "true" ]; then msg2 "Defaulting to Archlinux target\n" @@ -526,7 +522,7 @@ _tkg_initscript() { echo -e "_custom_pkgbase=\"$_custom_pkgbase\"" >> "$_where"/BIG_UGLY_FROGMINER fi - _linux_git_branch_checkout + _setup_kernel_work_folder } user_patcher() { @@ -593,12 +589,16 @@ _tkg_patcher() { patch -Np1 -i "$tkgpatch" >> "$_where"/logs/prepare.log.txt || error "An error was encountered applying patches. It was logged to the prepare.log.txt file." echo -e "\n" >> "$_where"/logs/prepare.log.txt else - msg2 "Skipping patch %s...\n (unavailable for this kernel version)" "${tkgpatch##*/}" #" + msg2 "Skipping patch ${tkgpatch##*/}...\n (unavailable for this kernel version)" fi } _tkg_srcprep() { + _define_kernel_abs_paths + + cd "$_kernel_work_folder_abs" + msg2 "Setting version..." scripts/setlocalversion --save-scmversion @@ -610,10 +610,6 @@ _tkg_srcprep() { if [ "${_distro}" = "Void" ]; then pkgver="${version}" fi - if [ "${_distro}" = "Arch" ] || [ "${_distro}" = "Void" ]; then - tkgpatch="$srcdir/patch-${pkgver}" - _msg="Patching from $_basekernel to $pkgver" && _tkg_patcher - fi # Hardened Patches if [ "${_configfile}" = "config_hardened.x86_64" ] && [ "${_cpusched}" = "cfs" ]; then @@ -1770,16 +1766,11 @@ CONFIG_DEBUG_INFO_BTF_MODULES=y\r fi if [ -z "$_diffconfig_name" ]; then echo 'No file name given, not generating config fragment.' - else ( - if [ "$_distro" = "Arch" ] || [ "$_ispkgbuild" = "true" ]; then - prev_pwd="${PWD:-$(pwd)}/linux-src-git" - cd "$_where" || exit - else - prev_pwd="${PWD:-$(pwd)}" - cd "$_where" || exit - fi - "${prev_pwd}/scripts/diffconfig" -m "${prev_pwd}/.config.orig" "${prev_pwd}/.config" > "$_diffconfig_name" - ) fi + else + pushd "$_kernel_work_folder_abs" + scripts/diffconfig -m .config.orig .config > "$_where/$_diffconfig_name" + popd + fi fi rm .config.orig fi @@ -1809,12 +1800,6 @@ exit_cleanup() { # Remove winesync rules file rm -f "$_where"/winesync.rules - # Remove RPM temporary files left - rm -rf ${HOME}/.cache/linux-tkg-rpmbuild - if [ "$_distro" != "Arch" ] && [ "$_use_tmpfs" = "true" ]; then - rm -rf "$_tmpfs_path/linux-tkg" - fi - # Community patches removal in case of failure for _p in ${_community_patches[@]}; do rm -f "$_where"/linux"$_basever"-tkg-userpatches/"$_p"