Improve customization of source and work folders (#672)

* customization.cfg: add new entries for kernel paths

Two choices:
- Where to put git bare repository,
  - Will be implemented as a bare repository, the folder will have only
    compressed blobs and will be around ~400MB. Should be good to have
    even in HDDs
  - This is meant to be in a permanent storage like HDD or SSD.
    But some uses could have it on a tmpfs if the storage is very slow
    when compared to a full git (shallow) clone
- Where to have the work/build folder
  - This will be implemented through a git worktree,
    where the bare repo will be decompressed. This is best on a tmpfs folder
    but needs >=32GB for a full kernel build. Otherwise should be okay with
    modprobed-db

* customization.cfg: remove obsolete entries

the tmpfs related entries can be set with the previously added entries,
i.e. _kernel_work_folder and _kernel_source_folder

* prepare: check if obsoleted vars get set, warn if so and exit

Like this users will get notified about the change and act accordingly

* prepare: remove unused kernel subver patching code

this used to bump from kernel x.y to x.y.z

* prepare: add func to define kernel abs paths

We will use these absolute paths to refer to kernel folders (work and source)

* prepare: rewrite git fetching code

- Rename to "_setup_kernel_work_folder"
- Now we simply fetch directly the "commit hash" / "git tag"
  from the user chosen remote, no branches
- The source folder (the one that is preferably on disk) is only a bare repo
  with a small size (~400MB)
- The work folder is then (re)created / updated (as need) as a git worktree

* prepare: tkg_patcher: embed patch name into the message

Otherwise it doesn't work on the install.sh approach,
which reimplements a simple msg2 function that doesn't support %s
replacement

* PKGBUILD: use new path variable for work folder

* install.sh: use new path variable for work folder

* CI: setup source and build folder in /tmp

* prepare: update diffconfig with new kernel work folder var

.github/workflows/build-current-kernel-arch.yml

@@ -10,6 +10,8 @@ env:
   _processor_opt: "generic"
   PKGDEST: "/tmp/linux-tkg"
   _debugdisable: "true"
+  _kernel_work_folder: /tmp/linux-src-git
+  _kernel_source_folder: /tmp/linux-kernel.git
   # _modprobeddb: "true"
   # _modprobeddb_db_path: ${{ github.workspace }}/modprobed.db
 

PKGBUILD

@@ -84,15 +84,15 @@ prepare() {
   ln -s "${_where}/customization.cfg" "${srcdir}" # workaround
   ln -s "${_where}/linux-src-git" "${srcdir}" # workaround, this doesn't respect tmpfs choice
 
-  cd "${srcdir}/${_srcpath}"
-
   source "${_where}/current_env"
 
   _tkg_srcprep
 }
 
 build() {
-  cd "${srcdir}/${_srcpath}"
+
+  _define_kernel_abs_paths
+  cd "$_kernel_work_folder_abs"
 
   # Use custom compiler paths if defined
   if [ "$_compiler_name" = "-llvm" ] && [ -n "${CUSTOM_LLVM_PATH}" ]; then
@@ -143,7 +143,8 @@ hackbase() {
   fi
   replaces=(virtualbox-guest-modules-arch wireguard-arch)
 
-  cd "${srcdir}/${_srcpath}"
+  _define_kernel_abs_paths
+  cd "$_kernel_work_folder_abs"
 
   # get kernel version
   local _kernver="$(<version)"
@@ -202,7 +203,9 @@ hackheaders() {
     ;;
   esac
 
-  cd "${srcdir}/${_srcpath}"
+  _define_kernel_abs_paths
+  cd "$_kernel_work_folder_abs"
+
   local builddir="${pkgdir}/usr/lib/modules/$(<version)/build"
 
   msg2 "Installing build files..."

customization.cfg

@@ -23,6 +23,18 @@ _NUKR="true"
 # Git mirror to use to get the kernel sources, possible values are "kernel.org", "googlesource.com", "github.com" and "torvalds"
 _git_mirror="kernel.org"
 
+# Folder where to checkout the kernel sources and build inside it
+# Note: - Start with a '/' for an absolute path
+#       - This setting can be used to set the work/build folder to a tmpfs folder
+#         - Requires >= 32GB ram when building a full kernel, should work with less ram with modprobed-db
+_kernel_work_folder="linux-src-git"
+
+# Permanent folder where to keep the git clone and fetch new blobs
+# Note: - Start with a '/' for an absolute path
+#       - If your internet is faster than your storage, it may be wise to put this folder
+#         in a tmpfs location (although it will reclone after each restart / tmpfs folder cleanup)
+_kernel_source_folder="linux-kernel.git"
+
 # Custom compiler root dirs - Leave empty to use system compilers
 # Example: CUSTOM_GCC_PATH="/home/frog/PKGBUILDS/mostlyportable-gcc/gcc-mostlyportable-9.2.0"
 CUSTOM_GCC_PATH=""
@@ -54,17 +66,6 @@ _diffconfig=""
 # Set to the file name where the generated config fragment should be written to. Only used if _diffconfig is active.
 _diffconfig_name=""
 
-# [install.sh specific] Use tmpfs as a work directory, recommended when RAM >= 32GB to reduce HDD/SSD usage. For more information, see https://wiki.archlinux.org/title/Tmpfs
-_use_tmpfs="false"
-
-# Always make a fresh clone of the source in tmpfs to speed up compilation times
-# ! This will take ~20GB of RAM by itself, so don't use on <32GB RAM systems !
-_source_in_tmpfs="false"
-
-# [install.sh specific] tmpfs folder path, only used when _use_tmpfs="true".
-#                       Creates a linux-tkg work folder within that pathmake sure to have nothing important in "$_tmpfs_path/linux-tkg"
-_tmpfs_path="/tmp"
-
 # [install.sh: Generic and Gentoo specific] Dracut options when generating initramfs
 _dracut_options="--lz4"
 

install.sh

@@ -83,7 +83,7 @@ _install_dependencies() {
 
 if [ "$1" != "install" ] && [ "$1" != "config" ] && [ "$1" != "uninstall-help" ]; then
   msg2 "Argument not recognised, options are:
-        - config : interactive script that shallow clones the linux 5.x.y git tree into the folder linux-src-git, then applies extra patches and prepares the .config file
+        - config : interactive script that shallow clones the linux 5.x.y git tree into the folder \$_kernel_work_folder, then applies extra patches and prepares the .config file
                    by copying the one from the currently running linux system and updates it.
         - install : does the config step, proceeds to compile, then prompts to install
                     - 'DEB' distros: it creates .deb packages that will be installed then stored in the DEBS folder.
@@ -136,24 +136,9 @@ if [ "$1" = "install" ] || [ "$1" = "config" ]; then
     _distro=""
   fi
 
-  # cd into the linux-src folder is important before calling _tkg_srcprep
-  cd "$_where/linux-src-git"
   _tkg_srcprep
 
-  _build_dir="$_where"
+  _build_dir="$_kernel_work_folder_abs/.."
-  if [ "$_use_tmpfs" = "true" ]; then
-    if [ -d "$_tmpfs_path/linux-tkg" ]; then
-      msg2 "Nuking linux-tkg tmpfs folder $_tmpfs_path/linux-tkg"
-      rm -rf "$_tmpfs_path/linux-tkg"
-    fi
-    mkdir "$_tmpfs_path/linux-tkg"
-    cp -r "$_where/linux-src-git" "$_tmpfs_path/linux-tkg/linux-src-git"
-
-    # cd into the linux-src folder is important before calling _tkg_srcprep
-    _build_dir="$_tmpfs_path/linux-tkg"
-    cd "$_tmpfs_path/linux-tkg/linux-src-git"
-  fi
-
 
   # Uppercase characters are not allowed in source package name for debian based distros
   if [[ "$_distro" =~ ^(Debian|Ubuntu)$ && "$_cpusched" = "MuQSS" ]]; then
@@ -221,6 +206,8 @@ if [ "$1" = "install" ]; then
     #_runtime=$( time ( schedtool -B -n 1 -e ionice -n 1 "$@" 2>&1 ) 3>&1 1>&2 2>&3 ) || _runtime=$( time ( "$@" 2>&1 ) 3>&1 1>&2 2>&3 ) - Bash 5.2 is broken https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1018727
   #}
 
+  cd "$_kernel_work_folder_abs"
+
   if [[ "$_distro" =~ ^(Ubuntu|Debian)$ ]]; then
 
     msg2 "Building kernel DEB packages"
@@ -262,10 +249,7 @@ if [ "$1" = "install" ]; then
       _extra_ver_str="_${_kernel_flavor}"
     fi
 
-    _fedora_work_dir="${HOME}/.cache/linux-tkg-rpmbuild"
+    _fedora_work_dir="$_kernel_work_folder/linux-tkg-rpmbuild"
-    if [ "$_use_tmpfs" = "true" ]; then
-      _fedora_work_dir="$_tmpfs_path/linux-tkg/linux-tkg-rpmbuild"
-    fi
 
     msg2 "Building kernel RPM packages"
     RPMOPTS="--define '_topdir ${_fedora_work_dir}'" make ${llvm_opt} -j ${_thread_num} rpm-pkg EXTRAVERSION="${_extra_ver_str}"

linux-tkg-config/prepare

@@ -382,8 +382,28 @@ _set_compiler(){
   echo -e "_compiler_name='$_compiler_name'\nllvm_opt='$llvm_opt'" >> "$_where"/BIG_UGLY_FROGMINER
 }
 
+_define_kernel_abs_paths() {
 
-_linux_git_branch_checkout() {
+  _kernel_work_folder_abs="$_kernel_work_folder"
+  if ! [[ "$_kernel_work_folder" == /* ]]; then
+    _kernel_work_folder_abs="$_where/$_kernel_work_folder"
+  fi
+
+  _kernel_source_folder_abs="$_kernel_source_folder"
+  if ! [[ "$_kernel_source_folder" == /* ]]; then
+    _kernel_source_folder_abs="$_where/$_kernel_source_folder"
+  fi
+}
+
+
+_setup_kernel_work_folder() {
+
+  _define_kernel_abs_paths
+
+  if [ "$_kernel_source_folder_abs" == "$_kernel_work_folder_abs" ]; then
+    warning "The kernel source folder cannot be the same as the kernel work folder"
+    exit 1
+  fi
 
   if [ -z "$_kernel_git_tag" ]; then
     warning "internal error: kernel version should be chosen before cloning kernel sources"
@@ -392,85 +412,44 @@ _linux_git_branch_checkout() {
 
   cd "$_where"
 
-  if ! [ -d linux-src-git ] || ( [ "$_source_in_tmpfs" = "true" ] && ! [ -d /tmp/linux-src-git ] ); then
+  if ! [ -d "$_kernel_source_folder_abs" ]; then
     msg2 "First initialization of the linux source code git folder"
-    if [ "$_source_in_tmpfs" = "true" ]; then
+    mkdir -p "$_kernel_source_folder_abs"
-      rm -rf "${_where}/linux-src-git"
+    cd "$_kernel_source_folder_abs"
-      mkdir "/tmp/linux-src-git"
+    git init --bare
-      ln -s "/tmp/linux-src-git" "${_where}"
+  fi
-    else
-      mkdir linux-src-git
-    fi
-    cd linux-src-git
-    git init
 
-    for remote in "${!_kernel_git_remotes[@]}"; do
+  cd "$_kernel_source_folder_abs"
+
+  # Add remotes if needed
+  for remote in "${!_kernel_git_remotes[@]}"; do
+    if ! git remote -v | grep -w "$remote" ; then
       git remote add "$remote" "${_kernel_git_remotes[$remote]}"
-    done
-  else
-    if [ "$_source_in_tmpfs" = "true" ]; then
-      rm -rf "${_where}/linux-src-git"
-      ln -s "/tmp/linux-src-git" "${_where}"
     fi
+  done
 
-    cd linux-src-git
+  msg2 "Fetching tag: $_kernel_git_tag from mirror $_git_mirror"
+  git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
 
-    # Remove "origin" remote if present
+  msg2 "Checking out tag: $_kernel_git_tag"
-    if git remote -v | grep -w "origin" ; then
+  msg2 "   in the work folder: $_kernel_work_folder_abs"
-      git remote rm origin
-    fi
 
-    for remote in "${!_kernel_git_remotes[@]}"; do
+  # The space ' ' in grep -w "$_kernel_work_folder_abs " is important
-      if ! git remote -v | grep -w "$remote" ; then
+  # to not match an existing folder with a longer name with the same prefix name
-        git remote add "$remote" "${_kernel_git_remotes[$remote]}"
+  if [ -d "$_kernel_work_folder_abs" ] && \
-      fi
+     ( git worktree list | grep -w "$_kernel_work_folder_abs " ) && \
-    done
+     ( cd "$_kernel_work_folder_abs" && git status > /dev/null 2>&1 ); then
-
+    # Worktree folder exists and is a valid worktree
-    msg2 "Current branch: $(git branch | grep "\*")"
+    cd "$_kernel_work_folder_abs"
-    msg2 "Reseting files to their original state"
+    git reset --hard
-
+    git clean -ffdx
-    git reset --hard HEAD
-    git clean -f -d -x
-  fi
-
-  if [[ "$_sub" = rc* ]]; then
-    msg2 "Switching to master branch for RC Kernel"
-
-    if ! git branch --list | grep "master-${_git_mirror}" ; then
-      msg2 "master branch doesn't locally exist, shallow cloning..."
-      git remote set-branches --add kernel.org master
-      git remote set-branches --add googlesource.com master
-      git remote set-branches --add torvalds master
-      git fetch --depth=1 $_git_mirror master
-      git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
-      git checkout -b master-${_git_mirror} ${_git_mirror}/master
-    else
-      msg2 "master branch exists locally, updating..."
-      git checkout master-${_git_mirror}
-      git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
-      git reset --hard ${_git_mirror}/master
-    fi
-    msg2 "Checking out latest RC tag: $_kernel_git_tag"
     git checkout "$_kernel_git_tag"
   else
-    msg2 "Switching to linux-${_basekernel}.y"
+    # In all other cases, just force create the work tree
-    if ! git branch --list | grep -w "linux-${_basekernel}-${_git_mirror}" ; then
+    rm -rf "$_kernel_work_folder_abs"
-      msg2 "${_basekernel}.y branch doesn't locally exist, shallow cloning..."
+    git worktree add -f "$_kernel_work_folder_abs" "$_kernel_git_tag"
-      git remote set-branches --add kernel.org linux-${_basekernel}.y
-      git remote set-branches --add googlesource.com linux-${_basekernel}.y
-      git remote set-branches --add torvalds linux-${_basekernel}.y
-      git fetch --depth=1 $_git_mirror linux-${_basekernel}.y
-      git fetch --depth=1 $_git_mirror tag "$_kernel_git_tag"
-      git checkout -b linux-${_basekernel}-${_git_mirror} ${_git_mirror}/linux-${_basekernel}.y
-    else
-      msg2 "${_basekernel}.y branch exists locally, updating..."
-      git checkout linux-${_basekernel}-${_git_mirror}
-      git fetch --depth 1 $_git_mirror tag "$_kernel_git_tag"
-      git reset --hard ${_git_mirror}/linux-${_basekernel}.y
-    fi
-    msg2 "Checking out latest release: $_kernel_git_tag"
-    git checkout "$_kernel_git_tag"
   fi
+
 }
 
 
@@ -488,6 +467,23 @@ _tkg_initscript() {
     exit 1
   fi
 
+  typeset -Ag _deprecated_config_var_set
+
+  # Check if user is defining deprecated config vars
+  _deprecated_config_var_set=(
+    ["_use_tmpfs"]=`[ -n "$_use_tmpfs" ]; echo $?`
+    ["_source_in_tmpfs"]=`[ -n "$_source_in_tmpfs" ]; echo $?`
+    ["_tmpfs_path"]=`[ -n "$_tmpfs_path" ]; echo $?`
+  )
+
+  for _deprectated_config_var in "${!_deprecated_config_var_set[@]}"; do
+    if [ "${_deprecated_config_var_set[$_deprectated_config_var]}" == "0" ]; then
+      warning "The deprecated config var $_deprectated_config_var has been set"
+      warning "Please check the latest customization.cfg file to see what replaces it"
+      exit 1
+    fi
+  done
+
   # Default to Arch
   if [ -z "$_distro" ] || [ "$_ispkgbuild" = "true" ]; then
     msg2 "Defaulting to Archlinux target\n"
@@ -526,7 +522,7 @@ _tkg_initscript() {
     echo -e "_custom_pkgbase=\"$_custom_pkgbase\"" >> "$_where"/BIG_UGLY_FROGMINER
   fi
 
-  _linux_git_branch_checkout
+  _setup_kernel_work_folder
 }
 
 user_patcher() {
@@ -593,12 +589,16 @@ _tkg_patcher() {
     patch -Np1 -i "$tkgpatch" >> "$_where"/logs/prepare.log.txt || error "An error was encountered applying patches. It was logged to the prepare.log.txt file."
     echo -e "\n" >> "$_where"/logs/prepare.log.txt
   else
-    msg2 "Skipping patch %s...\n         (unavailable for this kernel version)" "${tkgpatch##*/}" #"
+    msg2 "Skipping patch ${tkgpatch##*/}...\n         (unavailable for this kernel version)"
   fi
 }
 
 _tkg_srcprep() {
 
+  _define_kernel_abs_paths
+
+  cd "$_kernel_work_folder_abs"
+
   msg2 "Setting version..."
   scripts/setlocalversion --save-scmversion
 
@@ -610,10 +610,6 @@ _tkg_srcprep() {
   if [ "${_distro}" = "Void" ]; then
 	  pkgver="${version}"
   fi
-  if [ "${_distro}" = "Arch" ] || [ "${_distro}" = "Void" ]; then
-    tkgpatch="$srcdir/patch-${pkgver}"
-    _msg="Patching from $_basekernel to $pkgver" && _tkg_patcher
-  fi
 
   # Hardened Patches
   if [ "${_configfile}" = "config_hardened.x86_64" ] && [ "${_cpusched}" = "cfs" ]; then
@@ -1770,16 +1766,11 @@ CONFIG_DEBUG_INFO_BTF_MODULES=y\r
       fi
       if [ -z "$_diffconfig_name" ]; then
         echo 'No file name given, not generating config fragment.'
-      else (
+      else
-        if [ "$_distro" = "Arch" ] || [ "$_ispkgbuild" = "true" ]; then
+        pushd "$_kernel_work_folder_abs"
-          prev_pwd="${PWD:-$(pwd)}/linux-src-git"
+        scripts/diffconfig -m .config.orig .config > "$_where/$_diffconfig_name"
-          cd "$_where" || exit
+        popd
-        else
+      fi
-          prev_pwd="${PWD:-$(pwd)}"
-          cd "$_where" || exit
-        fi
-        "${prev_pwd}/scripts/diffconfig" -m "${prev_pwd}/.config.orig" "${prev_pwd}/.config" > "$_diffconfig_name"
-      ) fi
     fi
     rm .config.orig
   fi
@@ -1809,12 +1800,6 @@ exit_cleanup() {
   # Remove winesync rules file
   rm -f "$_where"/winesync.rules
 
-  # Remove RPM temporary files left
-  rm -rf ${HOME}/.cache/linux-tkg-rpmbuild
-  if [ "$_distro" != "Arch" ] && [ "$_use_tmpfs" = "true" ]; then
-    rm -rf "$_tmpfs_path/linux-tkg"
-  fi
-
   # Community patches removal in case of failure
   for _p in ${_community_patches[@]}; do
     rm -f "$_where"/linux"$_basever"-tkg-userpatches/"$_p"