mirror of
https://github.com/unraid/api.git
synced 2026-02-18 05:58:28 -06:00
refactor(api): vendor node_modules instead of pnpm store (#1346)
due to issues and redundancies in vendoring postinstall side-effects, such as compiled bindings for libvirt, we reverted to vendoring `node_modules`, installed via `npm` instead of a global pnpm store generated by `pnpm`. This should resolve runtime issues with e.g. the libvirt bindings because `node_modules` will contain the correct "side-effects." ## Summary by CodeRabbit - **New Features** - Introduced a command to remove stale archive files during the cleanup process. - Added functionality to archive the `node_modules` directory. - Enhanced dependency resolution with new overrides for specific packages. - **Chores** - Updated dependency settings by replacing one key dependency with an alternative and removing two unused ones, ensuring optimal deployment. - Enhanced the installation process to operate strictly in offline mode. - Updated artifact naming conventions for clarity and consistency in workflows. - Modified volume mappings in the Docker Compose configuration to reflect new artifact names. - Improved error handling in the GitHub Actions workflow by adding checks for required files. - Updated references in the build process to use a vendor store instead of the PNPM store. - Removed the management of PNPM store archives from the build process.
This commit is contained in:
Pujit Mehrotra
7
.github/workflows/build-plugin.yml
vendored
7
.github/workflows/build-plugin.yml
vendored
@@ -103,7 +103,7 @@ jobs:
|
||||
- name: Download PNPM Store
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: packed-pnpm-store
|
||||
name: packed-node-modules
|
||||
path: ${{ github.workspace }}/plugin/
|
||||
- name: Extract Unraid API
|
||||
run: |
|
||||
@@ -130,6 +130,11 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f ./deploy/*.tar.xz ]; then
|
||||
echo "Error: .tar.xz file not found in plugin/deploy/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload to GHA
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
|
||||
6
.github/workflows/main.yml
vendored
6
.github/workflows/main.yml
vendored
@@ -191,11 +191,11 @@ jobs:
|
||||
with:
|
||||
name: unraid-api
|
||||
path: ${{ github.workspace }}/api/deploy/unraid-api.tgz
|
||||
- name: Upload PNPM Store to Github artifacts
|
||||
- name: Upload Node Modules to Github artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: packed-pnpm-store
|
||||
path: ${{ github.workspace }}/api/deploy/packed-pnpm-store.txz
|
||||
name: packed-node-modules
|
||||
path: ${{ github.workspace }}/api/deploy/packed-node-modules.tar.xz
|
||||
|
||||
build-unraid-ui-webcomponents:
|
||||
name: Build Unraid UI Library (Webcomponent Version)
|
||||
|
||||
@@ -201,6 +201,13 @@
|
||||
"overrides": {
|
||||
"eslint": {
|
||||
"jiti": "2"
|
||||
},
|
||||
"@as-integrations/fastify": {
|
||||
"fastify": "$fastify"
|
||||
},
|
||||
"nest-authz": {
|
||||
"@nestjs/common": "$@nestjs/common",
|
||||
"@nestjs/core": "$@nestjs/core"
|
||||
}
|
||||
},
|
||||
"private": true,
|
||||
|
||||
@@ -20,8 +20,6 @@ try {
|
||||
// Get package details
|
||||
const packageJson = await readFile('./package.json', 'utf-8');
|
||||
const parsedPackageJson = JSON.parse(packageJson);
|
||||
const rootPackageJson = await readFile('./../package.json', 'utf-8');
|
||||
const parsedRootPackageJson = JSON.parse(rootPackageJson);
|
||||
|
||||
const deploymentVersion = await getDeploymentVersion(process.env, parsedPackageJson.version);
|
||||
|
||||
@@ -30,9 +28,6 @@ try {
|
||||
// omit dev dependencies from release build
|
||||
parsedPackageJson.devDependencies = {};
|
||||
|
||||
// add all PNPM settings for pnpm install from root package.json
|
||||
parsedPackageJson.pnpm = parsedRootPackageJson.pnpm;
|
||||
|
||||
// Create a temporary directory for packaging
|
||||
await mkdir('./deploy/pack/', { recursive: true });
|
||||
|
||||
@@ -43,23 +38,20 @@ try {
|
||||
// Change to the pack directory and install dependencies
|
||||
cd('./deploy/pack');
|
||||
|
||||
console.log('Building production pnpm store...');
|
||||
console.log('Building production node_modules...');
|
||||
$.verbose = true;
|
||||
await $`pnpm install --prod --ignore-workspace --store-dir=../.pnpm-store`;
|
||||
await $`npm install --omit=dev`;
|
||||
|
||||
// Now remove the onlybuilddependencies from the package json
|
||||
delete parsedPackageJson.pnpm;
|
||||
// Now write the package.json back to the pack directoryaw
|
||||
// Now write the package.json back to the pack directory
|
||||
await writeFile('package.json', JSON.stringify(parsedPackageJson, null, 4));
|
||||
|
||||
await $`rm -rf node_modules`; // Don't include node_modules in final package
|
||||
|
||||
const sudoCheck = await $`command -v sudo`.nothrow();
|
||||
const SUDO = sudoCheck.exitCode === 0 ? 'sudo' : '';
|
||||
await $`${SUDO} chown -R 0:0 ../.pnpm-store`;
|
||||
await $`${SUDO} chown -R 0:0 node_modules`;
|
||||
|
||||
await $`XZ_OPT=-5 tar -cJf ../packed-pnpm-store.txz ../.pnpm-store`;
|
||||
await $`${SUDO} rm -rf ../.pnpm-store`;
|
||||
await $`XZ_OPT=-5 tar -cJf packed-node-modules.tar.xz node_modules`;
|
||||
await $`mv packed-node-modules.tar.xz ../`;
|
||||
await $`${SUDO} rm -rf node_modules`;
|
||||
|
||||
// chmod the cli
|
||||
await $`chmod +x ./dist/cli.js`;
|
||||
|
||||
@@ -22,13 +22,11 @@
|
||||
},
|
||||
"onlyBuiltDependencies": [
|
||||
"@apollo/protobufjs",
|
||||
"@nestjs/core",
|
||||
"protobufjs",
|
||||
"@parcel/watcher",
|
||||
"@swc/core",
|
||||
"@unraid/libvirt",
|
||||
"cpu-features",
|
||||
"esbuild",
|
||||
"nestjs-pino",
|
||||
"ssh2",
|
||||
"vue-demi"
|
||||
]
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
} from "./utils/paths";
|
||||
import { PluginEnv, setupPluginEnv } from "./cli/setup-plugin-environment";
|
||||
import { cleanupPluginFiles } from "./utils/cleanup";
|
||||
import { bundlePnpmStore, getPnpmBundleName } from "./build-pnpm-store";
|
||||
import { bundleVendorStore, getVendorBundleName } from "./build-vendor-store";
|
||||
|
||||
/**
|
||||
* Check if git is available
|
||||
@@ -61,8 +61,8 @@ const buildPlugin = async ({
|
||||
pluginURL: getPluginUrl({ baseUrl, tag }),
|
||||
MAIN_TXZ: getMainTxzUrl({ baseUrl, pluginVersion, tag }),
|
||||
TXZ_SHA256: txzSha256,
|
||||
VENDOR_STORE_URL: getAssetUrl({ baseUrl, tag }, getPnpmBundleName()),
|
||||
VENDOR_STORE_FILENAME: getPnpmBundleName(),
|
||||
VENDOR_STORE_URL: getAssetUrl({ baseUrl, tag }, getVendorBundleName()),
|
||||
VENDOR_STORE_FILENAME: getVendorBundleName(),
|
||||
...(tag ? { TAG: tag } : {}),
|
||||
};
|
||||
|
||||
@@ -108,7 +108,7 @@ const main = async () => {
|
||||
|
||||
await buildPlugin(validatedEnv);
|
||||
await moveTxzFile(validatedEnv.txzPath, validatedEnv.pluginVersion);
|
||||
await bundlePnpmStore();
|
||||
await bundleVendorStore();
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
import { apiDir, deployDir } from "./utils/paths";
|
||||
import { join } from "path";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { startingDir } from "./utils/consts";
|
||||
import { copyFile } from "node:fs/promises";
|
||||
|
||||
/**
|
||||
* Get the version of the API from the package.json file
|
||||
*
|
||||
* Throws if package.json is not found or is invalid JSON.
|
||||
* @returns The version of the API
|
||||
*/
|
||||
function getVersion(): string {
|
||||
const packageJsonPath = join(apiDir, "package.json");
|
||||
const packageJsonString = readFileSync(packageJsonPath, "utf8");
|
||||
const packageJson = JSON.parse(packageJsonString);
|
||||
return packageJson.version;
|
||||
}
|
||||
|
||||
/**
|
||||
* The name of the pnpm store archive that will be vendored with the plugin.
|
||||
* @returns The name of the pnpm store bundle file
|
||||
*/
|
||||
export function getPnpmBundleName(): string {
|
||||
const version = getVersion();
|
||||
return `pnpm-store-for-v${version}.txz`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare a versioned bundle of the API's pnpm store to vendor dependencies.
|
||||
*
|
||||
* It expects a generic `packed-pnpm-store.txz` archive to be available in the `startingDir`.
|
||||
* It copies this archive to the `deployDir` directory and adds a version to the filename.
|
||||
* It does not actually create the packed pnpm store archive; that is done inside the API's build script.
|
||||
*
|
||||
* After this operation, the vendored store will be available inside the `deployDir`.
|
||||
*/
|
||||
export async function bundlePnpmStore(): Promise<void> {
|
||||
const storeArchive = join(startingDir, "packed-pnpm-store.txz");
|
||||
const pnpmStoreTarPath = join(deployDir, getPnpmBundleName());
|
||||
await copyFile(storeArchive, pnpmStoreTarPath);
|
||||
}
|
||||
42
plugin/builder/build-vendor-store.ts
Normal file
42
plugin/builder/build-vendor-store.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { apiDir, deployDir } from "./utils/paths";
|
||||
import { join } from "path";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { startingDir } from "./utils/consts";
|
||||
import { copyFile } from "node:fs/promises";
|
||||
|
||||
/**
|
||||
* Get the version of the API from the package.json file
|
||||
*
|
||||
* Throws if package.json is not found or is invalid JSON.
|
||||
* @returns The version of the API
|
||||
*/
|
||||
function getVersion(): string {
|
||||
const packageJsonPath = join(apiDir, "package.json");
|
||||
const packageJsonString = readFileSync(packageJsonPath, "utf8");
|
||||
const packageJson = JSON.parse(packageJsonString);
|
||||
return packageJson.version;
|
||||
}
|
||||
|
||||
/**
|
||||
* The name of the node_modules archive that will be vendored with the plugin.
|
||||
* @returns The name of the node_modules bundle file
|
||||
*/
|
||||
export function getVendorBundleName(): string {
|
||||
const version = getVersion();
|
||||
return `node_modules-for-v${version}.tar.xz`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare a versioned bundle of the API's node_modules to vendor dependencies.
|
||||
*
|
||||
* It expects a generic `packed-node-modules.tar.xz` archive to be available in the `startingDir`.
|
||||
* It copies this archive to the `deployDir` directory and adds a version to the filename.
|
||||
* It does not actually create the packed node_modules archive; that is done inside the API's build script.
|
||||
*
|
||||
* After this operation, the vendored node_modules will be available inside the `deployDir`.
|
||||
*/
|
||||
export async function bundleVendorStore(): Promise<void> {
|
||||
const storeArchive = join(startingDir, "packed-node-modules.tar.xz");
|
||||
const vendorStoreTarPath = join(deployDir, getVendorBundleName());
|
||||
await copyFile(storeArchive, vendorStoreTarPath);
|
||||
}
|
||||
@@ -12,7 +12,7 @@ services:
|
||||
- ../unraid-ui/dist-wc:/app/source/dynamix.unraid.net/usr/local/emhttp/plugins/dynamix.my.servers/unraid-components/uui
|
||||
- ../web/.nuxt/nuxt-custom-elements/dist/unraid-components:/app/source/dynamix.unraid.net/usr/local/emhttp/plugins/dynamix.my.servers/unraid-components/nuxt
|
||||
- ../api/deploy/pack/:/app/source/dynamix.unraid.net/usr/local/unraid-api
|
||||
- ../api/deploy/packed-pnpm-store.txz:/app/packed-pnpm-store.txz
|
||||
- ../api/deploy/packed-node-modules.tar.xz:/app/packed-node-modules.tar.xz
|
||||
stdin_open: true # equivalent to -i
|
||||
tty: true # equivalent to -t
|
||||
environment:
|
||||
|
||||
@@ -152,8 +152,9 @@ exit 0
|
||||
# deprecated Apr 2025. kept to remove unused archives for users upgrading from versioned node downloads.
|
||||
find /boot/config/plugins/dynamix.my.servers/ -name "node-v*-linux-x64.tar.xz" ! -name "${NODE_FILE}" -delete
|
||||
|
||||
# Remove stale pnpm store archives from the boot drive
|
||||
# Remove stale pnpm store and node_modules archives from the boot drive
|
||||
find /boot/config/plugins/dynamix.my.servers/ -name "pnpm-store-for-v*.txz" ! -name "${VENDOR_ARCHIVE}" -delete
|
||||
find /boot/config/plugins/dynamix.my.servers/ -name "node_modules-for-v*.tar.xz" ! -name "${VENDOR_ARCHIVE}" -delete
|
||||
|
||||
# Remove the legacy node directory
|
||||
rm -rf /usr/local/node
|
||||
@@ -811,11 +812,8 @@ cp -f "${PNPM_BINARY_FILE}" /usr/local/bin/pnpm
|
||||
chmod +x /usr/local/bin/pnpm
|
||||
|
||||
/etc/rc.d/rc.unraid-api restore-dependencies "$VENDOR_ARCHIVE"
|
||||
/etc/rc.d/rc.unraid-api pnpm-install
|
||||
echo
|
||||
echo "About to start the Unraid API"
|
||||
|
||||
|
||||
echo "Starting flash backup (if enabled)"
|
||||
logger "Starting flash backup (if enabled)"
|
||||
echo "/etc/rc.d/rc.flash_backup start" | at -M now &>/dev/null
|
||||
. /root/.bashrc
|
||||
|
||||
@@ -8,7 +8,7 @@ flash="/boot/config/plugins/dynamix.my.servers"
|
||||
[[ ! -d "${flash}" ]] && echo "Please reinstall the Unraid Connect plugin" && exit 1
|
||||
[[ ! -f "${flash}/env" ]] && echo 'env=production' >"${flash}/env"
|
||||
unraid_binary_path="/usr/local/bin/unraid-api"
|
||||
pnpm_store_dir="/usr/.pnpm-store"
|
||||
dependencies_dir="/usr/local/unraid-api/node_modules"
|
||||
|
||||
# Placeholder functions for plugin installation/uninstallation
|
||||
install() {
|
||||
@@ -18,53 +18,13 @@ uninstall() {
|
||||
true
|
||||
}
|
||||
|
||||
# Creates a backup of the global pnpm store directory
|
||||
# Args:
|
||||
# $1 - Path to the backup file (tar.xz format)
|
||||
# Returns:
|
||||
# 0 on success, 1 on failure
|
||||
backup_pnpm_store() {
|
||||
# Check if backup file path is provided
|
||||
if [ -z "$1" ]; then
|
||||
echo "Error: Backup file path is required"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local backup_file="$1"
|
||||
|
||||
# Check if pnpm command exists
|
||||
if ! command -v pnpm >/dev/null 2>&1; then
|
||||
echo "pnpm is not installed. Skipping backup."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Determine the global pnpm store directory
|
||||
mkdir -p "$pnpm_store_dir"
|
||||
|
||||
echo "Backing up pnpm store from '$pnpm_store_dir' to '$backup_file'"
|
||||
|
||||
# Create a tar.gz archive of the global pnpm store
|
||||
if tar -cJf "$backup_file" -C "$(dirname "$pnpm_store_dir")" "$(basename "$pnpm_store_dir")"; then
|
||||
echo "pnpm store backup completed successfully."
|
||||
else
|
||||
echo "Error: Failed to create pnpm store backup."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Restores the pnpm store from a backup file
|
||||
# Restores the node_modules directory from a backup file
|
||||
# Args:
|
||||
# $1 - Path to the backup file (tar.xz format)
|
||||
# Returns:
|
||||
# 0 on success, 1 on failure
|
||||
# Note: Requires 1.5x the backup size in free space for safe extraction
|
||||
restore_pnpm_store() {
|
||||
# Check if pnpm command exists
|
||||
if ! command -v pnpm >/dev/null 2>&1; then
|
||||
echo "pnpm is not installed. Cannot restore store."
|
||||
return 1
|
||||
fi
|
||||
|
||||
restore_dependencies() {
|
||||
local backup_file="$1"
|
||||
# Check if backup file exists
|
||||
if [ ! -f "$backup_file" ]; then
|
||||
@@ -76,7 +36,7 @@ restore_pnpm_store() {
|
||||
local backup_size
|
||||
backup_size=$(stat -c%s "$backup_file")
|
||||
local dest_space
|
||||
dest_space=$(df --output=avail "$(dirname "$pnpm_store_dir")" | tail -n1)
|
||||
dest_space=$(df --output=avail "$(dirname "$dependencies_dir")" | tail -n1)
|
||||
dest_space=$((dest_space * 1024)) # Convert KB to bytes
|
||||
|
||||
# Require 1.5x the backup size for safe extraction
|
||||
@@ -87,53 +47,48 @@ restore_pnpm_store() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Restoring pnpm store from '$backup_file' to '$pnpm_store_dir'"
|
||||
echo "Restoring node_modules from '$backup_file' to '$dependencies_dir'"
|
||||
# Remove existing store directory if it exists and ensure its parent directory exists
|
||||
rm -rf "$pnpm_store_dir"
|
||||
mkdir -p "$(dirname "$pnpm_store_dir")"
|
||||
rm -rf "$dependencies_dir"
|
||||
mkdir -p "$(dirname "$dependencies_dir")"
|
||||
|
||||
# Extract directly to final location
|
||||
if ! tar -xJf "$backup_file" -C "$(dirname "$pnpm_store_dir")" --preserve-permissions; then
|
||||
if ! tar -xJf "$backup_file" -C "$(dirname "$dependencies_dir")" --preserve-permissions; then
|
||||
echo "Error: Failed to extract backup to final location."
|
||||
rm -rf "$pnpm_store_dir"
|
||||
rm -rf "$dependencies_dir"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "pnpm store restored successfully."
|
||||
echo "node_modules restored successfully."
|
||||
}
|
||||
|
||||
# Executes pnpm install with production dependencies and offline preference
|
||||
# Captures and logs build script warnings to a dedicated log file at /var/log/unraid-api/build-scripts.log
|
||||
# Archives the node_modules directory to a specified location
|
||||
# Args: none
|
||||
# Output: Streams install progress and logs build script warnings
|
||||
run_pnpm_install() {
|
||||
local log_file="/var/log/unraid-api/build-scripts.log"
|
||||
stdbuf -oL pnpm install --prod --prefer-offline --reporter=append-only 2>&1 | sed -e "/^╭ Warning/,/^╰/w $log_file" -e "/^╭ Warning/,/^╰/c\Build scripts completed. See $log_file for details."
|
||||
echo "Note: This warning is expected. Build scripts are intentionally ignored for security and performance reasons." >> "$log_file"
|
||||
}
|
||||
|
||||
# Installs production dependencies for the unraid-api using pnpm. Prefers offline mode.
|
||||
# Uses the api_base_directory variable or defaults to /usr/local/unraid-api
|
||||
# Returns:
|
||||
# 0 on success, 1 on failure
|
||||
pnpm_install_unraid_api() {
|
||||
# Check if pnpm command exists
|
||||
if ! command -v pnpm >/dev/null 2>&1; then
|
||||
echo "Error: pnpm command not found. Cannot install dependencies."
|
||||
archive_dependencies() {
|
||||
local source_dir="/usr/local/unraid-api/node_modules"
|
||||
local dest_dir="/boot/config/plugins/dynamix.my.servers"
|
||||
local archive_file="${dest_dir}/node_modules.tar.xz"
|
||||
|
||||
# Check if source directory exists
|
||||
if [ ! -d "$source_dir" ]; then
|
||||
echo "Error: Source node_modules directory '$source_dir' does not exist."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Use the api_base_directory variable if set, otherwise default to /usr/local/unraid-api
|
||||
local unraid_api_dir="${api_base_directory:-/usr/local/unraid-api}"
|
||||
if [ ! -d "$unraid_api_dir" ]; then
|
||||
echo "Error: unraid API directory '$unraid_api_dir' does not exist."
|
||||
# Create destination directory if it doesn't exist
|
||||
mkdir -p "$dest_dir"
|
||||
|
||||
echo "Archiving node_modules from '$source_dir' to '$archive_file'"
|
||||
|
||||
# Create archive with XZ compression level 5, preserving symlinks
|
||||
if XZ_OPT=-5 tar -cJf "$archive_file" -C "$(dirname "$source_dir")" "$(basename "$source_dir")"; then
|
||||
echo "node_modules archive created successfully."
|
||||
else
|
||||
echo "Error: Failed to create node_modules archive."
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Executing 'pnpm install' in $unraid_api_dir"
|
||||
rm -rf /usr/local/unraid-api/node_modules
|
||||
# Run pnpm install in a subshell to prevent changing the current working directory of the script
|
||||
(cd "$unraid_api_dir" && run_pnpm_install)
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
@@ -146,14 +101,11 @@ case "$1" in
|
||||
'uninstall')
|
||||
uninstall
|
||||
;;
|
||||
'pnpm-install')
|
||||
pnpm_install_unraid_api
|
||||
;;
|
||||
'backup-dependencies')
|
||||
backup_pnpm_store "$2"
|
||||
;;
|
||||
'restore-dependencies')
|
||||
restore_pnpm_store "$2"
|
||||
restore_dependencies "$2"
|
||||
;;
|
||||
'archive-dependencies')
|
||||
archive_dependencies
|
||||
;;
|
||||
*)
|
||||
# Pass all other commands to unraid-api
|
||||
|
||||
Reference in New Issue
Block a user