feat: split plugin builds

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced containerized plugin deployment support with updated Docker
Compose configurations.
- Added continuous build watch modes for API, web, and UI components for
smoother development iterations.
  - Added a new job for API testing in the CI/CD workflow.
- Added a new shell script to determine the local host's IP address for
Docker configurations.
- Introduced a new entry point and HTTP server setup in the plugin's
Docker environment.
- Added new scripts for building and watching plugin changes in
real-time.
- Added a new script for building the project in watch mode for the API
and UI components.

- **Improvements**
- Streamlined the plugin installation process and refined release
workflows for a more reliable user experience.
- Enhanced overall CI/CD pipelines to ensure efficient, production-ready
deployments.
- Updated artifact upload paths and job definitions for clarity and
efficiency.
- Implemented new utility functions for better URL management and
changelog generation.
- Modified the `.dockerignore` file to ignore all contents within the
`node_modules` directory.
- Added new constants and functions for managing plugin paths and
configurations.
- Updated the build process in the Dockerfile to focus on release
operations.

- **Tests**
- Expanded automated testing to validate environment setups and build
stability, ensuring higher reliability during updates.
- Introduced new test suites for validating plugin environment setups
and configurations.
- Added tests for validating environment variables and handling of
manifest files.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Michael Datelle <mdatelle@icloud.com>
Co-authored-by: mdatelle <mike@datelle.net>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Pujit Mehrotra <pujit@lime-technology.com>

plugin/scripts/build-plugin-and-txz.ts

@@ -1,328 +0,0 @@
-import { execSync } from "child_process";
-import { cp, readFile, writeFile, mkdir, readdir } from "fs/promises";
-import { basename, join } from "path";
-import { createHash } from "node:crypto";
-import { $, cd } from "zx";
-import conventionalChangelog from "conventional-changelog";
-import { escape as escapeHtml } from "html-sloppy-escaper";
-import { existsSync } from "fs";
-import { format as formatDate } from "date-fns";
-import { setupEnvironment } from "./setup-environment";
-import { dirname } from "node:path";
-const pluginName = "dynamix.unraid.net" as const;
-const startingDir = process.cwd();
-
-const validatedEnv = await setupEnvironment(startingDir);
-
-const BASE_URLS = {
-  STABLE: "https://stable.dl.unraid.net/unraid-api",
-  PREVIEW: "https://preview.dl.unraid.net/unraid-api",
-  ...(validatedEnv.LOCAL_FILESERVER_URL
-    ? { LOCAL: validatedEnv.LOCAL_FILESERVER_URL }
-    : {}),
-} as const;
-
-// Setup environment variables
-// Ensure that git is available
-
-try {
-  await $`git log -1 --pretty=%B`;
-} catch (err) {
-  console.error(`Error: git not available: ${err}`);
-  process.exit(1);
-}
-
-const createBuildDirectory = async () => {
-  await execSync(`rm -rf deploy/pre-pack/*`);
-  await execSync(`rm -rf deploy/release/*`);
-  await execSync(`rm -rf deploy/test/*`);
-  await mkdir("deploy/pre-pack", { recursive: true });
-  await mkdir("deploy/release/plugins", { recursive: true });
-  await mkdir("deploy/release/archive", { recursive: true });
-  await mkdir("deploy/test", { recursive: true });
-};
-
-function updateEntityValue(
-  xmlString: string,
-  entityName: string,
-  newValue: string
-) {
-  const regex = new RegExp(`<!ENTITY ${entityName} "[^"]*">`);
-  if (regex.test(xmlString)) {
-    return xmlString.replace(regex, `<!ENTITY ${entityName} "${newValue}">`);
-  }
-  throw new Error(`Entity ${entityName} not found in XML`);
-}
-
-const validateSourceDir = async () => {
-  console.log("Validating TXZ source directory");
-  const sourceDir = join(startingDir, "source");
-  if (!existsSync(sourceDir)) {
-    throw new Error(`Source directory ${sourceDir} does not exist`);
-  }
-  // Validate existence of webcomponent files:
-  // source/dynamix.unraid.net/usr/local/emhttp/plugins/dynamix.my.servers/unraid-components
-  const webcomponentDir = join(
-    sourceDir,
-    "dynamix.unraid.net",
-    "usr",
-    "local",
-    "emhttp",
-    "plugins",
-    "dynamix.my.servers",
-    "unraid-components"
-  );
-  if (!existsSync(webcomponentDir)) {
-    throw new Error(`Webcomponent directory ${webcomponentDir} does not exist`);
-  }
-  // Validate that there are webcomponents
-  const webcomponents = await readdir(webcomponentDir);
-  if (webcomponents.length === 1 && webcomponents[0] === ".gitkeep") {
-    throw new Error(`No webcomponents found in ${webcomponentDir}`);
-  }
-  // Check for the existence of "ui.manifest.json" as well as "manifest.json" in webcomponents
-  if (
-    !webcomponents.includes("ui.manifest.json") ||
-    !webcomponents.includes("manifest.json")
-  ) {
-    throw new Error(
-      `Webcomponents must contain both "ui.manifest.json" and "manifest.json"`
-    );
-  }
-
-  const apiDir = join(
-    startingDir,
-    "source/dynamix.unraid.net/usr/local/unraid-api/package.json"
-  );
-  if (!existsSync(apiDir)) {
-    throw new Error(`API directory ${apiDir} does not exist`);
-  }
-};
-
-const buildTxz = async (
-  version: string
-): Promise<{
-  txzName: string;
-  txzSha256: string;
-}> => {
-  if (
-    validatedEnv.SKIP_VALIDATION !== "true" ||
-    validatedEnv.LOCAL_FILESERVER_URL
-  ) {
-    await validateSourceDir();
-  }
-
-  const txzName = `${pluginName}-${version}.txz`;
-  const txzPath = join(startingDir, "deploy/release/archive", txzName);
-  const prePackDir = join(startingDir, "deploy/pre-pack");
-
-  // Copy all files from source to temp dir, excluding specific files
-  await cp(join(startingDir, "source/dynamix.unraid.net"), prePackDir, {
-    recursive: true,
-    filter: (src) => {
-      const filename = basename(src);
-      return ![
-        ".DS_Store",
-        "pkg_build.sh",
-        "makepkg",
-        "explodepkg",
-        "sftp-config.json",
-        ".gitkeep",
-      ].includes(filename);
-    },
-  });
-
-  // Create package - must be run from within the pre-pack directory
-  // Use cd option to run command from prePackDir
-  await cd(prePackDir);
-  $.verbose = true;
-
-  await $`${join(
-    startingDir,
-    "scripts/makepkg"
-  )} -l y -c y --compress -1 "${txzPath}"`;
-  $.verbose = false;
-  await cd(startingDir);
-
-  // Calculate hashes
-  const sha256 = createHash("sha256")
-    .update(await readFile(txzPath))
-    .digest("hex");
-  console.log(`TXZ SHA256: ${sha256}`);
-
-  return { txzSha256: sha256, txzName };
-};
-
-const getStagingChangelogFromGit = async (
-  apiVersion: string,
-  tag: string | null = null
-): Promise<string | null> => {
-  console.debug("Getting changelog from git" + (tag ? " for TAG" : ""));
-  try {
-    const changelogStream = conventionalChangelog(
-      {
-        preset: "conventionalcommits",
-      },
-      {
-        version: apiVersion,
-      },
-      tag
-        ? {
-            from: "origin/main",
-            to: "HEAD",
-          }
-        : {},
-      undefined,
-      tag
-        ? {
-            headerPartial: `## [${tag}](https://github.com/unraid/api/${tag})\n\n`,
-          }
-        : undefined
-    );
-    let changelog = "";
-    for await (const chunk of changelogStream) {
-      changelog += chunk;
-    }
-    // Encode HTML entities using the 'he' library
-    return escapeHtml(changelog) ?? null;
-  } catch (err) {
-    console.error(`Error: failed to get changelog from git: ${err}`);
-    process.exit(1);
-  }
-};
-
-const buildPlugin = async ({
-  type,
-  txzSha256,
-  txzName,
-  version,
-  tag = "",
-  apiVersion,
-}: {
-  type: "staging" | "pr" | "production" | "local";
-  txzSha256: string;
-  txzName: string;
-  version: string;
-  tag?: string;
-  apiVersion: string;
-}) => {
-  const rootPlgFile = join(startingDir, "/plugins/", `${pluginName}.plg`);
-  // Set up paths
-  const newPluginFile = join(
-    startingDir,
-    "/deploy/release/plugins/",
-    type,
-    `${pluginName}.plg`
-  );
-
-  // Define URLs
-  let PLUGIN_URL = "";
-  let MAIN_TXZ = "";
-  let RELEASE_NOTES: string | null = null;
-  switch (type) {
-    case "production":
-      PLUGIN_URL = `${BASE_URLS.STABLE}/${pluginName}.plg`;
-      MAIN_TXZ = `${BASE_URLS.STABLE}/${txzName}`;
-      break;
-    case "pr":
-      PLUGIN_URL = `${BASE_URLS.PREVIEW}/tag/${tag}/${pluginName}.plg`;
-      MAIN_TXZ = `${BASE_URLS.PREVIEW}/tag/${tag}/${txzName}`;
-      RELEASE_NOTES = await getStagingChangelogFromGit(apiVersion, tag);
-      break;
-    case "staging":
-      PLUGIN_URL = `${BASE_URLS.PREVIEW}/${pluginName}.plg`;
-      MAIN_TXZ = `${BASE_URLS.PREVIEW}/${txzName}`;
-      RELEASE_NOTES = await getStagingChangelogFromGit(apiVersion);
-      break;
-    case "local":
-      PLUGIN_URL = `${BASE_URLS.LOCAL}/plugins/${type}/${pluginName}.plg`;
-      MAIN_TXZ = `${BASE_URLS.LOCAL}/archive/${txzName}`;
-      RELEASE_NOTES = await getStagingChangelogFromGit(apiVersion, tag);
-      break;
-  }
-
-  // Update plg file
-  let plgContent = await readFile(rootPlgFile, "utf8");
-
-  // Update entity values
-  const entities: Record<string, string> = {
-    name: pluginName,
-    env: type === "pr" ? "staging" : type,
-    version: version,
-    pluginURL: PLUGIN_URL,
-    SHA256: txzSha256,
-    MAIN_TXZ: MAIN_TXZ,
-    TAG: tag,
-    API_version: apiVersion,
-  };
-
-  // Iterate over entities and update them
-  Object.entries(entities).forEach(([key, value]) => {
-    if (key !== "TAG" && !value) {
-      throw new Error(`Entity ${key} not set in entities : ${value}`);
-    }
-    plgContent = updateEntityValue(plgContent, key, value);
-  });
-
-  if (RELEASE_NOTES) {
-    // Update the CHANGES section with release notes
-    plgContent = plgContent.replace(
-      /<CHANGES>.*?<\/CHANGES>/s,
-      `<CHANGES>\n${RELEASE_NOTES}\n</CHANGES>`
-    );
-  }
-
-  await mkdir(dirname(newPluginFile), { recursive: true });
-  await writeFile(newPluginFile, plgContent);
-  console.log(`${type} plugin: ${newPluginFile}`);
-};
-
-/**
- * Main build script
- */
-
-const main = async () => {
-  await createBuildDirectory();
-
-  const version = formatDate(new Date(), "yyyy.MM.dd.HHmm");
-  console.log(`Version: ${version}`);
-  const { txzSha256, txzName } = await buildTxz(version);
-  const { API_VERSION, TAG, LOCAL_FILESERVER_URL } = validatedEnv;
-
-  if (LOCAL_FILESERVER_URL) {
-    await buildPlugin({
-      type: "local",
-      txzSha256,
-      txzName,
-      version,
-      tag: TAG,
-      apiVersion: API_VERSION,
-    });
-  } else if (TAG) {
-    await buildPlugin({
-      type: "pr",
-      txzSha256,
-      txzName,
-      version,
-      tag: TAG,
-      apiVersion: API_VERSION,
-    });
-  }
-
-  await buildPlugin({
-    type: "staging",
-    txzSha256,
-    txzName,
-    version,
-    apiVersion: API_VERSION,
-  });
-  await buildPlugin({
-    type: "production",
-    txzSha256,
-    txzName,
-    version,
-    apiVersion: API_VERSION,
-  });
-};
-
-await main();

plugin/scripts/dc.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Get host IP based on platform
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # macOS
+    HOST_LAN_IP=$(ipconfig getifaddr en0 || ipconfig getifaddr en1 || echo "127.0.0.1")
+else
+    # Linux and others
+    HOST_LAN_IP=$(hostname -I | awk '{print $1}' || echo "127.0.0.1")
+fi
+
+# Verify we have a valid IP
+if [[ ! $HOST_LAN_IP =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "Error: Could not determine valid host IP address. Using localhost."
+    HOST_LAN_IP="127.0.0.1"
+fi
+
+CI=${CI:-false}
+TAG="LOCAL_PLUGIN_BUILD"
+docker compose run --service-ports --rm -e HOST_LAN_IP="$HOST_LAN_IP" -e CI="$CI" -e TAG="$TAG" plugin-builder "$@" 

plugin/scripts/entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+mkdir -p /app/deploy/
+# Start http-server with common fileserver settings
+http-server /app/deploy/ \
+    --port 8080 \
+    --host 0.0.0.0 \
+    --cors \
+    --gzip \
+    --brotli \
+    --no-dotfiles \
+    -c-1 \
+    --silent &
+
+# Execute whatever command was passed (or default CMD)
+exec "$@"

plugin/scripts/makepkg

@@ -381,8 +381,17 @@ fi
 if [ "$CHOWN" = "y" ]; then
   # Set strict mode and fail if commands fail
   set -e
-  find . -type d -exec sudo chmod 755 {} + || exit 1
-  find . -type d -exec sudo chown 0:0 {} + || exit 1
+  echo "Setting permissions and ownerships"
+  
+  # Use sudo if available, otherwise run directly
+  if command -v sudo >/dev/null 2>&1; then
+    SUDO="sudo"
+  else
+    SUDO=""
+  fi
+  
+  $SUDO find . -type d -exec chmod 755 {} + || exit 1
+  $SUDO find . -exec chown 0:0 {} + || exit 1
   set +e
 fi
 
@@ -416,7 +425,7 @@ rm -f ${TARGET_NAME}/${TAR_NAME}.${EXTENSION}
 # find ./ | sed '2,$s,^\./,,' | cpio --quiet -ovHustar > ${TARGET_NAME}/${TAR_NAME}.tar
 
 # Create the package:
-find ./ | LC_COLLATE=C sort | sed '2,$s,^\./,,' | tar --no-recursion $ACLS $XATTRS $MTIME -T - -cvf - | $COMPRESSOR > ${TARGET_NAME}/${TAR_NAME}.${EXTENSION}
+find ./ | LC_COLLATE=C sort | sed '2,$s,^\./,,' | tar --no-recursion $ACLS $XATTRS $MTIME -T - -cf - | $COMPRESSOR > ${TARGET_NAME}/${TAR_NAME}.${EXTENSION}
 ERRCODE=$?
 if [ ! $ERRCODE = 0 ]; then
   echo "ERROR:  $COMPRESSOR returned error code $ERRCODE -- makepkg failed."

plugin/scripts/setup-environment.ts

@@ -1,73 +0,0 @@
-import { readFile } from "fs/promises";
-import { join } from "path";
-import { z } from "zod";
-import { parse } from "semver";
-import { dotenv } from "zx";
-
-const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-const envSchema = z.object({
-  API_VERSION: z.string().refine((v) => {
-    return parse(v) ?? false;
-  }, "Must be a valid semver version"),
-  TAG: z
-    .string()
-    .optional(),
-  SKIP_VALIDATION: z
-    .string()
-    .optional()
-    .default("false")
-    .refine((v) => v === "true" || v === "false", "Must be true or false"),
-  LOCAL_FILESERVER_URL: z.string().url().optional(),
-});
-
-type Env = z.infer<typeof envSchema>;
-
-export const setupEnvironment = async (
-  startingDir: string
-): Promise<Env> => {
-  const getLocalEnvironmentVariablesFromApiFolder = async (): Promise<Partial<Env>> => {
-    const apiDir = join(
-      startingDir,
-      "source/dynamix.unraid.net/usr/local/unraid-api"
-    );
-    const apiPackageJson = join(apiDir, "package.json");
-    const apiPackageJsonContent = await readFile(apiPackageJson, "utf8");
-    const apiPackageJsonObject = JSON.parse(apiPackageJsonContent);
-    return {
-      API_VERSION: apiPackageJsonObject.version,
-    };
-  };
-
-  const validatedEnv = envSchema.parse(
-    {
-      ...process.env,
-      ...(await dotenv.config()),
-      ...(await getLocalEnvironmentVariablesFromApiFolder()),
-    }
-  );
-  let shouldWait = false;
-
-  if (validatedEnv.SKIP_VALIDATION == "true") {
-    console.warn("SKIP_VALIDATION is true, skipping validation");
-    shouldWait = true;
-  }
-
-  if (validatedEnv.TAG) {
-    console.warn("TAG is set, will generate a TAGGED build");
-    shouldWait = true;
-  }
-
-  if (validatedEnv.LOCAL_FILESERVER_URL) {
-    console.warn("LOCAL_FILESERVER_URL is set, will generate a local build");
-    shouldWait = true;
-  }
-
-  console.log("validatedEnv", validatedEnv);
-
-  if (shouldWait) {
-    await wait(1000);
-  }
-
-  return validatedEnv;
-};