diff --git a/app/utils/extension.js b/app/utils/extension.js index bc850464..1e65d1c1 100644 --- a/app/utils/extension.js +++ b/app/utils/extension.js @@ -4,6 +4,7 @@ import _ from "lodash"; // Local imports +import { isCloudMode } from "@ogw_front/utils/stores"; import { useAppStore } from "@ogw_front/stores/app"; import { useInfraStore } from "@ogw_front/stores/infra"; @@ -105,9 +106,9 @@ function runExtensions() { const { projectFolderPath } = appStore; const { PROJECT: projectName } = useRuntimeConfig().public; const params = { projectFolderPath, projectName }; - + const endpoint = isCloudMode() ? "cloud" : "local"; const schema = { - $id: "/api/microservice/extensions/run", + $id: `/api/${endpoint}/extensions/run`, methods: ["POST"], type: "object", properties: { @@ -130,7 +131,7 @@ function killExtension(extensionId) { console.log(`[AppStore] Killing extension: ${extensionId}`, { params }); const schema = { - $id: "/api/microservice/extensions/kill", + $id: "/api/local/extensions/kill", methods: ["POST"], type: "object", properties: { diff --git a/server/api/cloud/extensions/run.post.js b/server/api/cloud/extensions/run.post.js new file mode 100644 index 00000000..43a31c53 --- /dev/null +++ b/server/api/cloud/extensions/run.post.js @@ -0,0 +1,72 @@ +// Node imports +import fs from "node:fs"; + +// Third party imports +import { createError, defineEventHandler, readBody } from "h3"; + +// Local imports +import { + addMicroserviceMetadatas, + runExtension, +} from "@geode/opengeodeweb-front/server/utils/microservices.js"; +import { + extensionBackendPath, + extensionFolderPath, +} from "@geode/opengeodeweb-front/server/utils/path.js"; +import { + readExtensionFrontend, + readExtensionMetadata, +} from "@geode/opengeodeweb-front/server/utils/extension.js"; +import { extensionsConf } from "@geode/opengeodeweb-front/server/utils/app_config.js"; +import { unzipFile } from "@geode/opengeodeweb-front/server/utils/server.js"; + +export default defineEventHandler(async (event) => { + try { + console.log("NITRO: runExtensions", event); + const { projectFolderPath, projectName } = await readBody(event); + const extensionsConfig = extensionsConf(projectName); + const extensionsArray = await Promise.all( + Object.keys(extensionsConfig).map(async (extensionId) => { + const extensionPath = extensionsConfig[extensionId].path; + const unzippedExtensionPath = await unzipFile( + extensionPath, + extensionFolderPath(projectFolderPath, extensionId), + ); + const { id, name, version, backendExecutable, frontendFile } = + await readExtensionMetadata(unzippedExtensionPath); + const frontendContent = await readExtensionFrontend( + unzippedExtensionPath, + frontendFile, + id, + ); + fs.chmodSync(extensionBackendPath(unzippedExtensionPath, backendExecutable), "755"); + const port = await runExtension(id, backendExecutable, unzippedExtensionPath, { + projectFolderPath, + }); + await addMicroserviceMetadatas(projectFolderPath, { + type: "back", + name, + port, + }); + return { + id, + name, + version, + frontendContent, + port, + }; + }), + ); + + return { + statusCode: 200, + extensionsArray, + }; + } catch (error) { + console.error("Error running extensions:", error); + throw createError({ + statusCode: 500, + statusMessage: error.message, + }); + } +}); diff --git a/server/api/microservice/extensions/kill.post.js b/server/api/local/extensions/kill.post.js similarity index 100% rename from server/api/microservice/extensions/kill.post.js rename to server/api/local/extensions/kill.post.js diff --git a/server/api/microservice/extensions/run.post.js b/server/api/local/extensions/run.post.js similarity index 52% rename from server/api/microservice/extensions/run.post.js rename to server/api/local/extensions/run.post.js index 537b153a..562c2256 100644 --- a/server/api/microservice/extensions/run.post.js +++ b/server/api/local/extensions/run.post.js @@ -1,6 +1,5 @@ // Node imports import fs from "node:fs"; -import path from "node:path"; // Third party imports import { createError, defineEventHandler, readBody } from "h3"; @@ -11,20 +10,21 @@ import { runBack, } from "@geode/opengeodeweb-front/server/utils/microservices.js"; import { - executableName, + extensionBackendPath, extensionFolderPath, - extensionFrontendPath, } from "@geode/opengeodeweb-front/server/utils/path.js"; +import { + readExtensionFrontend, + readExtensionMetadata, +} from "@geode/opengeodeweb-front/server/utils/extension.js"; import { extensionsConf } from "@geode/opengeodeweb-front/server/utils/app_config.js"; import { unzipFile } from "@geode/opengeodeweb-front/server/utils/server.js"; export default defineEventHandler(async (event) => { try { console.log("NITRO: runExtensions", event); - const body = await readBody(event); - const { projectFolderPath, projectName } = body; + const { projectFolderPath, projectName } = await readBody(event); const extensionsConfig = extensionsConf(projectName); - const extensionsArray = await Promise.all( Object.keys(extensionsConfig).map(async (extensionId) => { const extensionPath = extensionsConfig[extensionId].path; @@ -32,50 +32,14 @@ export default defineEventHandler(async (event) => { extensionPath, extensionFolderPath(projectFolderPath, extensionId), ); - const metadataPath = path.join(unzippedExtensionPath, "metadata.json"); - const metadataContent = await fs.promises.readFile(metadataPath, "utf8"); - - if (!metadataContent) { - throw createError({ - statusCode: 400, - statusMessage: "Invalid extension file: missing metadata.json", - }); - } - const metadata = JSON.parse(metadataContent); - console.log("runExtensions", { metadata }); - - const { id, name, version, backendExecutable, frontendFile } = metadata; - console.log("runExtensions", { id, name, version, backendExecutable }); - - if (!frontendFile) { - throw createError({ - statusCode: 400, - statusMessage: "Invalid extension file: missing frontend JavaScript", - }); - } - if (!backendExecutable) { - throw createError({ - statusCode: 400, - statusMessage: "Invalid extension file: missing backend executable", - }); - } - - const frontendFilePath = await extensionFrontendPath( + const { id, name, version, backendExecutable, frontendFile } = + await readExtensionMetadata(unzippedExtensionPath); + const frontendContent = await readExtensionFrontend( unzippedExtensionPath, frontendFile, - path.resolve(), id, ); - - console.log("runExtensions", { frontendFilePath }); - const frontendContent = await fs.promises.readFile(frontendFilePath, "utf8"); - - const backendExecutablePath = path.join( - unzippedExtensionPath, - executableName(backendExecutable), - ); - console.log("runExtensions", { backendExecutablePath }); - fs.chmodSync(backendExecutablePath, "755"); + fs.chmodSync(extensionBackendPath(unzippedExtensionPath, backendExecutable), "755"); const port = await runBack(backendExecutable, unzippedExtensionPath, { projectFolderPath, }); diff --git a/server/utils/cloud.js b/server/utils/cloud.js index 589ae4ca..414cf33b 100644 --- a/server/utils/cloud.js +++ b/server/utils/cloud.js @@ -5,6 +5,8 @@ import { google } from "googleapis"; // Local imports +const LOCATIONS_DIR = "/etc/nginx/locations"; + async function artifactImage(parent, authClient) { const projectName = process.env.PROJECT; const registry = google.artifactregistry({ @@ -92,4 +94,108 @@ function requestConfig(parent, image, email, projectName) { }; } -export { artifactImage, requestConfig }; +function addSupervisorProgram(name, command, executableArgs) { + const conf = ` +[program:${name}] +command=${command} ${executableArgs.join(" ")} +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +`; + const confPath = path.join("/etc/supervisor/conf.d", `${name}.conf`); + fs.writeFileSync(confPath, conf); + execFileSync("supervisorctl", ["reread"]); + execFileSync("supervisorctl", ["update"]); + const stdout = execFileSync("supervisorctl", ["start", name]); + console.log("addSupervisorProgram", stdout); +} + +function buildLocationBlock(routePath, port) { + if (!routePath.startsWith("/") || !routePath.endsWith("/")) { + throw new Error(`routePath must start and end with '/', got: ${routePath}`); + } + const methods = "GET, POST, PUT, DELETE, PATCH, OPTIONS"; + const limitMethods = methods + .split(",") + .map((method) => method.trim()) + .filter((method) => method !== "OPTIONS") + .join(" "); + + return `# ====================== ${routePath} location ====================== +location ~ "^${routePath}" { + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' $allow_origin always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' '${methods}' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization,X-CSRF-Token' always; + add_header 'Access-Control-Max-Age' 1728000 always; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + + limit_except ${limitMethods} { deny all; } + + add_header 'Access-Control-Allow-Origin' $allow_origin always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' '${methods}' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization,X-CSRF-Token' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + add_header 'Vary' 'Origin' always; + + rewrite "^${routePath}(.*)" /$1 break; + proxy_pass http://localhost:${port}; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} +`; +} + +function nginxConfigFile(name) { + return path.join(LOCATIONS_DIR, `${name}.conf`); +} + +function nginxReload() { + execFileSync("nginx", ["-t"]); + execFileSync("nginx", ["-s", "reload"]); +} + +function addNginxLocation(name, port) { + fs.mkdirSync(LOCATIONS_DIR, { recursive: true }); + const filePath = nginxConfigFile(name); + if (fs.existsSync(filePath)) { + throw new Error(`Location '${name}' already exists at ${filePath}`); + } + fs.writeFileSync(filePath, buildLocationBlock(`/${name}/`, port)); + try { + nginxReload(); + } catch (error) { + fs.unlinkSync(filePath); + throw error; + } + return filePath; +} + +function removeNginxLocation(name) { + const filePath = filePathFor(name); + if (!fs.existsSync(filePath)) { + return false; + } + fs.unlinkSync(filePath); + nginxReload(); + return true; +} + +export { + addNginxLocation, + addSupervisorProgram, + artifactImage, + removeNginxLocation, + requestConfig, +}; diff --git a/server/utils/extension.js b/server/utils/extension.js new file mode 100644 index 00000000..ce6b9bcf --- /dev/null +++ b/server/utils/extension.js @@ -0,0 +1,50 @@ +// Node imports +import { promises as fs } from "node:fs"; +import path from "node:path"; + +// Third party imports +import { createError } from "h3"; + +// Local imports +import { extensionFrontendPath } from "@geode/opengeodeweb-front/server/utils/path.js"; + +async function readExtensionMetadata(unzippedExtensionPath) { + const metadataPath = path.join(unzippedExtensionPath, "metadata.json"); + const metadataContent = await fs.readFile(metadataPath, "utf8"); + if (!metadataContent) { + throw createError({ + statusCode: 400, + statusMessage: "Invalid extension file: missing metadata.json", + }); + } + const metadata = JSON.parse(metadataContent); + console.log("readExtensionMetadata", { metadata }); + if (!metadata.frontendFile) { + throw createError({ + statusCode: 400, + statusMessage: "Invalid extension file: missing frontend JavaScript", + }); + } + if (!metadata.backendExecutable) { + throw createError({ + statusCode: 400, + statusMessage: "Invalid extension file: missing backend executable", + }); + } + + return metadata; +} + +async function readExtensionFrontend(unzippedExtensionPath, frontendFile, id) { + console.log("readExtensionFrontend", { id }); + const frontendFilePath = await extensionFrontendPath( + unzippedExtensionPath, + frontendFile, + path.resolve(), + id, + ); + console.log("readExtensionFrontend", { frontendFilePath }); + return fs.readFile(frontendFilePath, "utf8"); +} + +export { readExtensionFrontend, readExtensionMetadata }; diff --git a/server/utils/microservices.js b/server/utils/microservices.js index ee431633..e1e337e3 100644 --- a/server/utils/microservices.js +++ b/server/utils/microservices.js @@ -7,6 +7,7 @@ import path from "node:path"; import back_schemas from "@geode/opengeodeweb-back/opengeodeweb_back_schemas.json" with { type: "json" }; // Local imports +import { addNginxLocation, addSupervisorProgram } from "./cloud.js"; import { getAvailablePort, waitForReady } from "./scripts.js"; import { microservicesMetadatasPath, projectMicroservices } from "./cleanup.js"; import { executablePath } from "./path.js"; @@ -57,33 +58,11 @@ function isPortInUseError(errorMessage) { } async function runBack(execName, execPath, args = {}, attempts = 0) { - const { projectFolderPath } = args; - if (!projectFolderPath) { - throw new Error("projectFolderPath is required"); - } - let { uploadFolderPath } = args; - if (!uploadFolderPath) { - uploadFolderPath = path.join(projectFolderPath, "uploads"); - } try { const port = await getAvailablePort(); - const backArgs = [ - "--port", - String(port), - "--project_folder_path", - projectFolderPath, - "--upload_folder_path", - uploadFolderPath, - "--allowed_origins", - "http://localhost:*", - "--timeout", - "0", - ]; - if (process.env.NODE_ENV === "development" || !process.env.NODE_ENV) { - backArgs.push("--debug"); - } - console.log("runBack", execPath, execName, backArgs); - await runScript(execPath, execName, backArgs, "Serving Flask app"); + const executableArgs = backArgs(args, port); + console.log("runBack", execPath, execName, executableArgs); + await runScript(execPath, execName, executableArgs, "Serving Flask app"); return port; } catch (error) { if (!isPortInUseError(error)) { @@ -129,6 +108,52 @@ async function runViewer(execName, execPath, args = {}, attempts = 0) { } } +function backArgs(args, port) { + const { projectFolderPath } = args; + if (!projectFolderPath) { + throw new Error("projectFolderPath is required"); + } + const uploadFolderPath = args.uploadFolderPath || path.join(projectFolderPath, "uploads"); + const executableArgs = [ + "--port", + String(port), + "--project_folder_path", + projectFolderPath, + "--upload_folder_path", + uploadFolderPath, + "--allowed_origins", + "http://localhost:*", + "--timeout", + "0", + ]; + if (process.env.NODE_ENV === "development" || !process.env.NODE_ENV) { + executableArgs.push("--debug"); + } + return executableArgs; +} + +async function runExtension(extensionId, execName, execPath, args = {}, attempts = 0) { + try { + const port = await getAvailablePort(); + const executableArgs = backArgs(args, port); + const command = executablePath(execPath, execName); + console.log("runExtension", execPath, execName, executableArgs); + addSupervisorProgram(extensionId, command, executableArgs); + addNginxLocation(extensionId, port); + return port; + } catch (error) { + if (!isPortInUseError(error)) { + console.log("runBack error", error); + throw error; + } + if (attempts <= MAX_PORT_RETRIES) { + console.log("Retrying runExtension on conflicting port", port); + const port = await runExtension(extensionId, execName, execPath, args, attempts + 1); + return port; + } + } +} + function addMicroserviceMetadatas(projectFolderPath, serviceObj) { const microservices = projectMicroservices(projectFolderPath); if (serviceObj.type === "back") { @@ -147,4 +172,4 @@ function addMicroserviceMetadatas(projectFolderPath, serviceObj) { ); } -export { addMicroserviceMetadatas, runBack, runViewer }; +export { addMicroserviceMetadatas, runBack, runExtension, runViewer }; diff --git a/server/utils/path.js b/server/utils/path.js index b34b7670..0f51a781 100644 --- a/server/utils/path.js +++ b/server/utils/path.js @@ -138,8 +138,18 @@ async function extensionFrontendPath(unzippedExtensionPath, frontendFile, rootPa throw new Error(`Failed to find ${unzippedfrontendFilePath}`); } +function extensionBackendPath(unzippedExtensionPath, backendExecutableName) { + const backendExecutablePath = path.join( + unzippedExtensionPath, + executableName(backendExecutableName), + ); + console.log("runExtensions", { backendExecutablePath }); + return backendExecutablePath; +} + export { createPath, + extensionBackendPath, extensionFrontendPath, extensionFolderPath, executablePath,