1989 lines
68 KiB
TypeScript
1989 lines
68 KiB
TypeScript
import { execFile, spawn } from 'node:child_process';
|
|
import fs from 'node:fs';
|
|
import http from 'node:http';
|
|
import { mkdir, open, readFile, rm, stat } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { promisify } from 'node:util';
|
|
import { env } from '../config/env.js';
|
|
import { resolveMainProjectRoot } from './main-project-root-service.js';
|
|
import {
|
|
readTestServerDeploymentState,
|
|
startTestServerDeployment,
|
|
type TestServerDeploymentSnapshot,
|
|
} from './test-server-deployment-service.js';
|
|
import {
|
|
getRuntimeWorkServerBuildInfo,
|
|
readLatestWorkServerBuildInfo,
|
|
readLatestWorkServerSourceChange,
|
|
} from './work-server-build-service.js';
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
export const serverCommandKeys = ['test', 'rel', 'prod', 'work-server', 'command-runner'] as const;
|
|
|
|
export type ServerCommandKey = (typeof serverCommandKeys)[number];
|
|
|
|
type ServerDefinition = {
|
|
key: ServerCommandKey;
|
|
label: string;
|
|
summary: string;
|
|
environment: string;
|
|
publicUrl: string | null;
|
|
checkUrl: string;
|
|
composeFile: string;
|
|
serviceName: string;
|
|
containerName: string;
|
|
commandScript: string;
|
|
commandWorkingDirectory: string;
|
|
commandEnvironment: Record<string, string>;
|
|
restartStrategy: 'wait' | 'deferred';
|
|
deferredResponseMode?: 'wait-for-result' | 'accept-immediately';
|
|
};
|
|
|
|
export type ServerCommandSnapshot = {
|
|
key: ServerCommandKey;
|
|
label: string;
|
|
summary: string;
|
|
environment: string;
|
|
publicUrl: string | null;
|
|
checkUrl: string;
|
|
composeFile: string;
|
|
serviceName: string;
|
|
availability: 'online' | 'degraded' | 'offline';
|
|
httpStatus: number | null;
|
|
contentType: string | null;
|
|
responsePreview: string | null;
|
|
checkedAt: string;
|
|
startedAt: string | null;
|
|
runningVersion: string | null;
|
|
runningBuiltAt: string | null;
|
|
latestVersion: string | null;
|
|
latestBuiltAt: string | null;
|
|
latestSourceChangeAt: string | null;
|
|
latestSourceChangePath: string | null;
|
|
buildRequired: boolean;
|
|
updateAvailable: boolean;
|
|
updateSummary: string | null;
|
|
responseTimeMs: number | null;
|
|
composeStatus: string | null;
|
|
composeDetails: string | null;
|
|
lastCommand: string;
|
|
commandScript: string;
|
|
commandWorkingDirectory: string;
|
|
errorMessage: string | null;
|
|
deployment: WorkServerDeploymentSnapshot | null;
|
|
};
|
|
|
|
export type ServerCommandRestartResult = {
|
|
server: ServerCommandSnapshot;
|
|
commandOutput: string | null;
|
|
restartState: 'completed' | 'accepted';
|
|
deployment?: WorkServerDeploymentSnapshot | null;
|
|
testDeployment?: TestServerDeploymentSnapshot | null;
|
|
};
|
|
|
|
type ServerCommandScriptExecutionOptions = {
|
|
commandScript?: string;
|
|
environment?: Record<string, string>;
|
|
timeoutMs?: number;
|
|
};
|
|
|
|
type ExecFileFailure = Error & {
|
|
code?: number | string;
|
|
signal?: NodeJS.Signals | string | null;
|
|
stdout?: string;
|
|
stderr?: string;
|
|
};
|
|
|
|
type RemoteRestartPayload = {
|
|
server?: ServerCommandSnapshot;
|
|
commandOutput: string | null;
|
|
restartState: 'completed' | 'accepted';
|
|
};
|
|
|
|
type HealthCheckAttempt = {
|
|
url: string;
|
|
httpStatus: number | null;
|
|
contentType: string | null;
|
|
responsePreview: string | null;
|
|
availability: ServerCommandSnapshot['availability'];
|
|
errorMessage: string | null;
|
|
};
|
|
|
|
type RuntimeInspectionResult = {
|
|
startedAt: string | null;
|
|
composeStatus: string | null;
|
|
composeDetails: string | null;
|
|
availability?: ServerCommandSnapshot['availability'];
|
|
responsePreview?: string | null;
|
|
errorMessage?: string | null;
|
|
};
|
|
|
|
type BuildInspectionResult = {
|
|
runningVersion: string | null;
|
|
runningBuiltAt: string | null;
|
|
latestVersion: string | null;
|
|
latestBuiltAt: string | null;
|
|
latestSourceChangeAt: string | null;
|
|
latestSourceChangePath: string | null;
|
|
buildRequired: boolean;
|
|
updateAvailable: boolean;
|
|
updateSummary: string | null;
|
|
};
|
|
|
|
type WorkServerSlot = 'blue' | 'green';
|
|
|
|
export type WorkServerDeploymentStepKey =
|
|
| 'build-target-slot'
|
|
| 'verify-target-health'
|
|
| 'switch-proxy'
|
|
| 'drain-previous-slot'
|
|
| 'rebuild-previous-slot'
|
|
| 'recover-interrupted-chat';
|
|
|
|
export type WorkServerDeploymentStepStatus = 'pending' | 'running' | 'completed' | 'failed';
|
|
|
|
export type WorkServerDeploymentStepSnapshot = {
|
|
key: WorkServerDeploymentStepKey;
|
|
status: WorkServerDeploymentStepStatus;
|
|
detail: string | null;
|
|
updatedAt: string | null;
|
|
};
|
|
|
|
export type WorkServerDeploymentPhase =
|
|
| 'idle'
|
|
| 'build-target-slot'
|
|
| 'verify-target-health'
|
|
| 'switch-proxy'
|
|
| 'drain-previous-slot'
|
|
| 'rebuild-previous-slot'
|
|
| 'recover-interrupted-chat'
|
|
| 'completed'
|
|
| 'failed';
|
|
|
|
export type WorkServerDeploymentSnapshot = {
|
|
status: 'idle' | 'running' | 'completed' | 'failed';
|
|
phase: WorkServerDeploymentPhase;
|
|
summary: string | null;
|
|
startedAt: string | null;
|
|
updatedAt: string | null;
|
|
completedAt: string | null;
|
|
activeSlot: WorkServerSlot | null;
|
|
targetSlot: WorkServerSlot | null;
|
|
previousSlot: WorkServerSlot | null;
|
|
targetContainer: string | null;
|
|
previousContainer: string | null;
|
|
previousSlotActiveChatRequestCount: number | null;
|
|
previousSlotQueuedChatRequestCount: number | null;
|
|
recoveredSessionCount: number | null;
|
|
recoveredRestartedCount: number | null;
|
|
recoveredRequeuedCount: number | null;
|
|
lastError: string | null;
|
|
logExcerpt: string | null;
|
|
steps: WorkServerDeploymentStepSnapshot[];
|
|
};
|
|
|
|
const RUNNER_HEARTBEAT_FRESHNESS_MS = 30_000;
|
|
const DEFERRED_RESTART_DELAY_MS = 2_000;
|
|
const DEFERRED_RESTART_CONFIRM_TIMEOUT_MS = 4_500;
|
|
const DEFERRED_RESTART_POLL_INTERVAL_MS = 150;
|
|
const APP_SOURCE_TARGET_PATHS = [
|
|
'src',
|
|
'public',
|
|
'index.html',
|
|
'package.json',
|
|
'tsconfig.json',
|
|
'tsconfig.app.json',
|
|
'vite.config.ts',
|
|
'scripts',
|
|
] as const;
|
|
const APP_BUILD_INFO_FILE_CANDIDATES = [
|
|
'/tmp/ai-code-test-app-dist/index.html',
|
|
'/tmp/ai-code-test-app-dist/manifest.webmanifest',
|
|
'/tmp/ai-code-test-app-dist/assets',
|
|
] as const;
|
|
const APP_BUILD_STAMP_RELATIVE_PATH = '.server-command-test-app-built-at';
|
|
const APP_SOURCE_EXCLUDED_PREFIXES = ['public/.codex_chat/'] as const;
|
|
const APP_SOURCE_EXCLUDED_FILE_PATTERNS = [/\.test\.[^/]+$/i, /\.spec\.[^/]+$/i] as const;
|
|
const WORK_SERVER_RESTART_LOCK_STALE_MS = 20 * 60 * 1000;
|
|
|
|
type WorkServerRestartLockPayload = {
|
|
startedAt: string;
|
|
key: ServerCommandKey;
|
|
pid: number;
|
|
};
|
|
|
|
type WorkServerDeploymentStateFilePayload = {
|
|
status?: unknown;
|
|
phase?: unknown;
|
|
summary?: unknown;
|
|
startedAt?: unknown;
|
|
updatedAt?: unknown;
|
|
completedAt?: unknown;
|
|
activeSlot?: unknown;
|
|
targetSlot?: unknown;
|
|
previousSlot?: unknown;
|
|
targetContainer?: unknown;
|
|
previousContainer?: unknown;
|
|
previousSlotActiveChatRequestCount?: unknown;
|
|
previousSlotQueuedChatRequestCount?: unknown;
|
|
recoveredSessionCount?: unknown;
|
|
recoveredRestartedCount?: unknown;
|
|
recoveredRequeuedCount?: unknown;
|
|
lastError?: unknown;
|
|
logExcerpt?: unknown;
|
|
steps?: unknown;
|
|
};
|
|
|
|
export async function readAppBuildTimestamp(definition: ServerDefinition, options?: { allowLocal?: boolean }) {
|
|
const allowLocal = options?.allowLocal ?? false;
|
|
let latestBuiltAt: string | null = null;
|
|
const mainProjectRoot = normalizePath(resolveMainProjectRoot());
|
|
const buildStampCandidates = [
|
|
path.join(mainProjectRoot, APP_BUILD_STAMP_RELATIVE_PATH),
|
|
path.join(normalizePath(env.SERVER_COMMAND_PROJECT_ROOT), APP_BUILD_STAMP_RELATIVE_PATH),
|
|
].filter((value, index, array) => array.indexOf(value) === index);
|
|
|
|
for (const targetPath of buildStampCandidates) {
|
|
const candidate = allowLocal ? await readLocalBuildTimestamp(targetPath) : null;
|
|
|
|
if (candidate && (!latestBuiltAt || candidate > latestBuiltAt)) {
|
|
latestBuiltAt = candidate;
|
|
}
|
|
}
|
|
|
|
for (const targetPath of APP_BUILD_INFO_FILE_CANDIDATES) {
|
|
const candidates = [
|
|
allowLocal ? await readLocalBuildTimestamp(targetPath) : null,
|
|
await readContainerBuildTimestamp(definition, targetPath),
|
|
].filter((value): value is string => Boolean(value));
|
|
|
|
for (const candidate of candidates) {
|
|
if (!latestBuiltAt || candidate > latestBuiltAt) {
|
|
latestBuiltAt = candidate;
|
|
}
|
|
}
|
|
}
|
|
|
|
return latestBuiltAt;
|
|
}
|
|
|
|
async function readLocalBuildTimestamp(targetPath: string) {
|
|
try {
|
|
const targetStat = await stat(targetPath);
|
|
return normalizeDateTimeValue(targetStat.mtime.toISOString());
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readContainerBuildTimestamp(definition: ServerDefinition, targetPath: string) {
|
|
try {
|
|
const { stdout } = await execFileAsync(
|
|
'docker',
|
|
['exec', definition.containerName, 'sh', '-lc', `if [ -e ${JSON.stringify(targetPath)} ]; then stat -c '%y' ${JSON.stringify(targetPath)}; fi`],
|
|
{
|
|
cwd: definition.commandWorkingDirectory,
|
|
timeout: 8000,
|
|
maxBuffer: 1024 * 1024,
|
|
},
|
|
);
|
|
|
|
return normalizeDateTimeValue(stdout.trim());
|
|
} catch (error) {
|
|
if (!shouldRetryWithDockerSocket(error)) {
|
|
return null;
|
|
}
|
|
|
|
return readContainerBuildTimestampViaSocket(definition, targetPath);
|
|
}
|
|
}
|
|
|
|
type SourceChangeInfo = {
|
|
changedAt: string;
|
|
path: string;
|
|
};
|
|
|
|
function isExcludedAppSourcePath(rootPath: string, targetPath: string) {
|
|
const relativePath = path.relative(rootPath, targetPath).replace(/\\/g, '/');
|
|
|
|
if (APP_SOURCE_EXCLUDED_PREFIXES.some((prefix) => relativePath === prefix.slice(0, -1) || relativePath.startsWith(prefix))) {
|
|
return true;
|
|
}
|
|
|
|
const baseName = path.basename(relativePath);
|
|
return APP_SOURCE_EXCLUDED_FILE_PATTERNS.some((pattern) => pattern.test(baseName));
|
|
}
|
|
|
|
async function findLatestSourceChangeInPath(rootPath: string, targetPath: string): Promise<SourceChangeInfo | null> {
|
|
try {
|
|
if (isExcludedAppSourcePath(rootPath, targetPath)) {
|
|
return null;
|
|
}
|
|
|
|
const targetStat = await stat(targetPath);
|
|
|
|
if (targetStat.isFile()) {
|
|
return {
|
|
changedAt: normalizeDateTimeValue(targetStat.mtime.toISOString()) ?? targetStat.mtime.toISOString(),
|
|
path: path.relative(rootPath, targetPath) || path.basename(targetPath),
|
|
};
|
|
}
|
|
|
|
if (!targetStat.isDirectory()) {
|
|
return null;
|
|
}
|
|
|
|
const entries = await fs.promises.readdir(targetPath, { withFileTypes: true });
|
|
let latest: SourceChangeInfo | null = null;
|
|
|
|
for (const entry of entries) {
|
|
if (entry.name === 'node_modules' || entry.name === 'dist' || entry.name === '.git' || entry.name === '.docker') {
|
|
continue;
|
|
}
|
|
|
|
const childPath = path.join(targetPath, entry.name);
|
|
const candidate = await findLatestSourceChangeInPath(rootPath, childPath);
|
|
|
|
if (!candidate) {
|
|
continue;
|
|
}
|
|
|
|
if (!latest || candidate.changedAt > latest.changedAt) {
|
|
latest = candidate;
|
|
}
|
|
}
|
|
|
|
return latest;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readLatestAppSourceChange() {
|
|
const projectRoot = normalizePath(resolveMainProjectRoot());
|
|
let latest: SourceChangeInfo | null = null;
|
|
|
|
for (const relativePath of APP_SOURCE_TARGET_PATHS) {
|
|
const candidate = await findLatestSourceChangeInPath(projectRoot, path.join(projectRoot, relativePath));
|
|
|
|
if (!candidate) {
|
|
continue;
|
|
}
|
|
|
|
if (!latest || candidate.changedAt > latest.changedAt) {
|
|
latest = candidate;
|
|
}
|
|
}
|
|
|
|
return latest;
|
|
}
|
|
|
|
async function requestDockerEngine<T = unknown>(
|
|
method: string,
|
|
requestPath: string,
|
|
payload?: unknown,
|
|
): Promise<T> {
|
|
const socketPath = resolveDockerSocketPath(process.env);
|
|
const body = payload == null ? null : JSON.stringify(payload);
|
|
|
|
return await new Promise<T>((resolve, reject) => {
|
|
const request = http.request(
|
|
{
|
|
socketPath,
|
|
path: requestPath,
|
|
method,
|
|
headers: body
|
|
? {
|
|
'Content-Type': 'application/json',
|
|
'Content-Length': Buffer.byteLength(body),
|
|
}
|
|
: undefined,
|
|
},
|
|
(response) => {
|
|
const chunks: Buffer[] = [];
|
|
response.on('data', (chunk) => {
|
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
});
|
|
response.on('end', () => {
|
|
const responseText = Buffer.concat(chunks).toString('utf8');
|
|
|
|
if ((response.statusCode ?? 500) >= 400) {
|
|
reject(
|
|
new Error(
|
|
trimPreview(`Docker API ${method} ${requestPath} failed: ${response.statusCode} ${responseText}`, 400) ??
|
|
`Docker API ${method} ${requestPath} failed: ${response.statusCode}`,
|
|
),
|
|
);
|
|
return;
|
|
}
|
|
|
|
if (!responseText.trim()) {
|
|
resolve(undefined as T);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
resolve(JSON.parse(responseText) as T);
|
|
} catch (error) {
|
|
reject(error);
|
|
}
|
|
});
|
|
},
|
|
);
|
|
|
|
request.once('error', reject);
|
|
request.setTimeout(8000, () => {
|
|
request.destroy(new Error(`Docker API timeout: ${method} ${requestPath}`));
|
|
});
|
|
|
|
if (body) {
|
|
request.write(body);
|
|
}
|
|
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
type DockerContainerInspect = {
|
|
Id?: string;
|
|
Name?: string;
|
|
State?: {
|
|
StartedAt?: string;
|
|
Status?: string;
|
|
};
|
|
};
|
|
|
|
async function inspectContainerViaSocket(containerName: string) {
|
|
return requestDockerEngine<DockerContainerInspect>('GET', `/containers/${encodeURIComponent(containerName)}/json`);
|
|
}
|
|
|
|
async function execContainerCommandViaSocket(containerName: string, command: string[]) {
|
|
const execCreated = await requestDockerEngine<{ Id?: string }>('POST', `/containers/${encodeURIComponent(containerName)}/exec`, {
|
|
AttachStdout: true,
|
|
AttachStderr: true,
|
|
Cmd: command,
|
|
});
|
|
const execId = execCreated.Id?.trim();
|
|
|
|
if (!execId) {
|
|
return null;
|
|
}
|
|
|
|
const output = await new Promise<string>((resolve, reject) => {
|
|
const request = http.request(
|
|
{
|
|
socketPath: resolveDockerSocketPath(process.env),
|
|
path: `/exec/${encodeURIComponent(execId)}/start`,
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
},
|
|
(response) => {
|
|
const chunks: Buffer[] = [];
|
|
response.on('data', (chunk) => {
|
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
});
|
|
response.on('end', () => {
|
|
resolve(decodeDockerExecStream(Buffer.concat(chunks)));
|
|
});
|
|
},
|
|
);
|
|
|
|
request.once('error', reject);
|
|
request.setTimeout(8000, () => {
|
|
request.destroy(new Error(`Docker exec timeout: ${containerName}`));
|
|
});
|
|
request.write(JSON.stringify({ Detach: false, Tty: false }));
|
|
request.end();
|
|
});
|
|
|
|
const execState = await requestDockerEngine<{ ExitCode?: number | null }>('GET', `/exec/${encodeURIComponent(execId)}/json`);
|
|
if ((execState.ExitCode ?? 1) !== 0) {
|
|
return null;
|
|
}
|
|
|
|
return output.trim();
|
|
}
|
|
|
|
async function readContainerBuildTimestampViaSocket(definition: ServerDefinition, targetPath: string) {
|
|
try {
|
|
const output = await execContainerCommandViaSocket(definition.containerName, [
|
|
'sh',
|
|
'-lc',
|
|
`if [ -e ${JSON.stringify(targetPath)} ]; then stat -c '%y' ${JSON.stringify(targetPath)}; fi`,
|
|
]);
|
|
|
|
return normalizeDateTimeValue(output);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function normalizeUrl(value: string) {
|
|
return value.trim().replace(/\/+$/, '');
|
|
}
|
|
|
|
function normalizeOptionalUrl(value: string | null | undefined) {
|
|
const normalized = value?.trim();
|
|
return normalized ? normalizeUrl(normalized) : null;
|
|
}
|
|
|
|
function normalizePath(value: string) {
|
|
return path.resolve(value);
|
|
}
|
|
|
|
export function buildServerCommandApiRestartUrl(baseUrl: string, pathTemplate: string, key: ServerCommandKey) {
|
|
const normalizedBaseUrl = normalizeUrl(baseUrl);
|
|
const normalizedPath = pathTemplate.trim() || '/api/server-commands/{key}/actions/restart';
|
|
const pathName = normalizedPath.startsWith('/') ? normalizedPath : `/${normalizedPath}`;
|
|
const resolvedPath = pathName.replaceAll('{key}', encodeURIComponent(key));
|
|
const baseUrlObject = new URL(`${normalizedBaseUrl}/`);
|
|
const basePath = baseUrlObject.pathname.replace(/\/+$/, '');
|
|
const nextPath =
|
|
basePath && resolvedPath === basePath
|
|
? resolvedPath
|
|
: basePath && resolvedPath.startsWith(`${basePath}/`)
|
|
? resolvedPath
|
|
: `${basePath}${resolvedPath}`.replace(/\/{2,}/g, '/');
|
|
|
|
baseUrlObject.pathname = nextPath.startsWith('/') ? nextPath : `/${nextPath}`;
|
|
baseUrlObject.search = '';
|
|
baseUrlObject.hash = '';
|
|
|
|
return normalizeUrl(baseUrlObject.toString());
|
|
}
|
|
|
|
function resolveCommandScriptPath(scriptName: string, preferredRoots: string[]) {
|
|
const resolvedRoots = preferredRoots
|
|
.map((root) => normalizePath(root))
|
|
.filter((root, index, array) => Boolean(root) && array.indexOf(root) === index);
|
|
const candidates = resolvedRoots.map((root) => path.join(root, 'etc', 'commands', 'server-command', scriptName));
|
|
const existingPath = candidates.find((candidate) => fs.existsSync(candidate));
|
|
|
|
return existingPath ?? candidates[0];
|
|
}
|
|
|
|
export function resolveDockerSocketPath(source: NodeJS.ProcessEnv | Record<string, string> = process.env) {
|
|
const explicitSocketPath = source.SERVER_COMMAND_DOCKER_SOCKET?.trim();
|
|
if (explicitSocketPath) {
|
|
return explicitSocketPath;
|
|
}
|
|
|
|
const dockerHost = source.DOCKER_HOST?.trim();
|
|
if (dockerHost?.startsWith('unix://')) {
|
|
return dockerHost.slice('unix://'.length);
|
|
}
|
|
|
|
return '/var/run/docker.sock';
|
|
}
|
|
|
|
function getWorkServerActiveSlotFileCandidates() {
|
|
const mainProjectRoot = normalizePath(resolveMainProjectRoot());
|
|
const projectRoot = normalizePath(env.SERVER_COMMAND_PROJECT_ROOT);
|
|
|
|
return [
|
|
env.SERVER_COMMAND_WORK_SERVER_ACTIVE_SLOT_FILE?.trim() || null,
|
|
path.join(mainProjectRoot, 'etc', 'servers', 'work-server', '.docker', 'runtime', 'active-slot'),
|
|
path.join(projectRoot, 'etc', 'servers', 'work-server', '.docker', 'runtime', 'active-slot'),
|
|
path.join(projectRoot, '.docker', 'runtime', 'active-slot'),
|
|
].filter((value, index, array): value is string => Boolean(value) && array.indexOf(value) === index);
|
|
}
|
|
|
|
async function readWorkServerActiveSlot(): Promise<WorkServerSlot> {
|
|
for (const candidate of getWorkServerActiveSlotFileCandidates()) {
|
|
try {
|
|
const value = (await readFile(candidate, 'utf8')).trim();
|
|
if (value === 'blue' || value === 'green') {
|
|
return value;
|
|
}
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
return 'blue';
|
|
}
|
|
|
|
function resolveWorkServerContainerName(slot: WorkServerSlot) {
|
|
return slot === 'green' ? 'work-server-green' : 'work-server-blue';
|
|
}
|
|
|
|
function appendComposeDetails(detailParts: Array<string | null | undefined>) {
|
|
return trimPreview(detailParts.filter(Boolean).join(' '));
|
|
}
|
|
|
|
function shouldRetryWithDockerSocket(error: unknown) {
|
|
const failure = error instanceof Error ? (error as ExecFileFailure) : null;
|
|
const detail = [failure?.stderr, failure?.stdout, failure?.message].filter(Boolean).join('\n');
|
|
|
|
return failure?.code === 127 || failure?.code === 'ENOENT' || /docker CLI not found|spawn docker ENOENT/i.test(detail);
|
|
}
|
|
|
|
function decodeDockerExecStream(buffer: Buffer) {
|
|
let offset = 0;
|
|
const chunks: Buffer[] = [];
|
|
|
|
while (offset + 8 <= buffer.length) {
|
|
const frameLength = buffer.readUInt32BE(offset + 4);
|
|
const frameStart = offset + 8;
|
|
const frameEnd = frameStart + frameLength;
|
|
|
|
if (frameEnd > buffer.length) {
|
|
break;
|
|
}
|
|
|
|
chunks.push(buffer.subarray(frameStart, frameEnd));
|
|
offset = frameEnd;
|
|
}
|
|
|
|
return (chunks.length > 0 ? Buffer.concat(chunks) : buffer).toString('utf8');
|
|
}
|
|
|
|
export function buildHealthCheckUrls(key: ServerCommandKey, checkUrl: string) {
|
|
const normalized = normalizeUrl(checkUrl);
|
|
|
|
if (key !== 'command-runner') {
|
|
return [normalized];
|
|
}
|
|
|
|
let parsedUrl: URL;
|
|
|
|
try {
|
|
parsedUrl = new URL(normalized);
|
|
} catch {
|
|
return [normalized];
|
|
}
|
|
|
|
const hostVariants =
|
|
parsedUrl.hostname === 'host.docker.internal'
|
|
? ['host.docker.internal', '127.0.0.1', 'localhost']
|
|
: parsedUrl.hostname === '127.0.0.1' || parsedUrl.hostname === 'localhost'
|
|
? [parsedUrl.hostname, parsedUrl.hostname === '127.0.0.1' ? 'localhost' : '127.0.0.1', 'host.docker.internal']
|
|
: [parsedUrl.hostname];
|
|
|
|
const dedupedUrls: string[] = [];
|
|
|
|
for (const hostname of hostVariants) {
|
|
const candidate = new URL(parsedUrl.toString());
|
|
candidate.hostname = hostname;
|
|
const serialized = normalizeUrl(candidate.toString());
|
|
|
|
if (!dedupedUrls.includes(serialized)) {
|
|
dedupedUrls.push(serialized);
|
|
}
|
|
}
|
|
|
|
return dedupedUrls;
|
|
}
|
|
|
|
async function fetchHealthCheck(url: string): Promise<HealthCheckAttempt> {
|
|
try {
|
|
const response = await fetch(url, {
|
|
method: 'GET',
|
|
redirect: 'follow',
|
|
signal: AbortSignal.timeout(5000),
|
|
});
|
|
const bodyText = await response.text();
|
|
|
|
return {
|
|
url,
|
|
httpStatus: response.status,
|
|
contentType: response.headers.get('content-type'),
|
|
responsePreview: trimPreview(bodyText),
|
|
availability: response.ok ? 'online' : response.status < 500 ? 'degraded' : 'offline',
|
|
errorMessage: null,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
url,
|
|
httpStatus: null,
|
|
contentType: null,
|
|
responsePreview: null,
|
|
availability: 'offline',
|
|
errorMessage: error instanceof Error ? error.message : '서버 상태를 확인하지 못했습니다.',
|
|
};
|
|
}
|
|
}
|
|
|
|
async function restartViaDockerSocket(definition: ServerDefinition) {
|
|
const mergedEnv = {
|
|
...process.env,
|
|
...definition.commandEnvironment,
|
|
};
|
|
const socketPath = resolveDockerSocketPath(mergedEnv);
|
|
const socketRestartScript = path.join(path.dirname(definition.commandScript), 'restart-via-docker-socket.mjs');
|
|
|
|
return execFileAsync('node', [socketRestartScript, definition.containerName], {
|
|
cwd: definition.commandWorkingDirectory,
|
|
timeout: 30000,
|
|
maxBuffer: 1024 * 1024,
|
|
env: {
|
|
...mergedEnv,
|
|
SERVER_COMMAND_DOCKER_SOCKET: socketPath,
|
|
},
|
|
});
|
|
}
|
|
|
|
function getServerDefinitions(): ServerDefinition[] {
|
|
const mainProjectRoot = normalizePath(resolveMainProjectRoot());
|
|
const useLocalMainMode = Boolean(env.PLAN_LOCAL_MAIN_MODE);
|
|
const projectRoot = normalizePath(useLocalMainMode ? mainProjectRoot : env.SERVER_COMMAND_PROJECT_ROOT);
|
|
const scriptRootCandidates = [mainProjectRoot, projectRoot, '/workspace/main-project'];
|
|
|
|
return [
|
|
{
|
|
key: 'test',
|
|
label: 'PREVIEW',
|
|
summary: 'preview.sm-home.cloud 테스트 앱 컨테이너',
|
|
environment: 'test',
|
|
publicUrl: normalizeUrl(env.SERVER_COMMAND_TEST_URL),
|
|
checkUrl: normalizeUrl(env.SERVER_COMMAND_TEST_CHECK_URL || env.SERVER_COMMAND_TEST_URL),
|
|
composeFile: path.join(mainProjectRoot, 'docker-compose.yml'),
|
|
serviceName: env.SERVER_COMMAND_TEST_SERVICE,
|
|
containerName: 'ai-code-app-app-1',
|
|
commandScript: resolveCommandScriptPath('restart-test.sh', scriptRootCandidates),
|
|
commandWorkingDirectory: mainProjectRoot,
|
|
commandEnvironment: {
|
|
MAIN_PROJECT_ROOT: mainProjectRoot,
|
|
SERVER_COMMAND_COMPOSE_FILE: path.join(mainProjectRoot, 'docker-compose.yml'),
|
|
SERVER_COMMAND_SERVICE: env.SERVER_COMMAND_TEST_SERVICE,
|
|
SERVER_COMMAND_CONTAINER_NAME: 'ai-code-app-app-1',
|
|
SERVER_COMMAND_TEST_GIT_REMOTE: 'origin',
|
|
SERVER_COMMAND_TEST_GIT_BRANCH: env.PLAN_MAIN_BRANCH,
|
|
},
|
|
restartStrategy: 'wait',
|
|
},
|
|
{
|
|
key: 'rel',
|
|
label: 'REL',
|
|
summary: 'release 브랜치를 서비스하는 릴리즈 앱 컨테이너',
|
|
environment: 'release',
|
|
publicUrl: normalizeUrl(env.SERVER_COMMAND_REL_URL),
|
|
checkUrl: normalizeUrl(env.SERVER_COMMAND_REL_CHECK_URL || env.SERVER_COMMAND_REL_URL),
|
|
composeFile: path.join(mainProjectRoot, 'docker-compose.yml'),
|
|
serviceName: env.SERVER_COMMAND_REL_SERVICE,
|
|
containerName: 'ai-code-app-release',
|
|
commandScript: resolveCommandScriptPath('restart-rel.sh', scriptRootCandidates),
|
|
commandWorkingDirectory: mainProjectRoot,
|
|
commandEnvironment: {
|
|
MAIN_PROJECT_ROOT: mainProjectRoot,
|
|
SERVER_COMMAND_COMPOSE_FILE: path.join(mainProjectRoot, 'docker-compose.yml'),
|
|
SERVER_COMMAND_SERVICE: env.SERVER_COMMAND_REL_SERVICE,
|
|
},
|
|
restartStrategy: 'wait',
|
|
},
|
|
{
|
|
key: 'prod',
|
|
label: 'PROD',
|
|
summary: '프로덕션 앱 컨테이너',
|
|
environment: 'production',
|
|
publicUrl: normalizeUrl(env.SERVER_COMMAND_PROD_URL),
|
|
checkUrl: normalizeUrl(env.SERVER_COMMAND_PROD_CHECK_URL || env.SERVER_COMMAND_PROD_URL),
|
|
composeFile: path.join(mainProjectRoot, 'docker-compose.yml'),
|
|
serviceName: env.SERVER_COMMAND_PROD_SERVICE,
|
|
containerName: 'ai-code-app-prod',
|
|
commandScript: resolveCommandScriptPath('restart-prod.sh', scriptRootCandidates),
|
|
commandWorkingDirectory: mainProjectRoot,
|
|
commandEnvironment: {
|
|
MAIN_PROJECT_ROOT: mainProjectRoot,
|
|
SERVER_COMMAND_COMPOSE_FILE: path.join(mainProjectRoot, 'docker-compose.yml'),
|
|
SERVER_COMMAND_SERVICE: env.SERVER_COMMAND_PROD_SERVICE,
|
|
SERVER_COMMAND_CONTAINER_NAME: 'ai-code-app-prod',
|
|
},
|
|
restartStrategy: 'deferred',
|
|
deferredResponseMode: 'wait-for-result',
|
|
},
|
|
{
|
|
key: 'work-server',
|
|
label: 'WORK-SERVER',
|
|
summary: 'Plan, Board, History API를 제공하는 워크서버',
|
|
environment: 'internal-api',
|
|
publicUrl: null,
|
|
checkUrl: normalizeUrl(env.SERVER_COMMAND_WORK_SERVER_URL),
|
|
composeFile: path.join(projectRoot, 'etc', 'servers', 'work-server', 'docker-compose.yml'),
|
|
serviceName: env.SERVER_COMMAND_WORK_SERVER_SERVICE,
|
|
containerName: 'work-server',
|
|
commandScript: resolveCommandScriptPath('restart-work-server.sh', scriptRootCandidates),
|
|
commandWorkingDirectory: mainProjectRoot,
|
|
commandEnvironment: {
|
|
REPO_ROOT: mainProjectRoot,
|
|
},
|
|
restartStrategy: 'deferred',
|
|
deferredResponseMode: 'accept-immediately',
|
|
},
|
|
{
|
|
key: 'command-runner',
|
|
label: 'COMMAND-RUNNER',
|
|
summary: 'nohup으로 실행 중인 서버 명령 host runner',
|
|
environment: 'host-runner',
|
|
publicUrl: null,
|
|
checkUrl: normalizeUrl(env.SERVER_COMMAND_RUNNER_URL),
|
|
composeFile: path.join(projectRoot, 'scripts', 'run-server-command-runner.mjs'),
|
|
serviceName: 'server-command-runner',
|
|
containerName: 'server-command-runner',
|
|
commandScript: resolveCommandScriptPath('restart-server-command-runner.sh', scriptRootCandidates),
|
|
commandWorkingDirectory: mainProjectRoot,
|
|
commandEnvironment: {
|
|
PROJECT_ROOT: mainProjectRoot,
|
|
},
|
|
restartStrategy: 'deferred',
|
|
deferredResponseMode: 'wait-for-result',
|
|
},
|
|
];
|
|
}
|
|
|
|
function getServerDefinition(key: ServerCommandKey) {
|
|
const definition = getServerDefinitions().find((item) => item.key === key);
|
|
|
|
if (!definition) {
|
|
throw new Error('지원하지 않는 서버입니다.');
|
|
}
|
|
|
|
return definition;
|
|
}
|
|
|
|
async function executeServerCommandScript(
|
|
definition: ServerDefinition,
|
|
options: ServerCommandScriptExecutionOptions = {},
|
|
) {
|
|
const commandScript = options.commandScript ?? definition.commandScript;
|
|
const timeoutMs = options.timeoutMs ?? 30000;
|
|
|
|
return execFileAsync('sh', [commandScript], {
|
|
cwd: definition.commandWorkingDirectory,
|
|
timeout: timeoutMs,
|
|
maxBuffer: 1024 * 1024,
|
|
env: {
|
|
...process.env,
|
|
...definition.commandEnvironment,
|
|
...options.environment,
|
|
},
|
|
});
|
|
}
|
|
|
|
function trimPreview(value: string | null | undefined, maxLength = 220) {
|
|
const normalized = value?.replace(/\s+/g, ' ').trim() ?? '';
|
|
|
|
if (!normalized) {
|
|
return null;
|
|
}
|
|
|
|
return normalized.length > maxLength ? `${normalized.slice(0, maxLength - 3)}...` : normalized;
|
|
}
|
|
|
|
function normalizeDateTimeValue(value: string | null | undefined) {
|
|
const normalized = value?.trim();
|
|
|
|
if (!normalized) {
|
|
return null;
|
|
}
|
|
|
|
const parsed = new Date(normalized);
|
|
return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString();
|
|
}
|
|
|
|
function getWorkServerRestartLockPath() {
|
|
return path.join(resolveMainProjectRoot(), "etc", "servers", "work-server", ".docker", "runtime", "restart-in-progress.json");
|
|
}
|
|
|
|
function getWorkServerDeploymentStatePath() {
|
|
return path.join(resolveMainProjectRoot(), 'etc', 'servers', 'work-server', '.docker', 'runtime', 'deployment-state.json');
|
|
}
|
|
|
|
const WORK_SERVER_DEPLOYMENT_STEP_KEYS: WorkServerDeploymentStepKey[] = [
|
|
'build-target-slot',
|
|
'verify-target-health',
|
|
'switch-proxy',
|
|
'drain-previous-slot',
|
|
'rebuild-previous-slot',
|
|
'recover-interrupted-chat',
|
|
];
|
|
|
|
function normalizeWorkServerDeploymentStepKey(value: unknown): WorkServerDeploymentStepKey | null {
|
|
return WORK_SERVER_DEPLOYMENT_STEP_KEYS.includes(value as WorkServerDeploymentStepKey)
|
|
? (value as WorkServerDeploymentStepKey)
|
|
: null;
|
|
}
|
|
|
|
function normalizeWorkServerSlotValue(value: unknown): WorkServerSlot | null {
|
|
return value === 'blue' || value === 'green' ? value : null;
|
|
}
|
|
|
|
function normalizeWorkServerDeploymentPhase(value: unknown): WorkServerDeploymentPhase {
|
|
return value === 'build-target-slot'
|
|
|| value === 'verify-target-health'
|
|
|| value === 'switch-proxy'
|
|
|| value === 'drain-previous-slot'
|
|
|| value === 'rebuild-previous-slot'
|
|
|| value === 'recover-interrupted-chat'
|
|
|| value === 'completed'
|
|
|| value === 'failed'
|
|
? value
|
|
: 'idle';
|
|
}
|
|
|
|
function normalizeWorkServerDeploymentStatus(value: unknown): WorkServerDeploymentSnapshot['status'] {
|
|
return value === 'running' || value === 'completed' || value === 'failed' ? value : 'idle';
|
|
}
|
|
|
|
function normalizeNumberOrNull(value: unknown) {
|
|
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function buildEmptyWorkServerDeploymentSnapshot(): WorkServerDeploymentSnapshot {
|
|
return {
|
|
status: 'idle',
|
|
phase: 'idle',
|
|
summary: null,
|
|
startedAt: null,
|
|
updatedAt: null,
|
|
completedAt: null,
|
|
activeSlot: null,
|
|
targetSlot: null,
|
|
previousSlot: null,
|
|
targetContainer: null,
|
|
previousContainer: null,
|
|
previousSlotActiveChatRequestCount: null,
|
|
previousSlotQueuedChatRequestCount: null,
|
|
recoveredSessionCount: null,
|
|
recoveredRestartedCount: null,
|
|
recoveredRequeuedCount: null,
|
|
lastError: null,
|
|
logExcerpt: null,
|
|
steps: WORK_SERVER_DEPLOYMENT_STEP_KEYS.map((key) => ({
|
|
key,
|
|
status: 'pending',
|
|
detail: null,
|
|
updatedAt: null,
|
|
})),
|
|
};
|
|
}
|
|
|
|
function normalizeWorkServerDeploymentSteps(value: unknown) {
|
|
const fallback = buildEmptyWorkServerDeploymentSnapshot().steps;
|
|
|
|
if (!Array.isArray(value)) {
|
|
return fallback;
|
|
}
|
|
|
|
const normalizedByKey = new Map<WorkServerDeploymentStepKey, WorkServerDeploymentStepSnapshot>();
|
|
|
|
value.forEach((item) => {
|
|
if (!item || typeof item !== 'object') {
|
|
return;
|
|
}
|
|
|
|
const candidate = item as Record<string, unknown>;
|
|
const key = normalizeWorkServerDeploymentStepKey(candidate.key);
|
|
|
|
if (!key) {
|
|
return;
|
|
}
|
|
|
|
const status =
|
|
candidate.status === 'running'
|
|
|| candidate.status === 'completed'
|
|
|| candidate.status === 'failed'
|
|
|| candidate.status === 'pending'
|
|
? candidate.status
|
|
: 'pending';
|
|
|
|
normalizedByKey.set(key, {
|
|
key,
|
|
status,
|
|
detail: typeof candidate.detail === 'string' ? candidate.detail : null,
|
|
updatedAt: normalizeDateTimeValue(typeof candidate.updatedAt === 'string' ? candidate.updatedAt : null),
|
|
});
|
|
});
|
|
|
|
return WORK_SERVER_DEPLOYMENT_STEP_KEYS.map((key) => normalizedByKey.get(key) ?? fallback.find((item) => item.key === key)!);
|
|
}
|
|
|
|
function normalizeWorkServerDeploymentSnapshot(value: unknown): WorkServerDeploymentSnapshot {
|
|
if (!value || typeof value !== 'object') {
|
|
return buildEmptyWorkServerDeploymentSnapshot();
|
|
}
|
|
|
|
const candidate = value as WorkServerDeploymentStateFilePayload;
|
|
|
|
return {
|
|
status: normalizeWorkServerDeploymentStatus(candidate.status),
|
|
phase: normalizeWorkServerDeploymentPhase(candidate.phase),
|
|
summary: typeof candidate.summary === 'string' ? candidate.summary : null,
|
|
startedAt: normalizeDateTimeValue(typeof candidate.startedAt === 'string' ? candidate.startedAt : null),
|
|
updatedAt: normalizeDateTimeValue(typeof candidate.updatedAt === 'string' ? candidate.updatedAt : null),
|
|
completedAt: normalizeDateTimeValue(typeof candidate.completedAt === 'string' ? candidate.completedAt : null),
|
|
activeSlot: normalizeWorkServerSlotValue(candidate.activeSlot),
|
|
targetSlot: normalizeWorkServerSlotValue(candidate.targetSlot),
|
|
previousSlot: normalizeWorkServerSlotValue(candidate.previousSlot),
|
|
targetContainer: typeof candidate.targetContainer === 'string' ? candidate.targetContainer : null,
|
|
previousContainer: typeof candidate.previousContainer === 'string' ? candidate.previousContainer : null,
|
|
previousSlotActiveChatRequestCount: normalizeNumberOrNull(candidate.previousSlotActiveChatRequestCount),
|
|
previousSlotQueuedChatRequestCount: normalizeNumberOrNull(candidate.previousSlotQueuedChatRequestCount),
|
|
recoveredSessionCount: normalizeNumberOrNull(candidate.recoveredSessionCount),
|
|
recoveredRestartedCount: normalizeNumberOrNull(candidate.recoveredRestartedCount),
|
|
recoveredRequeuedCount: normalizeNumberOrNull(candidate.recoveredRequeuedCount),
|
|
lastError: typeof candidate.lastError === 'string' ? candidate.lastError : null,
|
|
logExcerpt: typeof candidate.logExcerpt === 'string' ? candidate.logExcerpt : null,
|
|
steps: normalizeWorkServerDeploymentSteps(candidate.steps),
|
|
};
|
|
}
|
|
|
|
export async function readWorkServerDeploymentState(): Promise<WorkServerDeploymentSnapshot | null> {
|
|
try {
|
|
const raw = await readFile(getWorkServerDeploymentStatePath(), 'utf8');
|
|
return normalizeWorkServerDeploymentSnapshot(JSON.parse(raw));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function acquireWorkServerRestartLock() {
|
|
const lockPath = getWorkServerRestartLockPath();
|
|
await mkdir(path.dirname(lockPath), { recursive: true });
|
|
const startedAt = new Date().toISOString();
|
|
|
|
try {
|
|
const handle = await open(lockPath, "wx");
|
|
|
|
try {
|
|
await handle.writeFile(JSON.stringify({ startedAt, key: "work-server", pid: process.pid }) + "\n", "utf8");
|
|
} finally {
|
|
await handle.close();
|
|
}
|
|
|
|
return lockPath;
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
|
|
throw error;
|
|
}
|
|
|
|
let existingStartedAt: string | null = null;
|
|
|
|
try {
|
|
const raw = await readFile(lockPath, "utf8");
|
|
const parsed = JSON.parse(raw) as Partial<WorkServerRestartLockPayload>;
|
|
existingStartedAt = normalizeDateTimeValue(typeof parsed.startedAt === "string" ? parsed.startedAt : null);
|
|
const lockStat = await stat(lockPath).catch(() => null);
|
|
const freshnessSource = existingStartedAt ?? normalizeDateTimeValue(lockStat?.mtime.toISOString() ?? null);
|
|
|
|
if (!freshnessSource || Date.now() - Date.parse(freshnessSource) > WORK_SERVER_RESTART_LOCK_STALE_MS) {
|
|
await rm(lockPath, { force: true }).catch(() => undefined);
|
|
return acquireWorkServerRestartLock();
|
|
}
|
|
} catch {
|
|
// ignore read failures and keep conflict response below
|
|
}
|
|
|
|
const conflictError = new Error(
|
|
existingStartedAt
|
|
? "WORK-SERVER 무중단 재기동이 이미 진행 중입니다. 시작 시각 " + existingStartedAt
|
|
: "WORK-SERVER 무중단 재기동이 이미 진행 중입니다.",
|
|
);
|
|
(conflictError as Error & { statusCode?: number }).statusCode = 409;
|
|
throw conflictError;
|
|
}
|
|
}
|
|
|
|
function buildRestartCommandPreview(definition: ServerDefinition) {
|
|
return `sh ${definition.commandScript}`;
|
|
}
|
|
|
|
function buildDeferredRestartProbePaths(definition: ServerDefinition) {
|
|
const token = `${definition.key}-${Date.now()}-${process.pid}`;
|
|
|
|
return {
|
|
logPath: path.join('/tmp', `${token}.log`),
|
|
statusPath: path.join('/tmp', `${token}.status`),
|
|
};
|
|
}
|
|
|
|
function buildAcceptedRestartSnapshot(definition: ServerDefinition): ServerCommandSnapshot {
|
|
return {
|
|
key: definition.key,
|
|
label: definition.label,
|
|
summary: definition.summary,
|
|
environment: definition.environment,
|
|
publicUrl: definition.publicUrl,
|
|
checkUrl: definition.checkUrl,
|
|
composeFile: definition.composeFile,
|
|
serviceName: definition.serviceName,
|
|
availability: 'degraded',
|
|
httpStatus: null,
|
|
contentType: null,
|
|
responsePreview: null,
|
|
checkedAt: new Date().toISOString(),
|
|
startedAt: null,
|
|
runningVersion: null,
|
|
runningBuiltAt: null,
|
|
latestVersion: null,
|
|
latestBuiltAt: null,
|
|
latestSourceChangeAt: null,
|
|
latestSourceChangePath: null,
|
|
buildRequired: false,
|
|
updateAvailable: false,
|
|
updateSummary: null,
|
|
responseTimeMs: null,
|
|
composeStatus: 'restarting',
|
|
composeDetails: 'restart requested',
|
|
lastCommand: buildRestartCommandPreview(definition),
|
|
commandScript: definition.commandScript,
|
|
commandWorkingDirectory: definition.commandWorkingDirectory,
|
|
errorMessage: null,
|
|
deployment: definition.key === 'work-server' ? buildEmptyWorkServerDeploymentSnapshot() : null,
|
|
};
|
|
}
|
|
|
|
function coerceServerSnapshot(
|
|
definition: ServerDefinition,
|
|
value: unknown,
|
|
fallback: ServerCommandSnapshot,
|
|
): ServerCommandSnapshot {
|
|
if (!value || typeof value !== 'object') {
|
|
return fallback;
|
|
}
|
|
|
|
const item = value as Partial<Record<keyof ServerCommandSnapshot, unknown>>;
|
|
|
|
return {
|
|
...fallback,
|
|
key: item.key === definition.key ? definition.key : fallback.key,
|
|
label: typeof item.label === 'string' ? item.label : fallback.label,
|
|
summary: typeof item.summary === 'string' ? item.summary : fallback.summary,
|
|
environment: typeof item.environment === 'string' ? item.environment : fallback.environment,
|
|
publicUrl: typeof item.publicUrl === 'string' ? item.publicUrl : item.publicUrl === null ? null : fallback.publicUrl,
|
|
checkUrl: typeof item.checkUrl === 'string' ? item.checkUrl : fallback.checkUrl,
|
|
composeFile: typeof item.composeFile === 'string' ? item.composeFile : fallback.composeFile,
|
|
serviceName: typeof item.serviceName === 'string' ? item.serviceName : fallback.serviceName,
|
|
availability:
|
|
item.availability === 'online' || item.availability === 'degraded' || item.availability === 'offline'
|
|
? item.availability
|
|
: fallback.availability,
|
|
httpStatus: typeof item.httpStatus === 'number' ? item.httpStatus : item.httpStatus === null ? null : fallback.httpStatus,
|
|
contentType:
|
|
typeof item.contentType === 'string' ? item.contentType : item.contentType === null ? null : fallback.contentType,
|
|
responsePreview:
|
|
typeof item.responsePreview === 'string'
|
|
? item.responsePreview
|
|
: item.responsePreview === null
|
|
? null
|
|
: fallback.responsePreview,
|
|
checkedAt: typeof item.checkedAt === 'string' ? item.checkedAt : fallback.checkedAt,
|
|
startedAt: typeof item.startedAt === 'string' ? item.startedAt : item.startedAt === null ? null : fallback.startedAt,
|
|
runningVersion:
|
|
typeof item.runningVersion === 'string' ? item.runningVersion : item.runningVersion === null ? null : fallback.runningVersion,
|
|
runningBuiltAt:
|
|
typeof item.runningBuiltAt === 'string' ? item.runningBuiltAt : item.runningBuiltAt === null ? null : fallback.runningBuiltAt,
|
|
latestVersion:
|
|
typeof item.latestVersion === 'string' ? item.latestVersion : item.latestVersion === null ? null : fallback.latestVersion,
|
|
latestBuiltAt:
|
|
typeof item.latestBuiltAt === 'string' ? item.latestBuiltAt : item.latestBuiltAt === null ? null : fallback.latestBuiltAt,
|
|
latestSourceChangeAt:
|
|
typeof item.latestSourceChangeAt === 'string'
|
|
? item.latestSourceChangeAt
|
|
: item.latestSourceChangeAt === null
|
|
? null
|
|
: fallback.latestSourceChangeAt,
|
|
latestSourceChangePath:
|
|
typeof item.latestSourceChangePath === 'string'
|
|
? item.latestSourceChangePath
|
|
: item.latestSourceChangePath === null
|
|
? null
|
|
: fallback.latestSourceChangePath,
|
|
buildRequired: typeof item.buildRequired === 'boolean' ? item.buildRequired : fallback.buildRequired,
|
|
updateAvailable: typeof item.updateAvailable === 'boolean' ? item.updateAvailable : fallback.updateAvailable,
|
|
updateSummary:
|
|
typeof item.updateSummary === 'string' ? item.updateSummary : item.updateSummary === null ? null : fallback.updateSummary,
|
|
responseTimeMs:
|
|
typeof item.responseTimeMs === 'number' ? item.responseTimeMs : item.responseTimeMs === null ? null : fallback.responseTimeMs,
|
|
composeStatus:
|
|
typeof item.composeStatus === 'string' ? item.composeStatus : item.composeStatus === null ? null : fallback.composeStatus,
|
|
composeDetails:
|
|
typeof item.composeDetails === 'string'
|
|
? item.composeDetails
|
|
: item.composeDetails === null
|
|
? null
|
|
: fallback.composeDetails,
|
|
lastCommand: typeof item.lastCommand === 'string' ? item.lastCommand : fallback.lastCommand,
|
|
commandScript: typeof item.commandScript === 'string' ? item.commandScript : fallback.commandScript,
|
|
commandWorkingDirectory:
|
|
typeof item.commandWorkingDirectory === 'string' ? item.commandWorkingDirectory : fallback.commandWorkingDirectory,
|
|
errorMessage:
|
|
typeof item.errorMessage === 'string' ? item.errorMessage : item.errorMessage === null ? null : fallback.errorMessage,
|
|
};
|
|
}
|
|
|
|
export function buildRestartFailureMessage(label: string, error: unknown) {
|
|
const failure = error instanceof Error ? (error as ExecFileFailure) : null;
|
|
const output = trimPreview([failure?.stderr, failure?.stdout, failure?.message].filter(Boolean).join('\n'), 400);
|
|
const exitInfo = [
|
|
failure?.code != null ? `exit:${String(failure.code)}` : null,
|
|
failure?.signal ? `signal:${String(failure.signal)}` : null,
|
|
]
|
|
.filter(Boolean)
|
|
.join(' ');
|
|
|
|
return trimPreview(
|
|
[`${label} 재기동에 실패했습니다.`, exitInfo || null, output || null].filter(Boolean).join(' '),
|
|
400,
|
|
) || `${label} 재기동에 실패했습니다.`;
|
|
}
|
|
|
|
async function waitForDeferredRestartResult(
|
|
definition: ServerDefinition,
|
|
statusPath: string,
|
|
logPath: string,
|
|
): Promise<string | null> {
|
|
const deadline = Date.now() + DEFERRED_RESTART_CONFIRM_TIMEOUT_MS;
|
|
|
|
while (Date.now() <= deadline) {
|
|
try {
|
|
const statusText = (await readFile(statusPath, 'utf8')).trim();
|
|
const output = trimPreview(await readFile(logPath, 'utf8'), 400);
|
|
const exitCode = Number.parseInt(statusText, 10);
|
|
|
|
await rm(statusPath, { force: true });
|
|
await rm(logPath, { force: true });
|
|
|
|
if (!Number.isNaN(exitCode) && exitCode !== 0) {
|
|
const restartError = new Error(
|
|
buildRestartFailureMessage(
|
|
definition.label,
|
|
Object.assign(new Error(`deferred restart exited with ${exitCode}`), {
|
|
code: exitCode,
|
|
stderr: output ?? '',
|
|
stdout: '',
|
|
}),
|
|
),
|
|
);
|
|
(restartError as Error & { statusCode?: number }).statusCode = 500;
|
|
throw restartError;
|
|
}
|
|
|
|
return output;
|
|
} catch (error) {
|
|
if (
|
|
error &&
|
|
typeof error === 'object' &&
|
|
'code' in error &&
|
|
(error as NodeJS.ErrnoException).code === 'ENOENT'
|
|
) {
|
|
await new Promise((resolve) => {
|
|
setTimeout(resolve, DEFERRED_RESTART_POLL_INTERVAL_MS);
|
|
});
|
|
continue;
|
|
}
|
|
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async function restartServerCommandDeferred(definition: ServerDefinition): Promise<ServerCommandRestartResult> {
|
|
const { logPath, statusPath } = buildDeferredRestartProbePaths(definition);
|
|
const workServerLockPath = definition.key === "work-server" ? await acquireWorkServerRestartLock() : null;
|
|
const shellCommand = [
|
|
`sleep ${Math.ceil(DEFERRED_RESTART_DELAY_MS / 1000)}`,
|
|
`sh ${JSON.stringify(definition.commandScript)} >${JSON.stringify(logPath)} 2>&1`,
|
|
'status=$?',
|
|
`printf '%s' \"$status\" >${JSON.stringify(statusPath)}`,
|
|
].join('; ');
|
|
|
|
try {
|
|
await new Promise<void>((resolve, reject) => {
|
|
const child = spawn('sh', ['-c', shellCommand], {
|
|
cwd: definition.commandWorkingDirectory,
|
|
detached: true,
|
|
stdio: 'ignore',
|
|
env: {
|
|
...process.env,
|
|
...definition.commandEnvironment,
|
|
...(workServerLockPath ? { WORK_SERVER_RESTART_LOCK_FILE: workServerLockPath } : {}),
|
|
},
|
|
});
|
|
|
|
child.once('error', reject);
|
|
child.once('spawn', () => {
|
|
child.unref();
|
|
resolve();
|
|
});
|
|
});
|
|
} catch (error) {
|
|
if (workServerLockPath) {
|
|
await rm(workServerLockPath, { force: true }).catch(() => undefined);
|
|
}
|
|
|
|
throw error;
|
|
}
|
|
|
|
if (definition.deferredResponseMode === 'accept-immediately') {
|
|
return {
|
|
server: buildAcceptedRestartSnapshot(definition),
|
|
commandOutput: `${definition.label} 재기동 요청을 접수했습니다. 잠시 후 상태를 다시 확인해 주세요.`,
|
|
restartState: 'accepted',
|
|
deployment: definition.key === 'work-server' ? await readWorkServerDeploymentState() : null,
|
|
};
|
|
}
|
|
|
|
const commandOutput = await waitForDeferredRestartResult(definition, statusPath, logPath);
|
|
|
|
return {
|
|
server: buildAcceptedRestartSnapshot(definition),
|
|
commandOutput: commandOutput ?? `${definition.label} 재기동 요청을 접수했습니다. 잠시 후 상태를 다시 확인해 주세요.`,
|
|
restartState: 'accepted',
|
|
deployment: definition.key === 'work-server' ? await readWorkServerDeploymentState() : null,
|
|
};
|
|
}
|
|
|
|
async function parseRemoteRestartPayload(response: Response): Promise<RemoteRestartPayload> {
|
|
const responseText = await response.text();
|
|
const parsed =
|
|
responseText.trim().length > 0
|
|
? (() => {
|
|
try {
|
|
return JSON.parse(responseText) as {
|
|
item?: unknown;
|
|
server?: unknown;
|
|
commandOutput?: unknown;
|
|
output?: unknown;
|
|
restartState?: unknown;
|
|
data?: {
|
|
item?: unknown;
|
|
server?: unknown;
|
|
commandOutput?: unknown;
|
|
output?: unknown;
|
|
restartState?: unknown;
|
|
};
|
|
};
|
|
} catch {
|
|
return null;
|
|
}
|
|
})()
|
|
: null;
|
|
const nestedData = parsed?.data && typeof parsed.data === 'object' && !Array.isArray(parsed.data) ? parsed.data : null;
|
|
|
|
return {
|
|
server:
|
|
parsed?.item && typeof parsed.item === 'object'
|
|
? (parsed.item as ServerCommandSnapshot)
|
|
: parsed?.server && typeof parsed.server === 'object'
|
|
? (parsed.server as ServerCommandSnapshot)
|
|
: nestedData?.item && typeof nestedData.item === 'object'
|
|
? (nestedData.item as ServerCommandSnapshot)
|
|
: nestedData?.server && typeof nestedData.server === 'object'
|
|
? (nestedData.server as ServerCommandSnapshot)
|
|
: undefined,
|
|
commandOutput: trimPreview(
|
|
[
|
|
typeof parsed?.commandOutput === 'string' ? parsed.commandOutput : null,
|
|
typeof parsed?.output === 'string' ? parsed.output : null,
|
|
typeof nestedData?.commandOutput === 'string' ? nestedData.commandOutput : null,
|
|
typeof nestedData?.output === 'string' ? nestedData.output : null,
|
|
!parsed ? responseText : null,
|
|
]
|
|
.filter(Boolean)
|
|
.join('\n'),
|
|
400,
|
|
),
|
|
restartState:
|
|
parsed?.restartState === 'accepted' || nestedData?.restartState === 'accepted' ? 'accepted' : 'completed',
|
|
};
|
|
}
|
|
|
|
async function restartServerCommandViaApi(definition: ServerDefinition): Promise<ServerCommandRestartResult> {
|
|
const apiBaseUrl = normalizeOptionalUrl(env.SERVER_COMMAND_API_BASE_URL);
|
|
if (!apiBaseUrl) {
|
|
throw new Error('SERVER_COMMAND_API_BASE_URL이 비어 있습니다.');
|
|
}
|
|
|
|
const requestUrl = buildServerCommandApiRestartUrl(
|
|
apiBaseUrl,
|
|
env.SERVER_COMMAND_API_RESTART_PATH_TEMPLATE,
|
|
definition.key,
|
|
);
|
|
const headers = new Headers();
|
|
|
|
if (env.SERVER_COMMAND_API_ACCESS_TOKEN?.trim()) {
|
|
headers.set('X-Access-Token', env.SERVER_COMMAND_API_ACCESS_TOKEN.trim());
|
|
}
|
|
|
|
let response: Response;
|
|
|
|
try {
|
|
response = await fetch(requestUrl, {
|
|
method: 'POST',
|
|
headers,
|
|
signal: AbortSignal.timeout(15000),
|
|
});
|
|
} catch (error) {
|
|
const restartError = new Error(buildRestartFailureMessage(definition.label, error));
|
|
(restartError as Error & { statusCode?: number }).statusCode = 500;
|
|
throw restartError;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const detail = await response.text();
|
|
const failure = Object.assign(new Error(`HTTP ${response.status}`), {
|
|
code: response.status,
|
|
stderr: detail,
|
|
stdout: '',
|
|
});
|
|
const restartError = new Error(buildRestartFailureMessage(definition.label, failure));
|
|
(restartError as Error & { statusCode?: number }).statusCode = 500;
|
|
throw restartError;
|
|
}
|
|
|
|
const remotePayload = await parseRemoteRestartPayload(response);
|
|
const fallbackSnapshot =
|
|
remotePayload.restartState === 'accepted' ? buildAcceptedRestartSnapshot(definition) : await checkServer(definition);
|
|
|
|
return {
|
|
server: coerceServerSnapshot(definition, remotePayload.server, fallbackSnapshot),
|
|
commandOutput: remotePayload.commandOutput,
|
|
restartState: remotePayload.restartState,
|
|
};
|
|
}
|
|
|
|
function shouldFallbackFromRemoteRestart(error: unknown) {
|
|
const detail = error instanceof Error ? error.message : String(error);
|
|
|
|
return /Failed to fetch|fetch failed|ECONNREFUSED|ENOTFOUND|EHOSTUNREACH|ETIMEDOUT|404|408/i.test(detail);
|
|
}
|
|
|
|
async function inspectComposeStatus(definition: ServerDefinition) {
|
|
try {
|
|
const { stdout } = await execFileAsync(
|
|
'docker',
|
|
['compose', '-f', definition.composeFile, 'ps', definition.serviceName, '--format', 'json'],
|
|
{
|
|
cwd: definition.commandWorkingDirectory,
|
|
timeout: 8000,
|
|
maxBuffer: 1024 * 1024,
|
|
},
|
|
);
|
|
const normalized = stdout.trim();
|
|
|
|
if (!normalized) {
|
|
return {
|
|
startedAt: null,
|
|
composeStatus: null,
|
|
composeDetails: null,
|
|
};
|
|
}
|
|
|
|
const parsed = JSON.parse(normalized) as Record<string, unknown> | Array<Record<string, unknown>>;
|
|
const firstRow = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
const status = typeof firstRow?.State === 'string' ? firstRow.State : typeof firstRow?.Status === 'string' ? firstRow.Status : null;
|
|
const details = trimPreview(
|
|
[
|
|
typeof firstRow?.Name === 'string' ? `name:${firstRow.Name}` : null,
|
|
typeof firstRow?.Publishers === 'string' ? `publishers:${firstRow.Publishers}` : null,
|
|
typeof firstRow?.Health === 'string' ? `health:${firstRow.Health}` : null,
|
|
]
|
|
.filter(Boolean)
|
|
.join(' '),
|
|
);
|
|
|
|
return {
|
|
startedAt: null,
|
|
composeStatus: status,
|
|
composeDetails: details,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
startedAt: null,
|
|
composeStatus: null,
|
|
composeDetails: trimPreview(error instanceof Error ? error.message : 'compose 상태 확인 실패'),
|
|
};
|
|
}
|
|
}
|
|
|
|
async function inspectContainerRuntime(
|
|
definition: ServerDefinition,
|
|
containerNameOverride?: string,
|
|
): Promise<RuntimeInspectionResult> {
|
|
const containerName = containerNameOverride ?? definition.containerName;
|
|
|
|
try {
|
|
const { stdout } = await execFileAsync(
|
|
'docker',
|
|
['inspect', '-f', '{{.State.StartedAt}}\t{{.State.Status}}\t{{.Name}}', containerName],
|
|
{
|
|
cwd: definition.commandWorkingDirectory,
|
|
timeout: 8000,
|
|
maxBuffer: 1024 * 1024,
|
|
},
|
|
);
|
|
const [startedAtRaw = '', statusRaw = '', nameRaw = ''] = stdout.trim().split('\t');
|
|
|
|
return {
|
|
startedAt: normalizeDateTimeValue(startedAtRaw),
|
|
composeStatus: statusRaw.trim() || null,
|
|
composeDetails: trimPreview(nameRaw.trim() ? `name:${nameRaw.trim().replace(/^\//, '')}` : null),
|
|
};
|
|
} catch (error) {
|
|
if (shouldRetryWithDockerSocket(error)) {
|
|
try {
|
|
const inspected = await inspectContainerViaSocket(containerName);
|
|
return {
|
|
startedAt: normalizeDateTimeValue(inspected.State?.StartedAt ?? null),
|
|
composeStatus: inspected.State?.Status?.trim() || null,
|
|
composeDetails: trimPreview(inspected.Name?.trim() ? `name:${inspected.Name.trim().replace(/^\//, '')}` : null),
|
|
};
|
|
} catch {
|
|
// fall through to compose inspection
|
|
}
|
|
}
|
|
|
|
return inspectComposeStatus(definition);
|
|
}
|
|
}
|
|
|
|
async function inspectRunnerRuntime(definition: ServerDefinition): Promise<RuntimeInspectionResult> {
|
|
try {
|
|
const runnerScriptName = path.basename(path.join(definition.commandWorkingDirectory, 'scripts', 'run-server-command-runner.mjs'));
|
|
const { stdout } = await execFileAsync(
|
|
'sh',
|
|
[
|
|
'-c',
|
|
`ps -eo lstart=,args= | grep ${JSON.stringify(runnerScriptName)} | grep -v grep | head -n 1`,
|
|
],
|
|
{
|
|
cwd: definition.commandWorkingDirectory,
|
|
timeout: 5000,
|
|
maxBuffer: 1024 * 1024,
|
|
},
|
|
);
|
|
const line = stdout.trim();
|
|
|
|
if (!line) {
|
|
return {
|
|
startedAt: null,
|
|
composeStatus: null,
|
|
composeDetails: null,
|
|
};
|
|
}
|
|
|
|
const match = line.match(/^([A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d+\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.+)$/);
|
|
const startedAt = normalizeDateTimeValue(match?.[1] ?? null);
|
|
|
|
return {
|
|
startedAt,
|
|
composeStatus: 'running',
|
|
composeDetails: trimPreview(match?.[2] ?? line),
|
|
};
|
|
} catch {
|
|
return {
|
|
startedAt: null,
|
|
composeStatus: null,
|
|
composeDetails: null,
|
|
};
|
|
}
|
|
}
|
|
|
|
async function inspectRunnerHeartbeat(): Promise<RuntimeInspectionResult> {
|
|
const heartbeatCandidates = [
|
|
env.SERVER_COMMAND_RUNNER_HEARTBEAT_FILE?.trim() || null,
|
|
path.join(normalizePath(env.SERVER_COMMAND_PROJECT_ROOT), '.server-command-runner-heartbeat.json'),
|
|
path.join(normalizePath(env.SERVER_COMMAND_MAIN_PROJECT_ROOT), '.server-command-runner-heartbeat.json'),
|
|
].filter((value, index, array): value is string => Boolean(value) && array.indexOf(value) === index);
|
|
|
|
let latestHeartbeatPath: string | null = null;
|
|
let latestHeartbeatStat: Awaited<ReturnType<typeof stat>> | null = null;
|
|
|
|
for (const candidate of heartbeatCandidates) {
|
|
try {
|
|
const candidateStat = await stat(candidate);
|
|
|
|
if (!latestHeartbeatStat || candidateStat.mtimeMs > latestHeartbeatStat.mtimeMs) {
|
|
latestHeartbeatPath = candidate;
|
|
latestHeartbeatStat = candidateStat;
|
|
}
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
const heartbeatPath = latestHeartbeatPath ?? heartbeatCandidates[0] ?? '.server-command-runner-heartbeat.json';
|
|
|
|
try {
|
|
const heartbeatStat = latestHeartbeatStat ?? (await stat(heartbeatPath));
|
|
const ageMs = Date.now() - heartbeatStat.mtimeMs;
|
|
const startedAt = normalizeDateTimeValue(heartbeatStat.birthtime.toISOString());
|
|
|
|
if (ageMs <= RUNNER_HEARTBEAT_FRESHNESS_MS) {
|
|
return {
|
|
startedAt,
|
|
composeStatus: 'running',
|
|
composeDetails: trimPreview(`heartbeat:${heartbeatPath}`),
|
|
availability: 'online',
|
|
responsePreview: trimPreview(`heartbeat ok · ${Math.max(0, Math.round(ageMs / 1000))}초 전 갱신`),
|
|
errorMessage: '로컬 전용 runner를 heartbeat 파일 기준으로 확인했습니다.',
|
|
};
|
|
}
|
|
|
|
return {
|
|
startedAt,
|
|
composeStatus: 'stale',
|
|
composeDetails: trimPreview(`heartbeat:${heartbeatPath}`),
|
|
availability: 'offline',
|
|
responsePreview: trimPreview(`heartbeat stale · ${Math.max(0, Math.round(ageMs / 1000))}초 경과`),
|
|
errorMessage: 'runner heartbeat 갱신이 오래되었습니다.',
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
startedAt: null,
|
|
composeStatus: null,
|
|
composeDetails: trimPreview(`heartbeat:${heartbeatPath}`),
|
|
availability: 'offline',
|
|
responsePreview: null,
|
|
errorMessage: error instanceof Error ? `runner heartbeat 확인 실패: ${error.message}` : 'runner heartbeat 확인 실패',
|
|
};
|
|
}
|
|
}
|
|
|
|
function inspectCurrentProcessRuntime(): RuntimeInspectionResult {
|
|
const startedAt = new Date(Date.now() - process.uptime() * 1000).toISOString();
|
|
|
|
return {
|
|
startedAt,
|
|
composeStatus: 'running',
|
|
composeDetails: trimPreview(`pid:${process.pid}`),
|
|
};
|
|
}
|
|
|
|
async function inspectRuntime(definition: ServerDefinition): Promise<RuntimeInspectionResult> {
|
|
if (definition.key === 'command-runner') {
|
|
const runtimeInfo = await inspectRunnerRuntime(definition);
|
|
|
|
if (runtimeInfo.startedAt) {
|
|
return runtimeInfo;
|
|
}
|
|
|
|
return inspectRunnerHeartbeat();
|
|
}
|
|
|
|
if (definition.key === 'work-server') {
|
|
const primarySlot = await readWorkServerActiveSlot();
|
|
const candidateSlots: WorkServerSlot[] = primarySlot === 'green' ? ['green', 'blue'] : ['blue', 'green'];
|
|
|
|
for (const slot of candidateSlots) {
|
|
const runtimeInfo = await inspectContainerRuntime(definition, resolveWorkServerContainerName(slot));
|
|
|
|
if (runtimeInfo.startedAt) {
|
|
return {
|
|
...runtimeInfo,
|
|
composeDetails: appendComposeDetails([`slot:${slot}`, runtimeInfo.composeDetails]),
|
|
};
|
|
}
|
|
}
|
|
|
|
const runtimeInfo = await inspectContainerRuntime(definition);
|
|
|
|
if (runtimeInfo.startedAt) {
|
|
return {
|
|
...runtimeInfo,
|
|
composeDetails: appendComposeDetails(['slot:proxy', runtimeInfo.composeDetails]),
|
|
};
|
|
}
|
|
|
|
return inspectCurrentProcessRuntime();
|
|
}
|
|
|
|
return inspectContainerRuntime(definition);
|
|
}
|
|
|
|
async function inspectAppContainerBuild(definition: ServerDefinition): Promise<BuildInspectionResult | null> {
|
|
if (definition.key !== 'test' && definition.key !== 'prod') {
|
|
return null;
|
|
}
|
|
|
|
if (definition.key === 'prod') {
|
|
const testDefinition = getServerDefinition('test');
|
|
const testBuiltAt = await readAppBuildTimestamp(testDefinition, { allowLocal: true });
|
|
const prodBuiltAt = await readAppBuildTimestamp(definition);
|
|
const updateAvailable = Boolean(testBuiltAt && (!prodBuiltAt || prodBuiltAt < testBuiltAt));
|
|
|
|
return {
|
|
runningVersion: null,
|
|
runningBuiltAt: prodBuiltAt,
|
|
latestVersion: null,
|
|
latestBuiltAt: testBuiltAt,
|
|
latestSourceChangeAt: null,
|
|
latestSourceChangePath: null,
|
|
buildRequired: false,
|
|
updateAvailable,
|
|
updateSummary:
|
|
updateAvailable && testBuiltAt
|
|
? `운영 반영 시각이 TEST보다 이전입니다. TEST ${testBuiltAt}, 운영 ${prodBuiltAt ?? '미확인'}`
|
|
: prodBuiltAt
|
|
? `운영 반영 기준: ${prodBuiltAt}`
|
|
: '운영 빌드 시각을 읽지 못했습니다.',
|
|
};
|
|
}
|
|
|
|
const latestSourceChange = await readLatestAppSourceChange();
|
|
const latestSourceChangedAt = latestSourceChange?.changedAt ?? null;
|
|
const builtAt = await readAppBuildTimestamp(definition, { allowLocal: true });
|
|
|
|
if (builtAt) {
|
|
return {
|
|
runningVersion: null,
|
|
runningBuiltAt: builtAt,
|
|
latestVersion: null,
|
|
latestBuiltAt: builtAt,
|
|
latestSourceChangeAt: latestSourceChangedAt,
|
|
latestSourceChangePath: latestSourceChange?.path ?? null,
|
|
buildRequired: Boolean(latestSourceChangedAt && latestSourceChangedAt > builtAt),
|
|
updateAvailable: false,
|
|
updateSummary:
|
|
latestSourceChangedAt && latestSourceChangedAt > builtAt
|
|
? `수정된 소스가 테스트 빌드보다 새롭습니다.${latestSourceChange?.path ? ` (${latestSourceChange.path})` : ''} 테스트 앱을 다시 빌드해야 합니다.`
|
|
: `테스트 빌드 기준: ${builtAt}`,
|
|
};
|
|
}
|
|
|
|
return {
|
|
runningVersion: null,
|
|
runningBuiltAt: null,
|
|
latestVersion: null,
|
|
latestBuiltAt: null,
|
|
latestSourceChangeAt: latestSourceChangedAt,
|
|
latestSourceChangePath: latestSourceChange?.path ?? null,
|
|
buildRequired: Boolean(latestSourceChangedAt),
|
|
updateAvailable: false,
|
|
updateSummary: latestSourceChangedAt
|
|
? `테스트 빌드 시각을 읽지 못했습니다.${latestSourceChange?.path ? ` 최근 소스 변경: ${latestSourceChange.path}` : ''}`
|
|
: '테스트 빌드 시각을 읽지 못했습니다.',
|
|
};
|
|
}
|
|
|
|
async function inspectBuild(definition: ServerDefinition): Promise<BuildInspectionResult> {
|
|
if (definition.key !== 'work-server') {
|
|
const appBuildInfo = await inspectAppContainerBuild(definition);
|
|
|
|
if (appBuildInfo) {
|
|
return appBuildInfo;
|
|
}
|
|
|
|
return {
|
|
runningVersion: null,
|
|
runningBuiltAt: null,
|
|
latestVersion: null,
|
|
latestBuiltAt: null,
|
|
latestSourceChangeAt: null,
|
|
latestSourceChangePath: null,
|
|
buildRequired: false,
|
|
updateAvailable: false,
|
|
updateSummary: null,
|
|
};
|
|
}
|
|
|
|
const runningBuild = getRuntimeWorkServerBuildInfo();
|
|
const latestBuild = await readLatestWorkServerBuildInfo();
|
|
const latestSourceChange = await readLatestWorkServerSourceChange();
|
|
const latestSourceChangedAt = latestSourceChange?.changedAt ?? null;
|
|
const buildRequired = latestSourceChangedAt
|
|
? !latestBuild?.builtAt || latestSourceChangedAt > latestBuild.builtAt
|
|
: false;
|
|
const updateAvailable =
|
|
!buildRequired &&
|
|
Boolean(runningBuild?.builtAt) &&
|
|
Boolean(latestBuild?.builtAt) &&
|
|
Boolean(latestSourceChangedAt) &&
|
|
runningBuild!.builtAt < latestBuild!.builtAt &&
|
|
runningBuild!.builtAt < latestSourceChangedAt!;
|
|
|
|
return {
|
|
runningVersion: runningBuild?.buildId ?? null,
|
|
runningBuiltAt: runningBuild?.builtAt ?? null,
|
|
latestVersion: latestBuild?.buildId ?? null,
|
|
latestBuiltAt: latestBuild?.builtAt ?? null,
|
|
latestSourceChangeAt: latestSourceChangedAt,
|
|
latestSourceChangePath: latestSourceChange?.path ?? null,
|
|
buildRequired,
|
|
updateAvailable,
|
|
updateSummary: buildRequired
|
|
? `수정된 소스가 최신 빌드보다 새롭습니다.${latestSourceChange?.path ? ` (${latestSourceChange?.path})` : ''} 재시작 시 다시 빌드한 뒤 적용합니다.`
|
|
: updateAvailable
|
|
? '새로운 work-server 빌드가 준비되어 있습니다. 재시작하면 최신 버전이 적용됩니다.'
|
|
: runningBuild && latestBuild
|
|
? '실행 중인 work-server가 최신 빌드입니다.'
|
|
: latestBuild
|
|
? latestSourceChangedAt
|
|
? '최신 빌드는 준비되어 있지만 실행 중 버전 정보를 읽지 못했습니다.'
|
|
: '최신 빌드는 준비되어 있지만 워크서버 소스 수정일을 읽지 못했습니다. /app 또는 /workspace/main-project의 work-server 소스 경로를 확인해 주세요.'
|
|
: latestSourceChangedAt
|
|
? '아직 확인된 work-server 빌드 정보가 없습니다.'
|
|
: '워크서버 소스 수정일과 빌드 정보를 읽지 못했습니다. /app 또는 /workspace/main-project의 work-server 소스 경로를 확인해 주세요.',
|
|
};
|
|
}
|
|
|
|
async function checkServer(definition: ServerDefinition): Promise<ServerCommandSnapshot> {
|
|
const startedAt = Date.now();
|
|
const attemptUrls = buildHealthCheckUrls(definition.key, definition.checkUrl);
|
|
const attempts: HealthCheckAttempt[] = [];
|
|
let selectedAttempt: HealthCheckAttempt | null = null;
|
|
|
|
for (const attemptUrl of attemptUrls) {
|
|
const attempt = await fetchHealthCheck(attemptUrl);
|
|
attempts.push(attempt);
|
|
|
|
if (attempt.availability !== 'offline') {
|
|
selectedAttempt = attempt;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!selectedAttempt) {
|
|
selectedAttempt = attempts[0] ?? {
|
|
url: definition.checkUrl,
|
|
httpStatus: null,
|
|
contentType: null,
|
|
responsePreview: null,
|
|
availability: 'offline',
|
|
errorMessage: '서버 상태를 확인하지 못했습니다.',
|
|
};
|
|
}
|
|
|
|
const runtimeInfo = await inspectRuntime(definition);
|
|
const buildInfo = await inspectBuild(definition);
|
|
const deployment = definition.key === 'work-server' ? await readWorkServerDeploymentState() : null;
|
|
const fallbackAttempt = selectedAttempt.url !== definition.checkUrl ? `fallback health check succeeded via ${selectedAttempt.url}` : null;
|
|
const collectedErrors = attempts
|
|
.filter((attempt) => attempt.errorMessage)
|
|
.map((attempt) => `${attempt.url} -> ${attempt.errorMessage}`);
|
|
const errorMessage =
|
|
runtimeInfo.errorMessage ??
|
|
selectedAttempt.errorMessage ??
|
|
(fallbackAttempt && collectedErrors.length > 0
|
|
? trimPreview([fallbackAttempt, ...collectedErrors].join(' | '), 400)
|
|
: collectedErrors.length > 0
|
|
? trimPreview(collectedErrors.join(' | '), 400)
|
|
: fallbackAttempt);
|
|
|
|
return {
|
|
key: definition.key,
|
|
label: definition.label,
|
|
summary: definition.summary,
|
|
environment: definition.environment,
|
|
publicUrl: definition.publicUrl,
|
|
checkUrl: definition.checkUrl,
|
|
composeFile: definition.composeFile,
|
|
serviceName: definition.serviceName,
|
|
availability: runtimeInfo.availability ?? selectedAttempt.availability,
|
|
httpStatus: selectedAttempt.httpStatus,
|
|
contentType: selectedAttempt.contentType,
|
|
responsePreview: runtimeInfo.responsePreview ?? selectedAttempt.responsePreview,
|
|
checkedAt: new Date().toISOString(),
|
|
startedAt: runtimeInfo.startedAt,
|
|
runningVersion: buildInfo.runningVersion,
|
|
runningBuiltAt: buildInfo.runningBuiltAt,
|
|
latestVersion: buildInfo.latestVersion,
|
|
latestBuiltAt: buildInfo.latestBuiltAt,
|
|
latestSourceChangeAt: buildInfo.latestSourceChangeAt,
|
|
latestSourceChangePath: buildInfo.latestSourceChangePath,
|
|
buildRequired: buildInfo.buildRequired,
|
|
updateAvailable: buildInfo.updateAvailable,
|
|
updateSummary: buildInfo.updateSummary,
|
|
responseTimeMs: Date.now() - startedAt,
|
|
composeStatus:
|
|
definition.key === 'work-server' && deployment?.status === 'running'
|
|
? 'deploying'
|
|
: runtimeInfo.composeStatus,
|
|
composeDetails:
|
|
definition.key === 'work-server' && deployment
|
|
? appendComposeDetails([
|
|
runtimeInfo.composeDetails,
|
|
deployment.status !== 'idle'
|
|
? `deploy:${deployment.status}${deployment.targetSlot ? `:${deployment.targetSlot}` : ''}`
|
|
: null,
|
|
])
|
|
: runtimeInfo.composeDetails,
|
|
lastCommand: buildRestartCommandPreview(definition),
|
|
commandScript: definition.commandScript,
|
|
commandWorkingDirectory: definition.commandWorkingDirectory,
|
|
errorMessage: deployment?.status === 'failed' && deployment.lastError
|
|
? trimPreview([deployment.lastError, errorMessage].filter(Boolean).join(' | '), 400)
|
|
: errorMessage,
|
|
deployment,
|
|
};
|
|
}
|
|
|
|
export async function listServerCommands() {
|
|
const definitions = getServerDefinitions();
|
|
return Promise.all(definitions.map((definition) => checkServer(definition)));
|
|
}
|
|
|
|
export async function restartServerCommand(key: ServerCommandKey): Promise<ServerCommandRestartResult> {
|
|
const definition = getServerDefinition(key);
|
|
if (normalizeOptionalUrl(env.SERVER_COMMAND_API_BASE_URL)) {
|
|
try {
|
|
return await restartServerCommandViaApi(definition);
|
|
} catch (error) {
|
|
if (!shouldFallbackFromRemoteRestart(error)) {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
let stdout = '';
|
|
let stderr = '';
|
|
|
|
if (definition.restartStrategy === 'deferred') {
|
|
return restartServerCommandDeferred(definition);
|
|
}
|
|
|
|
try {
|
|
const commandResult = await executeServerCommandScript(definition);
|
|
stdout = commandResult.stdout;
|
|
stderr = commandResult.stderr;
|
|
} catch (error) {
|
|
if (shouldRetryWithDockerSocket(error)) {
|
|
try {
|
|
const commandResult = await restartViaDockerSocket(definition);
|
|
stdout = commandResult.stdout;
|
|
stderr = commandResult.stderr;
|
|
} catch (socketError) {
|
|
const restartError = new Error(buildRestartFailureMessage(definition.label, socketError));
|
|
(restartError as Error & { statusCode?: number }).statusCode = 500;
|
|
throw restartError;
|
|
}
|
|
} else {
|
|
const restartError = new Error(buildRestartFailureMessage(definition.label, error));
|
|
(restartError as Error & { statusCode?: number }).statusCode = 500;
|
|
throw restartError;
|
|
}
|
|
}
|
|
|
|
const server = await checkServer(definition);
|
|
|
|
return {
|
|
server,
|
|
commandOutput: trimPreview([stdout, stderr].filter(Boolean).join('\n'), 400),
|
|
restartState: 'completed',
|
|
deployment: server.deployment,
|
|
};
|
|
}
|
|
|
|
export async function deployWorkServerCommand(): Promise<ServerCommandRestartResult> {
|
|
return restartServerCommand('work-server');
|
|
}
|
|
|
|
export async function deployTestServerCommand(): Promise<ServerCommandRestartResult> {
|
|
const testDefinition = getServerDefinition('test');
|
|
const testDeployment = await startTestServerDeployment();
|
|
const server = await checkServer(testDefinition);
|
|
|
|
return {
|
|
server,
|
|
commandOutput: 'TEST 배포를 시작했습니다. origin/main 푸시, 테스트 빌드, 테스트 배포 과정을 확인합니다.',
|
|
restartState: 'accepted',
|
|
testDeployment: testDeployment ?? (await readTestServerDeploymentState()),
|
|
};
|
|
}
|