1046 lines
30 KiB
JavaScript
1046 lines
30 KiB
JavaScript
import { createServer } from 'node:http';
|
|
import { execFile, spawn } from 'node:child_process';
|
|
import { access, cp, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { promisify } from 'node:util';
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const projectRoot = path.resolve(__dirname, '..');
|
|
const host = process.env.SERVER_COMMAND_RUNNER_HOST?.trim() || '0.0.0.0';
|
|
const port = Number(process.env.SERVER_COMMAND_RUNNER_PORT?.trim() || '3211');
|
|
const accessToken = process.env.SERVER_COMMAND_RUNNER_ACCESS_TOKEN?.trim() || 'local-server-command-runner';
|
|
const runnerLogFile = process.env.SERVER_COMMAND_RUNNER_LOG_FILE?.trim() || '/tmp/server-command-runner.log';
|
|
const heartbeatFile =
|
|
process.env.SERVER_COMMAND_RUNNER_HEARTBEAT_FILE?.trim() || path.join(projectRoot, '.server-command-runner-heartbeat.json');
|
|
const startedAt = new Date().toISOString();
|
|
const runnerLogMaxBytes = Math.max(256 * 1024, Number(process.env.SERVER_COMMAND_RUNNER_LOG_MAX_BYTES?.trim() || `${5 * 1024 * 1024}`));
|
|
const runnerLogTrimToBytes = Math.max(
|
|
128 * 1024,
|
|
Math.min(
|
|
runnerLogMaxBytes,
|
|
Number(process.env.SERVER_COMMAND_RUNNER_LOG_TRIM_TO_BYTES?.trim() || `${2 * 1024 * 1024}`),
|
|
),
|
|
);
|
|
const runnerLogTrimIntervalMs = Math.max(
|
|
15_000,
|
|
Number(process.env.SERVER_COMMAND_RUNNER_LOG_TRIM_INTERVAL_MS?.trim() || '60000'),
|
|
);
|
|
const cpuWatchdogEnabled = process.env.SERVER_COMMAND_CPU_WATCHDOG_ENABLED?.trim() !== 'false';
|
|
const cpuWatchdogIntervalMs = Math.max(15_000, Number(process.env.SERVER_COMMAND_CPU_WATCHDOG_INTERVAL_MS?.trim() || '60000'));
|
|
const cpuWatchdogThresholdPercent = Math.max(
|
|
10,
|
|
Number(process.env.SERVER_COMMAND_CPU_WATCHDOG_THRESHOLD_PERCENT?.trim() || '120'),
|
|
);
|
|
const cpuWatchdogConsecutiveLimit = Math.max(
|
|
2,
|
|
Math.round(Number(process.env.SERVER_COMMAND_CPU_WATCHDOG_CONSECUTIVE_LIMIT?.trim() || '8')),
|
|
);
|
|
const cpuWatchdogCooldownMs = Math.max(
|
|
60_000,
|
|
Number(process.env.SERVER_COMMAND_CPU_WATCHDOG_COOLDOWN_MS?.trim() || '1200000'),
|
|
);
|
|
const STREAM_CAPTURE_LIMIT = 256 * 1024;
|
|
const CODEX_HOME_RUNTIME_PATHS = [
|
|
'auth.json',
|
|
'config.toml',
|
|
'rules',
|
|
'skills',
|
|
'vendor_imports',
|
|
'models_cache.json',
|
|
'version.json',
|
|
];
|
|
const activeCodexExecutions = new Map();
|
|
|
|
const commandDefinitions = {
|
|
test: {
|
|
label: 'TEST',
|
|
scriptPath: path.join(projectRoot, 'etc', 'commands', 'server-command', 'restart-test.sh'),
|
|
workingDirectory: projectRoot,
|
|
env: {
|
|
MAIN_PROJECT_ROOT: projectRoot,
|
|
SERVER_COMMAND_COMPOSE_FILE: path.join(projectRoot, 'docker-compose.yml'),
|
|
SERVER_COMMAND_SERVICE: process.env.SERVER_COMMAND_TEST_SERVICE?.trim() || 'app',
|
|
SERVER_COMMAND_CONTAINER_NAME: process.env.SERVER_COMMAND_TEST_CONTAINER_NAME?.trim() || 'ai-code-app-app-1',
|
|
},
|
|
restartStrategy: 'deferred',
|
|
},
|
|
rel: {
|
|
label: 'REL',
|
|
scriptPath: path.join(projectRoot, 'etc', 'commands', 'server-command', 'restart-rel.sh'),
|
|
workingDirectory: projectRoot,
|
|
env: {
|
|
MAIN_PROJECT_ROOT: projectRoot,
|
|
SERVER_COMMAND_COMPOSE_FILE: path.join(projectRoot, 'docker-compose.yml'),
|
|
SERVER_COMMAND_SERVICE: process.env.SERVER_COMMAND_REL_SERVICE?.trim() || 'release-app',
|
|
},
|
|
restartStrategy: 'deferred',
|
|
},
|
|
'work-server': {
|
|
label: 'WORK-SERVER',
|
|
scriptPath: path.join(projectRoot, 'etc', 'commands', 'server-command', 'restart-work-server.sh'),
|
|
workingDirectory: projectRoot,
|
|
env: {
|
|
REPO_ROOT: projectRoot,
|
|
},
|
|
restartStrategy: 'deferred',
|
|
},
|
|
'command-runner': {
|
|
label: 'COMMAND-RUNNER',
|
|
scriptPath: path.join(projectRoot, 'etc', 'commands', 'server-command', 'restart-server-command-runner.sh'),
|
|
workingDirectory: projectRoot,
|
|
env: {
|
|
PROJECT_ROOT: projectRoot,
|
|
SERVER_COMMAND_RUNNER_NODE_BIN:
|
|
process.env.SERVER_COMMAND_RUNNER_NODE_BIN?.trim() || 'node',
|
|
SERVER_COMMAND_RUNNER_HOST: process.env.SERVER_COMMAND_RUNNER_HOST?.trim() || '127.0.0.1',
|
|
SERVER_COMMAND_RUNNER_PORT: process.env.SERVER_COMMAND_RUNNER_PORT?.trim() || '3211',
|
|
SERVER_COMMAND_RUNNER_ACCESS_TOKEN: process.env.SERVER_COMMAND_RUNNER_ACCESS_TOKEN?.trim() || accessToken,
|
|
},
|
|
restartStrategy: 'deferred',
|
|
},
|
|
};
|
|
|
|
function normalizeRepoPathCandidate(value) {
|
|
const normalized = String(value ?? '').trim();
|
|
return normalized ? path.resolve(normalized) : '';
|
|
}
|
|
|
|
function translateWorkspacePathToHost(inputPath) {
|
|
const normalizedInput = normalizeRepoPathCandidate(inputPath);
|
|
|
|
if (!normalizedInput) {
|
|
return normalizedInput;
|
|
}
|
|
|
|
const replacements = [
|
|
['/workspace/main-project', projectRoot],
|
|
['/workspace/auto_codex/repo', projectRoot],
|
|
];
|
|
|
|
for (const [workspacePrefix, hostPrefix] of replacements) {
|
|
if (normalizedInput === workspacePrefix) {
|
|
return hostPrefix;
|
|
}
|
|
|
|
if (normalizedInput.startsWith(`${workspacePrefix}${path.sep}`)) {
|
|
return path.join(hostPrefix, normalizedInput.slice(workspacePrefix.length + 1));
|
|
}
|
|
}
|
|
|
|
return normalizedInput;
|
|
}
|
|
|
|
const cpuWatchdogTargets = [
|
|
{
|
|
name: 'test-app',
|
|
containerName: 'ai-code-app-app-1',
|
|
restartMode: 'command',
|
|
restartKey: 'test',
|
|
},
|
|
{
|
|
name: 'release-app',
|
|
containerName: 'ai-code-app-release',
|
|
restartMode: 'command',
|
|
restartKey: 'rel',
|
|
},
|
|
{
|
|
name: 'prod-app',
|
|
containerName: 'ai-code-app-prod',
|
|
restartMode: 'docker',
|
|
},
|
|
{
|
|
name: 'work-server',
|
|
containerName: 'work-server',
|
|
restartMode: 'command',
|
|
restartKey: 'work-server',
|
|
},
|
|
];
|
|
|
|
const cpuWatchdogState = new Map(
|
|
cpuWatchdogTargets.map((target) => [
|
|
target.containerName,
|
|
{
|
|
lastCpuPercent: null,
|
|
breachCount: 0,
|
|
lastSampleAt: null,
|
|
lastRestartAt: null,
|
|
lastRestartReason: null,
|
|
},
|
|
]),
|
|
);
|
|
let cpuWatchdogBusy = false;
|
|
|
|
function trimOutput(value, maxLength = 400) {
|
|
const normalized = value.replace(/\s+/g, ' ').trim();
|
|
if (!normalized) {
|
|
return null;
|
|
}
|
|
|
|
return normalized.length > maxLength ? `${normalized.slice(0, maxLength - 3)}...` : normalized;
|
|
}
|
|
|
|
function sendJson(response, statusCode, payload) {
|
|
response.writeHead(statusCode, {
|
|
'content-type': 'application/json; charset=utf-8',
|
|
'cache-control': 'no-store',
|
|
});
|
|
response.end(JSON.stringify(payload));
|
|
}
|
|
|
|
async function trimRunnerLogIfNeeded() {
|
|
try {
|
|
const logStat = await stat(runnerLogFile);
|
|
|
|
if (!logStat.isFile() || logStat.size <= runnerLogMaxBytes) {
|
|
return;
|
|
}
|
|
|
|
const content = await readFile(runnerLogFile, 'utf8');
|
|
const trimmedContent = content.slice(-runnerLogTrimToBytes);
|
|
const boundaryIndex = trimmedContent.indexOf('\n');
|
|
const nextContent = boundaryIndex >= 0 ? trimmedContent.slice(boundaryIndex + 1) : trimmedContent;
|
|
|
|
await writeFile(runnerLogFile, `[log-trimmed ${new Date().toISOString()}]\n${nextContent}`, 'utf8');
|
|
} catch {
|
|
// ignore log trim failures so the runner itself keeps serving requests
|
|
}
|
|
}
|
|
|
|
async function writeHeartbeat() {
|
|
await mkdir(path.dirname(heartbeatFile), { recursive: true });
|
|
await writeFile(
|
|
heartbeatFile,
|
|
JSON.stringify(
|
|
{
|
|
ok: true,
|
|
service: 'server-command-runner',
|
|
pid: process.pid,
|
|
host,
|
|
port,
|
|
cwd: projectRoot,
|
|
startedAt,
|
|
updatedAt: new Date().toISOString(),
|
|
cpuWatchdog: {
|
|
enabled: cpuWatchdogEnabled,
|
|
intervalMs: cpuWatchdogIntervalMs,
|
|
thresholdPercent: cpuWatchdogThresholdPercent,
|
|
consecutiveLimit: cpuWatchdogConsecutiveLimit,
|
|
cooldownMs: cpuWatchdogCooldownMs,
|
|
targets: cpuWatchdogTargets.map((target) => {
|
|
const state = cpuWatchdogState.get(target.containerName);
|
|
return {
|
|
name: target.name,
|
|
containerName: target.containerName,
|
|
restartMode: target.restartMode,
|
|
lastCpuPercent: state?.lastCpuPercent ?? null,
|
|
breachCount: state?.breachCount ?? 0,
|
|
lastSampleAt: state?.lastSampleAt ?? null,
|
|
lastRestartAt: state?.lastRestartAt ?? null,
|
|
lastRestartReason: state?.lastRestartReason ?? null,
|
|
};
|
|
}),
|
|
},
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
'utf8',
|
|
);
|
|
}
|
|
|
|
function parseCpuPercentage(value) {
|
|
const numeric = Number(String(value ?? '').replace('%', '').trim());
|
|
return Number.isFinite(numeric) ? numeric : null;
|
|
}
|
|
|
|
async function restartContainerByDocker(containerName) {
|
|
await execFileAsync('docker', ['restart', containerName], {
|
|
cwd: projectRoot,
|
|
timeout: 30_000,
|
|
maxBuffer: 1024 * 1024,
|
|
});
|
|
}
|
|
|
|
async function sampleCpuWatchdog() {
|
|
if (!cpuWatchdogEnabled || cpuWatchdogBusy || cpuWatchdogTargets.length === 0) {
|
|
return;
|
|
}
|
|
|
|
cpuWatchdogBusy = true;
|
|
|
|
try {
|
|
const { stdout } = await execFileAsync(
|
|
'docker',
|
|
[
|
|
'stats',
|
|
'--no-stream',
|
|
'--format',
|
|
'{{json .}}',
|
|
...cpuWatchdogTargets.map((target) => target.containerName),
|
|
],
|
|
{
|
|
cwd: projectRoot,
|
|
timeout: 15_000,
|
|
maxBuffer: 1024 * 1024,
|
|
},
|
|
);
|
|
const now = new Date().toISOString();
|
|
const sampledContainers = new Set();
|
|
|
|
for (const line of stdout.split('\n')) {
|
|
const trimmedLine = line.trim();
|
|
|
|
if (!trimmedLine) {
|
|
continue;
|
|
}
|
|
|
|
let parsed;
|
|
|
|
try {
|
|
parsed = JSON.parse(trimmedLine);
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
const containerName = String(parsed.Name ?? '').trim();
|
|
const cpuPercent = parseCpuPercentage(parsed.CPUPerc);
|
|
const target = cpuWatchdogTargets.find((entry) => entry.containerName === containerName);
|
|
const state = cpuWatchdogState.get(containerName);
|
|
|
|
if (!target || !state) {
|
|
continue;
|
|
}
|
|
|
|
sampledContainers.add(containerName);
|
|
state.lastCpuPercent = cpuPercent;
|
|
state.lastSampleAt = now;
|
|
state.breachCount = cpuPercent != null && cpuPercent >= cpuWatchdogThresholdPercent ? state.breachCount + 1 : 0;
|
|
|
|
const cooldownPassed =
|
|
!state.lastRestartAt || Date.now() - new Date(state.lastRestartAt).getTime() >= cpuWatchdogCooldownMs;
|
|
|
|
if (state.breachCount < cpuWatchdogConsecutiveLimit || !cooldownPassed) {
|
|
continue;
|
|
}
|
|
|
|
const restartReason = `cpu ${cpuPercent?.toFixed(1) ?? '?'}% sustained for ${
|
|
state.breachCount
|
|
} samples`;
|
|
|
|
process.stdout.write(
|
|
`[cpu-watchdog] restarting ${target.containerName} because ${restartReason} (threshold ${cpuWatchdogThresholdPercent}%)\n`,
|
|
);
|
|
|
|
if (target.restartMode === 'command' && target.restartKey) {
|
|
await runRestartCommand(target.restartKey);
|
|
} else {
|
|
await restartContainerByDocker(target.containerName);
|
|
}
|
|
|
|
state.breachCount = 0;
|
|
state.lastRestartAt = new Date().toISOString();
|
|
state.lastRestartReason = restartReason;
|
|
await writeHeartbeat().catch(() => {
|
|
// noop
|
|
});
|
|
}
|
|
|
|
for (const target of cpuWatchdogTargets) {
|
|
if (sampledContainers.has(target.containerName)) {
|
|
continue;
|
|
}
|
|
|
|
const state = cpuWatchdogState.get(target.containerName);
|
|
|
|
if (!state) {
|
|
continue;
|
|
}
|
|
|
|
state.lastCpuPercent = null;
|
|
state.lastSampleAt = now;
|
|
state.breachCount = 0;
|
|
}
|
|
} catch (error) {
|
|
process.stdout.write(
|
|
`[cpu-watchdog] sample failed: ${error instanceof Error ? error.message : String(error)}\n`,
|
|
);
|
|
} finally {
|
|
cpuWatchdogBusy = false;
|
|
}
|
|
}
|
|
|
|
void trimRunnerLogIfNeeded();
|
|
setInterval(() => {
|
|
void trimRunnerLogIfNeeded();
|
|
}, runnerLogTrimIntervalMs).unref();
|
|
|
|
function sendJsonLine(response, payload) {
|
|
response.write(`${JSON.stringify(payload)}\n`);
|
|
}
|
|
|
|
function readRequestBody(request, maxBytes = 4 * 1024 * 1024) {
|
|
return new Promise((resolve, reject) => {
|
|
let total = 0;
|
|
const chunks = [];
|
|
|
|
request.on('data', (chunk) => {
|
|
total += chunk.length;
|
|
|
|
if (total > maxBytes) {
|
|
reject(new Error('요청 본문이 너무 큽니다.'));
|
|
request.destroy();
|
|
return;
|
|
}
|
|
|
|
chunks.push(chunk);
|
|
});
|
|
|
|
request.on('end', () => {
|
|
resolve(Buffer.concat(chunks).toString('utf8'));
|
|
});
|
|
|
|
request.on('error', reject);
|
|
});
|
|
}
|
|
|
|
function summarizeCodexOutput(output) {
|
|
const normalized = String(output ?? '').trim();
|
|
|
|
if (!normalized) {
|
|
return 'Codex 실행 결과가 비어 있습니다.';
|
|
}
|
|
|
|
return normalized
|
|
.split('\n')
|
|
.map((line) => line.trimEnd())
|
|
.filter(Boolean)
|
|
.slice(-12)
|
|
.join('\n');
|
|
}
|
|
|
|
function summarizeCommand(command, limit = 180) {
|
|
const normalized = String(command ?? '').replace(/\s+/g, ' ').trim();
|
|
return normalized.length > limit ? `${normalized.slice(0, limit - 1).trimEnd()}...` : normalized;
|
|
}
|
|
|
|
function summarizeCommandOutput(output, maxLines = 3, maxLength = 220) {
|
|
const lines = String(output ?? '')
|
|
.replace(/\r/g, '')
|
|
.split('\n')
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
|
|
if (lines.length === 0) {
|
|
return '';
|
|
}
|
|
|
|
const joined = lines.slice(0, maxLines).join(' / ');
|
|
return joined.length > maxLength ? `${joined.slice(0, maxLength - 1).trimEnd()}...` : joined;
|
|
}
|
|
|
|
function inferCommandReason(command) {
|
|
const normalized = command.toLowerCase();
|
|
|
|
if (normalized.includes('rg ') || normalized.includes('ripgrep')) {
|
|
return '관련 파일이나 텍스트를 빠르게 찾기 위해 검색했습니다.';
|
|
}
|
|
|
|
if (normalized.includes('sed -n') || normalized.includes('cat ') || normalized.includes('less ')) {
|
|
return '해당 파일 내용을 직접 읽어 문맥을 확인했습니다.';
|
|
}
|
|
|
|
if (normalized.includes('ls ') || normalized === 'ls' || normalized.includes('rg --files')) {
|
|
return '디렉터리 구조와 대상 파일 위치를 확인했습니다.';
|
|
}
|
|
|
|
if (normalized.includes('git status')) {
|
|
return '작업 트리 상태와 변경 범위를 점검했습니다.';
|
|
}
|
|
|
|
if (normalized.includes('tsc ') || normalized.includes('npm test') || normalized.includes('node --test')) {
|
|
return '수정 후 타입이나 동작 검증을 진행했습니다.';
|
|
}
|
|
|
|
if (normalized.includes('curl ') || normalized.includes('fetch')) {
|
|
return '실제 응답이나 연결 상태를 확인했습니다.';
|
|
}
|
|
|
|
return '현재 요청을 해결하기 위한 다음 단계를 확인했습니다.';
|
|
}
|
|
|
|
function isIgnorableCodexDiagnosticLine(line) {
|
|
const normalized = String(line ?? '').trim();
|
|
|
|
if (!normalized) {
|
|
return true;
|
|
}
|
|
|
|
return (
|
|
/^\d{4}-\d{2}-\d{2}T\S+\s+WARN\b/.test(normalized) ||
|
|
normalized.includes('ignoring interface.defaultPrompt') ||
|
|
normalized.includes('failed to open state db') ||
|
|
normalized.includes('state db discrepancy') ||
|
|
normalized.includes('Failed to kill MCP process group') ||
|
|
normalized.includes('Failed to delete shell snapshot') ||
|
|
normalized.includes('failed to unwatch ')
|
|
);
|
|
}
|
|
|
|
function collectCodexTextFragments(value) {
|
|
if (typeof value === 'string') {
|
|
const normalized = value.trim();
|
|
return normalized ? [normalized] : [];
|
|
}
|
|
|
|
if (Array.isArray(value)) {
|
|
return value.flatMap((item) => collectCodexTextFragments(item));
|
|
}
|
|
|
|
if (!value || typeof value !== 'object') {
|
|
return [];
|
|
}
|
|
|
|
const record = value;
|
|
const directTextKeys = ['text', 'delta', 'output_text', 'content', 'message'];
|
|
|
|
for (const key of directTextKeys) {
|
|
const fragments = collectCodexTextFragments(record[key]);
|
|
|
|
if (fragments.length > 0) {
|
|
return fragments;
|
|
}
|
|
}
|
|
|
|
if (typeof record.type === 'string' && record.type.includes('output_text')) {
|
|
const fragments = collectCodexTextFragments(record.text ?? record.delta);
|
|
|
|
if (fragments.length > 0) {
|
|
return fragments;
|
|
}
|
|
}
|
|
|
|
return Object.values(record).flatMap((item) => collectCodexTextFragments(item));
|
|
}
|
|
|
|
function extractCodexStreamText(parsed) {
|
|
const type = typeof parsed.type === 'string' ? parsed.type : '';
|
|
|
|
if (type === 'item.completed') {
|
|
return {
|
|
completedText: collectCodexTextFragments(parsed.item).join(''),
|
|
deltaText: '',
|
|
};
|
|
}
|
|
|
|
if (type === 'item.delta' || type === 'response.output_text.delta') {
|
|
return {
|
|
completedText: '',
|
|
deltaText:
|
|
type === 'response.output_text.delta'
|
|
? collectCodexTextFragments(parsed.delta ?? parsed.text).join('')
|
|
: collectCodexTextFragments(parsed.delta).join(''),
|
|
};
|
|
}
|
|
|
|
if (type === 'response.completed') {
|
|
return {
|
|
completedText: collectCodexTextFragments(parsed.response).join(''),
|
|
deltaText: '',
|
|
};
|
|
}
|
|
|
|
return {
|
|
completedText: '',
|
|
deltaText: '',
|
|
};
|
|
}
|
|
|
|
function extractCodexActivityLog(parsed) {
|
|
const type = typeof parsed.type === 'string' ? parsed.type : '';
|
|
const item = parsed.item && typeof parsed.item === 'object' ? parsed.item : null;
|
|
const itemType = typeof item?.type === 'string' ? item.type : '';
|
|
|
|
if (!item || itemType !== 'command_execution') {
|
|
return '';
|
|
}
|
|
|
|
const command = summarizeCommand(typeof item.command === 'string' ? item.command : '');
|
|
|
|
if (!command) {
|
|
return '';
|
|
}
|
|
|
|
const reason = inferCommandReason(command);
|
|
|
|
if (type === 'item.started') {
|
|
return `# 이유: ${reason}\n$ ${command}`;
|
|
}
|
|
|
|
if (type === 'item.completed') {
|
|
const exitCode = typeof item.exit_code === 'number' && Number.isFinite(item.exit_code) ? Math.round(item.exit_code) : null;
|
|
const outputSummary = summarizeCommandOutput(typeof item.aggregated_output === 'string' ? item.aggregated_output : '');
|
|
const statusLabel = exitCode === null ? '# 결과: 완료' : exitCode === 0 ? '# 결과: 완료(0)' : `# 결과: 종료(${exitCode})`;
|
|
|
|
return outputSummary ? `${statusLabel}\n# 출력: ${outputSummary}` : statusLabel;
|
|
}
|
|
|
|
return '';
|
|
}
|
|
|
|
async function prepareWritableCodexHome(tempDir) {
|
|
const writableCodexHome = path.join(tempDir, '.codex');
|
|
const sourceCodexHome =
|
|
process.env.CODEX_HOME_TEMPLATE?.trim() ||
|
|
process.env.CODEX_HOME?.trim() ||
|
|
path.join(process.env.HOME ?? '/root', '.codex');
|
|
|
|
await mkdir(writableCodexHome, { recursive: true });
|
|
|
|
for (const relativePath of CODEX_HOME_RUNTIME_PATHS) {
|
|
const sourcePath = path.join(sourceCodexHome, relativePath);
|
|
const targetPath = path.join(writableCodexHome, relativePath);
|
|
|
|
try {
|
|
await access(sourcePath);
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
await mkdir(path.dirname(targetPath), { recursive: true });
|
|
await cp(sourcePath, targetPath, { recursive: true, force: true });
|
|
}
|
|
|
|
return writableCodexHome;
|
|
}
|
|
|
|
async function validateCodexExecutionRuntime(repoPath, codexBin) {
|
|
const issues = [];
|
|
|
|
try {
|
|
const repoStat = await stat(repoPath);
|
|
|
|
if (!repoStat.isDirectory()) {
|
|
issues.push(`repoPath 경로가 디렉터리가 아닙니다: ${repoPath}`);
|
|
}
|
|
} catch {
|
|
issues.push(`repoPath 경로를 찾지 못했습니다: ${repoPath}`);
|
|
}
|
|
|
|
const trimmedCodexBin = String(codexBin ?? '').trim();
|
|
const pathCandidates =
|
|
trimmedCodexBin && !trimmedCodexBin.includes(path.sep)
|
|
? (process.env.PATH ?? '')
|
|
.split(path.delimiter)
|
|
.map((entry) => entry.trim())
|
|
.filter(Boolean)
|
|
.map((entry) => path.join(entry, trimmedCodexBin))
|
|
: [trimmedCodexBin];
|
|
|
|
let codexResolved = false;
|
|
|
|
for (const candidate of pathCandidates) {
|
|
try {
|
|
await access(candidate);
|
|
codexResolved = true;
|
|
break;
|
|
} catch {
|
|
// try next candidate
|
|
}
|
|
}
|
|
|
|
if (!codexResolved) {
|
|
issues.push(`Codex 실행 파일을 찾지 못했습니다: ${codexBin}`);
|
|
}
|
|
|
|
if (issues.length > 0) {
|
|
throw new Error(issues.join(' / '));
|
|
}
|
|
}
|
|
|
|
async function runCodexLiveExecution(payload, response) {
|
|
const requestId = String(payload?.requestId ?? '').trim();
|
|
const sessionId = String(payload?.sessionId ?? '').trim();
|
|
const repoPath = translateWorkspacePathToHost(String(payload?.repoPath ?? '').trim() || projectRoot);
|
|
const prompt = String(payload?.prompt ?? '');
|
|
const resourceDir = translateWorkspacePathToHost(
|
|
String(payload?.resourceDir ?? path.join(repoPath, 'public', '.codex_chat', sessionId, 'resource')),
|
|
);
|
|
const uploadDir = translateWorkspacePathToHost(
|
|
String(payload?.uploadDir ?? path.join(resourceDir, 'uploads')),
|
|
);
|
|
const codexBin = process.env.SERVER_COMMAND_RUNNER_CODEX_BIN?.trim() || process.env.PLAN_CODEX_BIN?.trim() || 'codex';
|
|
|
|
if (!requestId || !sessionId || !prompt.trim()) {
|
|
sendJson(response, 400, {
|
|
message: 'requestId, sessionId, prompt는 필수입니다.',
|
|
});
|
|
return;
|
|
}
|
|
|
|
await validateCodexExecutionRuntime(repoPath, codexBin);
|
|
await mkdir(resourceDir, { recursive: true });
|
|
await mkdir(uploadDir, { recursive: true });
|
|
|
|
const tempDir = await mkdtemp(path.join(tmpdir(), 'command-runner-codex-'));
|
|
const writableCodexHome = await prepareWritableCodexHome(tempDir);
|
|
let stdoutTail = '';
|
|
let stderrTail = '';
|
|
let jsonLineBuffer = '';
|
|
let completedText = '';
|
|
let responseClosed = false;
|
|
|
|
response.writeHead(200, {
|
|
'content-type': 'application/x-ndjson; charset=utf-8',
|
|
'cache-control': 'no-store',
|
|
});
|
|
|
|
const child = spawn(
|
|
codexBin,
|
|
['exec', '--json', '--dangerously-bypass-approvals-and-sandbox', '-C', repoPath, '-'],
|
|
{
|
|
cwd: repoPath,
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
env: {
|
|
...process.env,
|
|
CODEX_HOME: writableCodexHome,
|
|
CODEX_LIVE_CHAT_SESSION_ID: sessionId,
|
|
CODEX_LIVE_CHAT_RESOURCE_DIR: resourceDir,
|
|
CODEX_LIVE_CHAT_UPLOAD_DIR: uploadDir,
|
|
},
|
|
},
|
|
);
|
|
|
|
activeCodexExecutions.set(requestId, {
|
|
child,
|
|
tempDir,
|
|
});
|
|
sendJsonLine(response, {
|
|
type: 'started',
|
|
pid: child.pid ?? null,
|
|
});
|
|
|
|
const cleanup = async () => {
|
|
activeCodexExecutions.delete(requestId);
|
|
await rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
|
|
};
|
|
|
|
const handleCodexJsonLine = (line) => {
|
|
let parsed;
|
|
|
|
try {
|
|
parsed = JSON.parse(line);
|
|
} catch {
|
|
return false;
|
|
}
|
|
|
|
const activityLog = extractCodexActivityLog(parsed);
|
|
|
|
if (activityLog) {
|
|
sendJsonLine(response, {
|
|
type: 'activity',
|
|
line: activityLog,
|
|
});
|
|
}
|
|
|
|
const { completedText: nextCompletedText, deltaText } = extractCodexStreamText(parsed);
|
|
|
|
if (nextCompletedText) {
|
|
completedText = nextCompletedText;
|
|
sendJsonLine(response, {
|
|
type: 'completed',
|
|
text: nextCompletedText,
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (deltaText) {
|
|
sendJsonLine(response, {
|
|
type: 'delta',
|
|
text: deltaText,
|
|
});
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
};
|
|
|
|
response.on('close', () => {
|
|
responseClosed = true;
|
|
});
|
|
|
|
child.stdout?.on('data', (chunk) => {
|
|
const text = String(chunk);
|
|
stdoutTail = (stdoutTail + text).slice(-STREAM_CAPTURE_LIMIT);
|
|
jsonLineBuffer += text;
|
|
const lines = jsonLineBuffer.split('\n');
|
|
jsonLineBuffer = lines.pop() ?? '';
|
|
|
|
for (const rawLine of lines) {
|
|
const line = rawLine.trim();
|
|
|
|
if (!line) {
|
|
continue;
|
|
}
|
|
|
|
if (handleCodexJsonLine(line)) {
|
|
continue;
|
|
}
|
|
|
|
if (!line.startsWith('{') && !isIgnorableCodexDiagnosticLine(line)) {
|
|
sendJsonLine(response, {
|
|
type: 'stdout',
|
|
line,
|
|
});
|
|
}
|
|
}
|
|
});
|
|
|
|
child.stderr?.on('data', (chunk) => {
|
|
const text = String(chunk);
|
|
stderrTail = (stderrTail + text).slice(-STREAM_CAPTURE_LIMIT);
|
|
text
|
|
.split('\n')
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.forEach((line) => {
|
|
if (isIgnorableCodexDiagnosticLine(line)) {
|
|
return;
|
|
}
|
|
|
|
sendJsonLine(response, {
|
|
type: 'stderr',
|
|
line,
|
|
});
|
|
});
|
|
});
|
|
|
|
child.on('error', async (error) => {
|
|
if (!responseClosed) {
|
|
sendJsonLine(response, {
|
|
type: 'error',
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
response.end();
|
|
}
|
|
|
|
await cleanup();
|
|
});
|
|
|
|
child.on('close', async (code) => {
|
|
const trailingLine = jsonLineBuffer.trim();
|
|
if (trailingLine) {
|
|
handleCodexJsonLine(trailingLine);
|
|
}
|
|
|
|
if (!responseClosed) {
|
|
if (code !== 0) {
|
|
sendJsonLine(response, {
|
|
type: 'error',
|
|
message: summarizeCodexOutput(`${stderrTail}\n${completedText}\n${stdoutTail}`),
|
|
});
|
|
}
|
|
|
|
response.end();
|
|
}
|
|
|
|
await cleanup();
|
|
});
|
|
|
|
child.stdin?.end(prompt);
|
|
}
|
|
|
|
function isAuthorized(request) {
|
|
const token = String(request.headers['x-access-token'] ?? '').trim();
|
|
return token.length > 0 && token === accessToken;
|
|
}
|
|
|
|
async function runRestartCommand(key) {
|
|
const definition = commandDefinitions[key];
|
|
if (!definition) {
|
|
return {
|
|
statusCode: 404,
|
|
payload: {
|
|
message: '지원하지 않는 서버입니다.',
|
|
},
|
|
};
|
|
}
|
|
|
|
if (definition.restartStrategy === 'deferred') {
|
|
await new Promise((resolve, reject) => {
|
|
const child = spawn(
|
|
'sh',
|
|
['-c', `sleep 1 && exec sh "${definition.scriptPath}" >/tmp/${key}-restart.log 2>&1`],
|
|
{
|
|
cwd: definition.workingDirectory,
|
|
detached: true,
|
|
stdio: 'ignore',
|
|
env: {
|
|
...process.env,
|
|
...definition.env,
|
|
},
|
|
},
|
|
);
|
|
|
|
child.once('error', reject);
|
|
child.once('spawn', () => {
|
|
child.unref();
|
|
resolve();
|
|
});
|
|
});
|
|
|
|
return {
|
|
statusCode: 202,
|
|
payload: {
|
|
ok: true,
|
|
restartState: 'accepted',
|
|
commandOutput: `${definition.label} 재기동 요청을 접수했습니다. 잠시 후 상태를 다시 확인해 주세요.`,
|
|
},
|
|
};
|
|
}
|
|
|
|
try {
|
|
const commandResult = await execFileAsync('sh', [definition.scriptPath], {
|
|
cwd: definition.workingDirectory,
|
|
timeout: 30000,
|
|
maxBuffer: 1024 * 1024,
|
|
env: {
|
|
...process.env,
|
|
...definition.env,
|
|
},
|
|
});
|
|
|
|
return {
|
|
statusCode: 200,
|
|
payload: {
|
|
ok: true,
|
|
restartState: 'completed',
|
|
commandOutput: trimOutput([commandResult.stdout, commandResult.stderr].filter(Boolean).join('\n')),
|
|
},
|
|
};
|
|
} catch (error) {
|
|
const failure = error instanceof Error ? error : new Error(String(error));
|
|
const detail = trimOutput(
|
|
[
|
|
failure instanceof Error && 'stderr' in failure ? String(failure.stderr ?? '') : '',
|
|
failure instanceof Error && 'stdout' in failure ? String(failure.stdout ?? '') : '',
|
|
failure.message,
|
|
]
|
|
.filter(Boolean)
|
|
.join('\n'),
|
|
);
|
|
const code =
|
|
failure instanceof Error && 'code' in failure && (typeof failure.code === 'number' || typeof failure.code === 'string')
|
|
? String(failure.code)
|
|
: null;
|
|
|
|
return {
|
|
statusCode: 500,
|
|
payload: {
|
|
message: `${definition.label} 재기동에 실패했습니다.${code ? ` exit:${code}` : ''}${detail ? ` ${detail}` : ''}`,
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
const server = createServer(async (request, response) => {
|
|
const requestUrl = new URL(request.url || '/', `http://${request.headers.host || `${host}:${port}`}`);
|
|
|
|
if (request.method === 'GET' && requestUrl.pathname === '/health') {
|
|
sendJson(response, 200, {
|
|
ok: true,
|
|
service: 'server-command-runner',
|
|
cwd: projectRoot,
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (!isAuthorized(request)) {
|
|
sendJson(response, 403, {
|
|
message: '권한 토큰이 필요합니다.',
|
|
});
|
|
return;
|
|
}
|
|
|
|
const matchedPath = requestUrl.pathname.match(/^\/api\/server-commands\/([^/]+)\/actions\/restart$/);
|
|
if (request.method === 'POST' && matchedPath) {
|
|
const key = decodeURIComponent(matchedPath[1]);
|
|
const result = await runRestartCommand(key);
|
|
sendJson(response, result.statusCode, result.payload);
|
|
return;
|
|
}
|
|
|
|
if (request.method === 'POST' && requestUrl.pathname === '/api/codex-live/execute') {
|
|
try {
|
|
const rawBody = await readRequestBody(request);
|
|
const payload = rawBody ? JSON.parse(rawBody) : {};
|
|
await runCodexLiveExecution(payload, response);
|
|
} catch (error) {
|
|
sendJson(response, 500, {
|
|
message: error instanceof Error ? error.message : 'Codex 실행 요청을 처리하지 못했습니다.',
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
|
|
const cancelMatch = requestUrl.pathname.match(/^\/api\/codex-live\/jobs\/([^/]+)\/cancel$/);
|
|
if (request.method === 'POST' && cancelMatch) {
|
|
const requestId = decodeURIComponent(cancelMatch[1]);
|
|
const activeExecution = activeCodexExecutions.get(requestId);
|
|
|
|
if (!activeExecution) {
|
|
sendJson(response, 404, {
|
|
cancelled: false,
|
|
message: '실행 중인 Codex 작업을 찾지 못했습니다.',
|
|
});
|
|
return;
|
|
}
|
|
|
|
try {
|
|
activeExecution.child.kill('SIGTERM');
|
|
setTimeout(() => {
|
|
const current = activeCodexExecutions.get(requestId);
|
|
|
|
if (current?.child && !current.child.killed) {
|
|
current.child.kill('SIGKILL');
|
|
}
|
|
}, 3000).unref?.();
|
|
sendJson(response, 200, {
|
|
cancelled: true,
|
|
});
|
|
} catch (error) {
|
|
sendJson(response, 500, {
|
|
cancelled: false,
|
|
message: error instanceof Error ? error.message : 'Codex 작업 취소에 실패했습니다.',
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
|
|
sendJson(response, 404, {
|
|
message: '지원하지 않는 경로입니다.',
|
|
});
|
|
});
|
|
|
|
server.listen(port, host, () => {
|
|
void writeHeartbeat().catch(() => {
|
|
// noop
|
|
});
|
|
const heartbeatTimer = setInterval(() => {
|
|
void writeHeartbeat().catch(() => {
|
|
// noop
|
|
});
|
|
}, 10_000);
|
|
heartbeatTimer.unref();
|
|
if (cpuWatchdogEnabled) {
|
|
const cpuWatchdogTimer = setInterval(() => {
|
|
void sampleCpuWatchdog();
|
|
}, cpuWatchdogIntervalMs);
|
|
cpuWatchdogTimer.unref();
|
|
void sampleCpuWatchdog();
|
|
}
|
|
process.stdout.write(`server-command-runner listening on http://${host}:${port}\n`);
|
|
});
|