Files
ai-code-app/etc/servers/work-server/src/workers/plan-worker.ts
2026-04-21 03:33:23 +09:00

1364 lines
46 KiB
TypeScript
Executable File

import { spawn } from 'node:child_process';
import { constants as fsConstants } from 'node:fs';
import { access } from 'node:fs/promises';
import type { FastifyBaseLogger } from 'fastify';
import { getEnv } from '../config/env.js';
import { getAppConfigSnapshot } from '../services/app-config-service.js';
import { createErrorLog, ERROR_LOG_VIEW_TOKEN } from '../services/error-log-service.js';
import { sendNotifications } from '../services/notification-service.js';
import { buildPlanNotificationData, shouldSendPlanNotification } from '../services/plan-notification-service.js';
import {
ensureDailyWorklogFile,
getKstNowParts,
hasWorklogAutomationRunForDate,
isWorklogCreationDue,
markWorklogAutomationRun,
normalizeDailyCreateTime,
} from '../services/worklog-automation-service.js';
import {
claimNextPlanForBranch,
claimNextPlanForExecution,
claimNextPlanForMerge,
claimNextPlanForMainMerge,
formatPlanNotificationLabel,
isPlanLockedByWorker,
mapPlanRow,
markPlanAsCompleted,
markPlanAutomationFailure,
markPlanMainMergeFailure,
markPlanBranchReady,
markPlanReleaseMerged,
markPlanMerged,
markPlanWorkCompleted,
upsertAutoPlanItem,
} from '../services/plan-service.js';
import {
cleanAutomationWorktree,
ensureBranchExists,
mergeBranchToRelease,
mergeReleaseToMain,
pullMainProjectBranch,
pushBranch,
} from '../services/git-service.js';
import { registerDuePlanScheduledTasks } from '../services/plan-schedule-service.js';
const STREAM_CAPTURE_LIMIT = 256 * 1024;
const FIRST_PROGRESS_NOTIFICATION_MS = 60_000;
const SECOND_PROGRESS_NOTIFICATION_MS = 120_000;
const REPEATED_PROGRESS_NOTIFICATION_MS = 180_000;
const STARTED_REQUEST_SUMMARY_LIMIT = 72;
const ERROR_SUMMARY_MAX_LENGTH = 500;
const ERROR_SUMMARY_LINE_PATTERN =
/(ERROR:|failed|failure|capacity|Read-only file system|permission|not permitted|sandbox|os error|ENOENT|EACCES|ECONN|SyntaxError|TypeError|ReferenceError)/i;
const ERROR_SUMMARY_NOISE_PATTERN =
/^(exec|codex|tokens used|succeeded in \d+ms:?|[><=]{3,}|[A-Za-z]:\\|\/bin\/bash\b|\d+\s*[:|])/i;
const PLAN_CODEX_RUNNER_MAX_ATTEMPTS = 2;
const PLAN_CODEX_RUNNER_RETRY_DELAY_MS = 5000;
const PLAN_CODEX_RUNNER_TRANSIENT_FAILURE_PATTERN =
/failed to record rollout items|failed to queue rollout items|channel closed/i;
const PLAN_CODEX_HOME_PERMISSION_PATTERN =
/failed to record rollout items|failed to queue rollout items|channel closed|read-only file system|eacces|permission/i;
const WEEKLY_DAY_TO_INDEX = {
sun: 0,
mon: 1,
tue: 2,
wed: 3,
thu: 4,
fri: 5,
sat: 6,
} as const;
type AutomationSnapshot = NonNullable<Awaited<ReturnType<typeof getAppConfigSnapshot>>['automation']>;
type AutoReceiveScheduleEvaluation = {
due: boolean;
nextEligibleAt: Date | null;
scheduleLabel: string;
};
type WorklogAutomationSnapshot = NonNullable<Awaited<ReturnType<typeof getAppConfigSnapshot>>['worklogAutomation']>;
function isValidTimeValue(value: string | undefined): value is string {
return typeof value === 'string' && /^\d{2}:\d{2}$/.test(value);
}
function withTime(baseDate: Date, timeValue: string) {
const [hours, minutes] = timeValue.split(':').map((part) => Number(part));
const nextDate = new Date(baseDate);
nextDate.setHours(hours ?? 0, minutes ?? 0, 0, 0);
return nextDate;
}
function getStartOfMinute(date: Date) {
const nextDate = new Date(date);
nextDate.setSeconds(0, 0);
return nextDate;
}
function normalizeRepeatIntervalMinutes(value: number | undefined) {
if (typeof value !== 'number' || !Number.isFinite(value)) {
return 60;
}
return Math.min(1440, Math.max(1, Math.round(value)));
}
function getNextWeeklyDate(now: Date, targetDay: keyof typeof WEEKLY_DAY_TO_INDEX, timeValue: string) {
const candidate = withTime(now, timeValue);
const currentDay = now.getDay();
const targetDayIndex = WEEKLY_DAY_TO_INDEX[targetDay];
let dayOffset = targetDayIndex - currentDay;
if (dayOffset < 0 || (dayOffset === 0 && candidate.getTime() <= now.getTime())) {
dayOffset += 7;
}
candidate.setDate(candidate.getDate() + dayOffset);
return candidate;
}
function buildAutoWorklogPlanWorkId(dateKey: string) {
return `auto-worklog-${dateKey}`;
}
function stripAnsi(text: string) {
return text.replace(/\u001B\[[0-9;]*m/g, '');
}
function normalizeErrorSummaryLine(line: string) {
return stripAnsi(line)
.replace(/^\[plan-progress\]\s*/u, '')
.replace(/^\d{4}-\d{2}-\d{2}T\S+\s+ERROR\s+[^:]+:\s*/u, '')
.replace(/\s+/g, ' ')
.trim();
}
function summarizeFailureOutput(output: string, fallback: string) {
const normalizedLines = output
.split('\n')
.map((line) => normalizeErrorSummaryLine(line))
.filter(Boolean)
.filter((line) => !ERROR_SUMMARY_NOISE_PATTERN.test(line));
const bestLine =
normalizedLines.filter((line) => ERROR_SUMMARY_LINE_PATTERN.test(line)).at(-1) ??
normalizedLines.at(-1) ??
fallback.trim();
const summary = bestLine.replace(/\s+/g, ' ').trim();
return summary.slice(0, ERROR_SUMMARY_MAX_LENGTH) || '자동 작업 실행에 실패했습니다.';
}
function isTransientPlanCodexFailure(error: unknown) {
const message = error instanceof Error ? error.message : String(error ?? '');
return PLAN_CODEX_RUNNER_TRANSIENT_FAILURE_PATTERN.test(message);
}
async function wait(ms: number) {
await new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
async function canAccessPath(pathValue: string | undefined, mode: number) {
if (!pathValue) {
return false;
}
try {
await access(pathValue, mode);
return true;
} catch {
return false;
}
}
function appendCodexHomeHint(message: string, env: ReturnType<typeof getEnv>) {
if (!PLAN_CODEX_HOME_PERMISSION_PATTERN.test(message)) {
return message;
}
const codexHome = process.env.CODEX_HOME?.trim() || '(unset)';
const templateHome = env.PLAN_CODEX_TEMPLATE_HOME?.trim() || process.env.CODEX_HOME_TEMPLATE?.trim() || '(unset)';
return `${message} [hint: CODEX_HOME=${codexHome}, template=${templateHome}, writable home/volume and permissions를 확인하세요.]`;
}
export class PlanWorker {
private readonly workerId: string;
private intervalMs: number;
private timer: NodeJS.Timeout | null = null;
private running = false;
private lastAutoReceiveAt: Date | null = null;
private lastWorklogAutomationRequestAt: Date | null = null;
constructor(private readonly logger: FastifyBaseLogger) {
const env = getEnv();
this.workerId = env.PLAN_WORKER_ID || `plan-worker-${process.pid}`;
this.intervalMs = env.PLAN_WORKER_INTERVAL_MS;
}
private isLocalMainMode() {
return Boolean(getEnv().PLAN_LOCAL_MAIN_MODE);
}
start() {
const env = getEnv();
if (!env.PLAN_WORKER_ENABLED) {
this.logger.info('Plan worker is disabled');
return;
}
if (this.timer) {
return;
}
this.intervalMs = env.PLAN_WORKER_INTERVAL_MS;
void (async () => {
const codexHome = process.env.CODEX_HOME?.trim() || '';
const codexTemplateHome = env.PLAN_CODEX_TEMPLATE_HOME?.trim() || process.env.CODEX_HOME_TEMPLATE?.trim() || '';
const [codexHomeWritable, codexTemplateReadable] = await Promise.all([
canAccessPath(codexHome, fsConstants.W_OK),
canAccessPath(codexTemplateHome, fsConstants.R_OK),
]);
this.logger.info(
{
workerId: this.workerId,
intervalMs: this.intervalMs,
repoPath: env.PLAN_GIT_REPO_PATH,
releaseBranch: env.PLAN_RELEASE_BRANCH,
codexHome: codexHome || null,
codexHomeWritable,
codexTemplateHome: codexTemplateHome || null,
codexTemplateReadable,
},
'Plan worker started',
);
})();
void this.tick();
this.timer = setInterval(() => {
void this.tick();
}, this.intervalMs);
}
async stop() {
if (this.timer) {
clearInterval(this.timer);
this.timer = null;
}
}
private async persistAutomationErrorLog(args: {
planId: number;
workId: string | null | undefined;
workerStatus: '브랜치실패' | '자동작업실패' | 'release반영실패' | 'main반영실패';
error: unknown;
detail: Record<string, unknown>;
}) {
const handledError = args.error instanceof Error ? args.error : new Error(String(args.error));
try {
await createErrorLog({
source: 'automation',
sourceLabel: 'Plan 자동화 워커',
errorType: args.workerStatus,
errorName: handledError.name,
errorMessage: handledError.message || '자동화 처리 중 오류가 발생했습니다.',
detail: JSON.stringify(args.detail, null, 2),
stackTrace: handledError.stack ?? null,
relatedPlanId: args.planId,
relatedWorkId: args.workId ?? null,
context: args.detail,
});
} catch (loggingError) {
this.logger.error({ err: loggingError, planId: args.planId }, 'Failed to persist automation error log');
}
}
private async notifyPlan(planId: number, workId: string, title: string, body: string, eventType: string) {
try {
if (!(await shouldSendPlanNotification(eventType))) {
this.logger.info({ planId, workId, eventType, skipped: true, reason: 'event-disabled' }, 'Notification send skipped');
return;
}
const result = await sendNotifications({
title,
body,
threadId: `plan-${planId}`,
data: buildPlanNotificationData(planId, workId, eventType),
});
this.logger.info(
{
planId,
workId,
eventType,
ios: {
skipped: result.ios.skipped,
sentCount: result.ios.sentCount,
failedCount: result.ios.failedCount,
invalidTokens: result.ios.invalidTokens ?? [],
},
web: {
skipped: result.web.skipped,
sentCount: result.web.sentCount,
failedCount: result.web.failedCount,
invalidEndpoints: result.web.invalidEndpoints,
},
},
'Notification send completed',
);
if (!result.ios.skipped && result.ios.failedCount > 0) {
this.logger.warn({ planId, failedCount: result.ios.failedCount }, 'iOS notification send had failures');
}
if (!result.web.skipped && result.web.failedCount > 0) {
this.logger.warn({ planId, failedCount: result.web.failedCount }, 'Web push notification send had failures');
}
} catch (error) {
this.logger.error({ planId, err: error }, 'Notification send failed');
}
}
private summarizeAutomationProgress(output: string) {
const rawLines = output
.split(/\r?\n/)
.map((line) => line.trim());
const sectionHeaders = [
'작업 ID',
'요청 요약',
'상세 요청',
'최근 조치 이력',
'최근 이슈 이력',
'규칙',
'Detailed Request',
'Request Summary',
'Recent Actions',
'Recent Issues',
'Rules',
];
const excludedSections = new Set(['요청 요약', '상세 요청', 'Detailed Request', 'Request Summary']);
let currentSection = '';
const lines = rawLines.filter((line) => {
if (!line) {
return false;
}
const matchedHeader = sectionHeaders.find((header) => line === header || line.startsWith(`${header}:`));
if (matchedHeader) {
currentSection = matchedHeader;
return !excludedSections.has(matchedHeader);
}
if (/^\d+\.\s/.test(line) && currentSection === '규칙') {
return false;
}
return !excludedSections.has(currentSection);
});
const meaningfulLines = lines.filter((line) => {
if (line.length < 6) {
return false;
}
if (/^(error|warn|info|debug|trace)\b/i.test(line)) {
return false;
}
if (/^(tokens?|cost|duration|elapsed|reading prompt|thinking)\b/i.test(line)) {
return false;
}
if (/^(\s*ID|\s*|\s*|\s*\s*|\s*\s*|)\s*:/u.test(line)) {
return false;
}
if (/^(you are|detailed request|request summary|recent actions|recent issues|rules)\s*:/i.test(line)) {
return false;
}
if (/^[|/\\-]+$/.test(line)) {
return false;
}
return true;
});
const recentLines = meaningfulLines.slice(-40);
const explicitProgressLine = [...recentLines]
.reverse()
.find((line) => line.startsWith('[plan-progress]'));
if (explicitProgressLine) {
return explicitProgressLine.replace(/^\[plan-progress\]\s*/u, '').trim().slice(0, 160);
}
const recentCommand = [...recentLines]
.reverse()
.map((line) => this.extractAutomationCommandSummary(line))
.find(Boolean);
const recentFile = [...recentLines]
.reverse()
.flatMap((line) => {
const matches = line.match(/(?:[A-Za-z0-9._-]+\/)+[A-Za-z0-9._-]+\.[A-Za-z0-9._-]+/g) ?? [];
return matches.map((match) => match.replace(/^\.?\//, ''));
})
.find((filePath) => !filePath.startsWith('node_modules/'));
const recentActionLine = [...recentLines].reverse().find((line) => {
return /(수정|변경|패치|로직|구현|추가|생성|작성|분석|탐색|확인|검증|테스트|edit|modify|update|patch|implement|create|add|write|analy|inspect|search|review|test|verif)/i.test(
line,
);
});
const actionSummary = this.summarizeAutomationAction(recentActionLine ?? recentFile ?? '');
if (recentFile && actionSummary && recentCommand) {
return `${recentFile} ${actionSummary} (${recentCommand})`.slice(0, 160);
}
if (recentFile && actionSummary) {
return `${recentFile} ${actionSummary}`.slice(0, 160);
}
if (recentCommand && actionSummary) {
return `${actionSummary} (${recentCommand})`.slice(0, 160);
}
if (recentFile) {
return recentCommand ? `${recentFile} 확인 중 (${recentCommand})`.slice(0, 160) : `${recentFile} 확인 중`;
}
if (actionSummary) {
return recentCommand ? `${actionSummary} (${recentCommand})`.slice(0, 160) : actionSummary;
}
if (recentCommand) {
return `커맨드 실행 중 (${recentCommand})`.slice(0, 160);
}
const latestLine = [...meaningfulLines].reverse()[0];
if (!latestLine) {
return '관련 파일과 최근 변경 지점을 확인하는 중입니다.';
}
return latestLine.replace(/\s+/g, ' ').slice(0, 160);
}
private extractAutomationCommandSummary(line: string) {
const text = String(line ?? '').trim();
if (!text) {
return '';
}
const rawCommand =
text.match(/^Command:\s+(.+)$/i)?.[1] ??
text.match(/^(?:Running|Execute(?:d|ing)?|Executing)\s*:?\s+(.+)$/i)?.[1] ??
text.match(/^\$\s+(.+)$/)?.[1] ??
'';
if (!rawCommand) {
return '';
}
const normalized = rawCommand
.replace(/^\/bin\/(?:bash|sh|zsh)\s+-lc\s+/i, '')
.replace(/^(?:bash|sh|zsh)\s+-lc\s+/i, '')
.replace(/^['"]|['"]$/g, '')
.replace(/\s+/g, ' ')
.trim();
if (!normalized) {
return '';
}
return normalized.length > 96 ? `${normalized.slice(0, 95).trim()}` : normalized;
}
private summarizeAutomationAction(line: string) {
const text = String(line ?? '').replace(/\s+/g, ' ').trim();
if (!text) {
return '';
}
if (this.extractAutomationCommandSummary(text)) {
if (/\b(git|npm|pnpm|yarn|node|npx|tsx)\b/i.test(text)) {
return '도구 실행 중';
}
return '커맨드 실행 중';
}
if (/(테스트|검증|확인|test|verify|check)/i.test(text)) {
return '검증 중';
}
if (/(분석|탐색|조사|읽|review|inspect|search|analy)/i.test(text)) {
return '코드 분석 중';
}
if (/(수정|변경|패치|fix|edit|modify|update|patch)/i.test(text)) {
return '수정 중';
}
if (/(로직|구현|추가|생성|작성|build|implement|create|add|write)/i.test(text)) {
return '로직 작성 중';
}
return text.slice(0, 160);
}
private buildProgressNotificationTitle(planId: number, planLabel: string, progressSummary: string) {
const workSummary = progressSummary || '작업 내용 확인 중';
const compactSummary = workSummary
.replace(/^ :\s*/u, '')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 48);
const normalizedPlanLabel = String(planLabel ?? '').trim();
const hasCustomLabel =
normalizedPlanLabel &&
normalizedPlanLabel !== `#${planId}` &&
normalizedPlanLabel !== '작업ID' &&
normalizedPlanLabel !== '[작업ID]' &&
normalizedPlanLabel.toLowerCase() !== 'undefined' &&
normalizedPlanLabel.toLowerCase() !== 'null';
const labelPrefix = hasCustomLabel ? `#${planId} ${normalizedPlanLabel}` : `#${planId}`;
return `[${labelPrefix}] ${compactSummary || '자동 작업 진행중'}`;
}
private buildExecutionCompletedBody(autoDeployToMain: boolean) {
return autoDeployToMain
? '자동 작업이 완료되었습니다. 이후 release/main 반영이 계속 진행됩니다.'
: '자동 작업이 완료되었습니다. 이후 release 반영은 별도 요청으로 진행됩니다.';
}
private summarizeExecutionRequest(note: unknown, limit = STARTED_REQUEST_SUMMARY_LIMIT) {
const text = String(note ?? '')
.replace(/\s+/g, ' ')
.trim();
if (!text) {
return '';
}
if (text.length <= limit) {
return text;
}
return `${text.slice(0, Math.max(0, limit - 1)).trim()}`;
}
private buildExecutionStartedBody(note: unknown) {
const requestSummary = this.summarizeExecutionRequest(note);
return requestSummary ? `자동 작업을 시작했습니다.\n요청: ${requestSummary}` : '자동 작업을 시작했습니다.';
}
private buildAutoWorklogPlanNote(dateKey: string, worklogAutomation: WorklogAutomationSnapshot) {
const worklogPath = `docs/worklogs/${dateKey}.md`;
const screenshotDir = `docs/assets/worklogs/${dateKey}/`;
const templateLabel = worklogAutomation.template === 'simple' ? '간단형' : '상세형';
return [
`${dateKey} 기준 업무일지 자동화 Plan입니다.`,
`${worklogPath} 파일에 금일 작업일지를 정리해 주세요.`,
`오늘 작업, 이슈 및 해결, 결정 사항, 상세 작업 내역을 최신 상태로 반영해 주세요.`,
`${worklogPath}의 작업일지 탭 본문과 상세 작업 내역에는 파일 목록, 경로 나열, raw diff, 중복 소스 설명을 쓰지 말고 작업 흐름과 판단만 정리해 주세요.`,
worklogAutomation.includeScreenshots
? `${screenshotDir} 경로에 오늘 작업 기준 전체 화면 스크린샷 1장 이상을 반드시 저장하고, 위젯/컴포넌트 단위 부분 스크린샷이 있으면 함께 연결해 주세요.`
: `${worklogPath}의 스크린샷 섹션은 현재 상태를 유지해도 됩니다.`,
worklogAutomation.includeChangedFiles
? `${worklogPath}의 소스 탭에서 이전 작업들도 Codex preview 스타일의 전체소스/raw diff를 볼 수 있게 변경/신규 파일 목록과 파일별 raw diff 기준 증적을 남겨 주세요.`
: `${worklogPath}의 파일 변경 증적은 꼭 추가하지 않아도 됩니다.`,
worklogAutomation.includeCommandLogs
? `${worklogPath}의 실행 커맨드 섹션에 오늘 사용한 주요 명령을 정리해 주세요.`
: `${worklogPath}의 실행 커맨드 섹션은 비워 두어도 됩니다.`,
`업무일지 템플릿은 ${templateLabel} 기준으로 정리해 주세요.`,
]
.filter(Boolean)
.join('\n');
}
private buildAutomationFinalStageTitle(planLabel: string) {
return `[${planLabel}] 자동화 마지막 단계`;
}
private async notifyAutomationFinalStage(
planId: number,
workId: string,
planLabel: string,
body: string,
eventType: string,
) {
await this.notifyPlan(
planId,
workId,
this.buildAutomationFinalStageTitle(planLabel),
body,
eventType,
);
}
private async runCodexCommandWithProgressNotifications(
planId: number,
workId: string,
note: unknown,
) {
const env = getEnv();
const runCodexCommandAttempt = async (attempt: number) =>
await new Promise<string>((resolve, reject) => {
let settled = false;
let stdout = '';
let stderr = '';
let notificationCount = 0;
let progressTimer: NodeJS.Timeout | null = null;
const planLabel = formatPlanNotificationLabel(workId, planId);
const child = spawn('node', [env.PLAN_CODEX_RUNNER_PATH], {
cwd: env.PLAN_GIT_REPO_PATH,
stdio: ['ignore', 'pipe', 'pipe'],
env: {
...process.env,
PLAN_REPO_PATH: env.PLAN_LOCAL_MAIN_MODE ? env.PLAN_MAIN_PROJECT_REPO_PATH : env.PLAN_GIT_REPO_PATH,
PLAN_API_BASE_URL: 'http://127.0.0.1:3100/api',
PLAN_ACCESS_TOKEN: ERROR_LOG_VIEW_TOKEN,
PLAN_ITEM_ID: String(planId),
PLAN_CODEX_BIN: env.PLAN_CODEX_BIN,
PLAN_CODEX_TEMPLATE_HOME: env.PLAN_CODEX_TEMPLATE_HOME,
PLAN_GIT_USER_NAME: env.PLAN_GIT_USER_NAME,
PLAN_GIT_USER_EMAIL: env.PLAN_GIT_USER_EMAIL,
PLAN_LOCAL_MAIN_MODE: env.PLAN_LOCAL_MAIN_MODE ? 'true' : 'false',
PLAN_SKIP_WORK_COMPLETE: 'true',
},
});
const clearProgressTimer = () => {
if (progressTimer) {
clearTimeout(progressTimer);
progressTimer = null;
}
};
const scheduleProgressNotification = (delayMs: number) => {
progressTimer = setTimeout(() => {
void (async () => {
const stillOwned = await isPlanLockedByWorker(planId, this.workerId);
if (!stillOwned) {
clearProgressTimer();
return;
}
const mergedOutput = `${stdout}\n${stderr}`.trim();
const progressSummary = this.summarizeAutomationProgress(mergedOutput);
const elapsedMinutes =
notificationCount === 0 ? 1 : notificationCount === 1 ? 3 : 3 + (notificationCount - 1) * 3;
notificationCount += 1;
await this.notifyPlan(
planId,
workId,
this.buildProgressNotificationTitle(planId, planLabel, progressSummary),
`${elapsedMinutes}분 경과했습니다.\n현재 처리: ${progressSummary}`,
'work-progress',
);
const nextDelayMs =
notificationCount === 1 ? SECOND_PROGRESS_NOTIFICATION_MS : REPEATED_PROGRESS_NOTIFICATION_MS;
scheduleProgressNotification(nextDelayMs);
})();
}, delayMs);
};
child.stdout?.on('data', (chunk) => {
stdout = (stdout + String(chunk)).slice(-STREAM_CAPTURE_LIMIT);
});
child.stderr?.on('data', (chunk) => {
stderr = (stderr + String(chunk)).slice(-STREAM_CAPTURE_LIMIT);
});
child.on('error', (error) => {
if (settled) {
return;
}
settled = true;
clearProgressTimer();
reject(error);
});
child.on('close', (code, signal) => {
if (settled) {
return;
}
settled = true;
clearProgressTimer();
if (code === 0) {
resolve(stdout.trim());
return;
}
const details = summarizeFailureOutput(
`${stderr}\n${stdout}`,
`Codex exited with code ${code ?? 'unknown'}${signal ? ` (${signal})` : ''}`,
);
reject(new Error(details));
});
if (attempt > 1) {
this.logger.warn(
{
planId,
workId,
attempt,
maxAttempts: PLAN_CODEX_RUNNER_MAX_ATTEMPTS,
},
'Retrying plan codex runner after transient failure',
);
}
scheduleProgressNotification(FIRST_PROGRESS_NOTIFICATION_MS);
});
let lastError: unknown = null;
for (let attempt = 1; attempt <= PLAN_CODEX_RUNNER_MAX_ATTEMPTS; attempt += 1) {
try {
return await runCodexCommandAttempt(attempt);
} catch (error) {
lastError = error;
if (attempt >= PLAN_CODEX_RUNNER_MAX_ATTEMPTS || !isTransientPlanCodexFailure(error)) {
throw error;
}
await this.notifyPlan(
planId,
workId,
`[${formatPlanNotificationLabel(workId, planId)}] 자동화 재시도`,
`Codex 내부 채널 오류로 자동 작업을 다시 시도합니다. (${attempt + 1}/${PLAN_CODEX_RUNNER_MAX_ATTEMPTS})`,
'work-progress',
);
await wait(PLAN_CODEX_RUNNER_RETRY_DELAY_MS);
}
}
throw lastError instanceof Error ? lastError : new Error(String(lastError ?? '자동 작업 실행에 실패했습니다.'));
}
private refreshInterval(env: ReturnType<typeof getEnv>) {
if (this.intervalMs === env.PLAN_WORKER_INTERVAL_MS) {
return;
}
this.intervalMs = env.PLAN_WORKER_INTERVAL_MS;
if (this.timer) {
clearInterval(this.timer);
this.timer = setInterval(() => {
void this.tick();
}, this.intervalMs);
}
}
private evaluateAutoReceiveSchedule(automation: AutomationSnapshot | undefined, now: Date): AutoReceiveScheduleEvaluation {
const scheduleType = automation?.autoReceiveScheduleType ?? 'interval';
if (scheduleType === 'daily') {
const dailyTime = isValidTimeValue(automation?.autoReceiveDailyTime) ? automation.autoReceiveDailyTime : '09:00';
const todayScheduledAt = withTime(now, dailyTime);
const nextEligibleAt = now.getTime() >= todayScheduledAt.getTime()
? todayScheduledAt
: todayScheduledAt;
const alreadyProcessedToday =
this.lastAutoReceiveAt &&
this.lastAutoReceiveAt.getFullYear() === now.getFullYear() &&
this.lastAutoReceiveAt.getMonth() === now.getMonth() &&
this.lastAutoReceiveAt.getDate() === now.getDate() &&
this.lastAutoReceiveAt.getTime() >= todayScheduledAt.getTime();
return {
due: now.getTime() >= todayScheduledAt.getTime() && !alreadyProcessedToday,
nextEligibleAt:
now.getTime() >= todayScheduledAt.getTime()
? withTime(new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1), dailyTime)
: todayScheduledAt,
scheduleLabel: `daily:${dailyTime}`,
};
}
if (scheduleType === 'weekly') {
const weeklyDay = automation?.autoReceiveWeeklyDay ?? 'mon';
const weeklyTime = isValidTimeValue(automation?.autoReceiveWeeklyTime) ? automation.autoReceiveWeeklyTime : '09:00';
const currentScheduledAt = getNextWeeklyDate(new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), weeklyDay, weeklyTime);
const alreadyProcessedThisWeek =
this.lastAutoReceiveAt !== null && this.lastAutoReceiveAt.getTime() >= currentScheduledAt.getTime();
return {
due: now.getTime() >= currentScheduledAt.getTime() && !alreadyProcessedThisWeek,
nextEligibleAt: getNextWeeklyDate(now, weeklyDay, weeklyTime),
scheduleLabel: `weekly:${weeklyDay}:${weeklyTime}`,
};
}
const intervalSeconds =
typeof automation?.autoReceiveIntervalSeconds === 'number' && Number.isFinite(automation.autoReceiveIntervalSeconds)
? Math.max(1, Math.round(automation.autoReceiveIntervalSeconds))
: 30;
const lastAutoReceiveAt = this.lastAutoReceiveAt;
const nextEligibleAt = lastAutoReceiveAt
? new Date(lastAutoReceiveAt.getTime() + intervalSeconds * 1000)
: getStartOfMinute(now);
return {
due: !lastAutoReceiveAt || now.getTime() >= nextEligibleAt.getTime(),
nextEligibleAt,
scheduleLabel: `interval:${intervalSeconds}`,
};
}
private async tick() {
if (this.running) {
return;
}
this.running = true;
try {
const env = getEnv();
const appConfig = await getAppConfigSnapshot();
const autoRefreshEnabled = appConfig.automation?.autoRefreshEnabled ?? true;
const autoRefreshIntervalSeconds = appConfig.automation?.autoRefreshIntervalSeconds;
const resolvedIntervalMs =
typeof autoRefreshIntervalSeconds === 'number' && Number.isFinite(autoRefreshIntervalSeconds)
? Math.max(1, Math.round(autoRefreshIntervalSeconds)) * 1000
: env.PLAN_WORKER_INTERVAL_MS;
if (!env.PLAN_WORKER_ENABLED || !autoRefreshEnabled) {
return;
}
this.refreshInterval({
...env,
PLAN_WORKER_INTERVAL_MS: resolvedIntervalMs,
});
await this.processWorklogAutomation(appConfig);
await this.processPlanScheduledTasks();
await this.processRegisteredPlans();
await this.processExecutablePlans(appConfig);
await this.processReleaseReadyPlans();
await this.processMainMergeReadyPlans();
} finally {
this.running = false;
}
}
private async processPlanScheduledTasks() {
try {
const registeredPlans = await registerDuePlanScheduledTasks();
if (registeredPlans.length > 0) {
this.logger.info({ count: registeredPlans.length }, 'Plan scheduled tasks registered');
}
} catch (error) {
const handledError = error instanceof Error ? error : new Error(String(error));
this.logger.error({ err: handledError }, 'Plan scheduled task processing failed');
await createErrorLog({
source: 'automation',
sourceLabel: 'Plan 스케줄',
errorType: 'PlanScheduleFailed',
errorName: handledError.name,
errorMessage: handledError.message,
stackTrace: handledError.stack ?? null,
context: {
workerId: this.workerId,
},
});
}
}
private async processRegisteredPlans() {
const env = getEnv();
const row = await claimNextPlanForBranch(this.workerId);
if (!row) {
return;
}
const item = mapPlanRow(row);
const planId = Number(item.id);
const planLabel = formatPlanNotificationLabel(String(item.workId), planId);
const assignedBranch = String(item.assignedBranch ?? '');
const releaseTarget = String(item.releaseTarget ?? env.PLAN_RELEASE_BRANCH);
try {
if (!this.isLocalMainMode()) {
await cleanAutomationWorktree(env.PLAN_GIT_REPO_PATH);
await ensureBranchExists(
{
repoPath: env.PLAN_GIT_REPO_PATH,
releaseBranch: env.PLAN_RELEASE_BRANCH,
mainBranch: env.PLAN_MAIN_BRANCH,
},
assignedBranch,
releaseTarget,
);
}
const updatedRow = await markPlanBranchReady(planId, this.workerId);
if (!updatedRow) {
this.logger.info({ planId }, 'Plan branch creation result ignored because ownership changed');
return;
}
this.logger.info(
{ planId, branch: assignedBranch, localMainMode: this.isLocalMainMode() },
this.isLocalMainMode() ? 'Plan local main execution prepared' : 'Plan branch created',
);
} catch (error) {
const message = error instanceof Error ? error.message : '브랜치 생성에 실패했습니다.';
await this.persistAutomationErrorLog({
planId,
workId: String(item.workId),
workerStatus: '브랜치실패',
error,
detail: {
stage: 'branch',
workerId: this.workerId,
assignedBranch,
releaseTarget,
},
});
const failureRow = await markPlanAutomationFailure(planId, this.workerId, '브랜치실패', message);
if (!failureRow) {
this.logger.info({ planId }, 'Plan branch failure ignored because ownership changed');
return;
}
await this.notifyAutomationFinalStage(
planId,
String(item.workId),
planLabel,
`브랜치 생성 실패\n${message}`,
'branch-failed',
);
this.logger.error({ planId, err: error }, 'Plan branch creation failed');
}
}
private async processReleaseReadyPlans() {
const env = getEnv();
const row = await claimNextPlanForMerge(this.workerId);
if (!row) {
return;
}
const item = mapPlanRow(row);
const planId = Number(item.id);
const planLabel = formatPlanNotificationLabel(String(item.workId), planId);
const assignedBranch = String(item.assignedBranch ?? '');
const releaseTarget = String(item.releaseTarget ?? env.PLAN_RELEASE_BRANCH);
const autoDeployToMain = Boolean(item.autoDeployToMain ?? true);
try {
if (this.isLocalMainMode()) {
const completedRow = await markPlanAsCompleted(
planId,
'로컬 main 직접 작업 모드에서 release/main 반영 단계를 건너뛰고 완료 처리했습니다.',
);
if (!completedRow) {
this.logger.info({ planId }, 'Plan local main completion ignored because ownership changed');
return;
}
await this.notifyAutomationFinalStage(
planId,
String(item.workId),
planLabel,
'로컬 main 직접 작업이 이미 반영되어 별도 release 반영 없이 완료 처리했습니다.',
'work-local-main-complete',
);
this.logger.info({ planId, branch: assignedBranch, releaseTarget }, 'Plan completed in local main mode without release merge');
return;
}
await cleanAutomationWorktree(env.PLAN_GIT_REPO_PATH);
await mergeBranchToRelease(
{
repoPath: env.PLAN_GIT_REPO_PATH,
releaseBranch: env.PLAN_RELEASE_BRANCH,
mainBranch: env.PLAN_MAIN_BRANCH,
},
assignedBranch,
releaseTarget,
);
await pushBranch(env.PLAN_GIT_REPO_PATH, releaseTarget);
const updatedRow = await markPlanReleaseMerged(planId, this.workerId);
if (!updatedRow) {
this.logger.info({ planId }, 'Plan release merge result ignored because ownership changed');
return;
}
if (!autoDeployToMain) {
await this.notifyAutomationFinalStage(
planId,
String(item.workId),
planLabel,
`${releaseTarget} 브랜치 반영이 완료되었습니다.`,
'release-merged',
);
}
this.logger.info(
{ planId, branch: assignedBranch, releaseTarget },
'Plan branch merged to release',
);
} catch (error) {
const message = error instanceof Error ? error.message : 'release 브랜치 반영에 실패했습니다.';
await this.persistAutomationErrorLog({
planId,
workId: String(item.workId),
workerStatus: 'release반영실패',
error,
detail: {
stage: 'release-merge',
workerId: this.workerId,
assignedBranch,
releaseTarget,
autoDeployToMain,
},
});
const failureRow = await markPlanAutomationFailure(planId, this.workerId, 'release반영실패', message);
if (!failureRow) {
this.logger.info({ planId }, 'Plan release merge failure ignored because ownership changed');
return;
}
await this.notifyAutomationFinalStage(
planId,
String(item.workId),
planLabel,
`release 반영 실패\n${message}`,
'release-failed',
);
this.logger.error({ planId, err: error }, 'Plan release merge failed');
}
}
private async processMainMergeReadyPlans() {
const env = getEnv();
const row = await claimNextPlanForMainMerge(this.workerId);
if (!row) {
return;
}
const item = mapPlanRow(row);
const planId = Number(item.id);
const assignedBranch = String(item.assignedBranch ?? '');
const releaseTarget = String(item.releaseTarget ?? env.PLAN_RELEASE_BRANCH);
const workId = String(item.workId);
const planLabel = formatPlanNotificationLabel(workId, planId);
try {
if (this.isLocalMainMode()) {
const completedRow = await markPlanAsCompleted(
planId,
'로컬 main 직접 작업 모드에서 main 반영 단계를 건너뛰고 완료 처리했습니다.',
'main반영완료',
);
if (!completedRow) {
this.logger.info({ planId }, 'Plan local main main-merge completion ignored because ownership changed');
return;
}
await this.notifyAutomationFinalStage(
planId,
workId,
planLabel,
'로컬 main 직접 작업이 이미 반영되어 별도 main merge 없이 완료 처리했습니다.',
'work-local-main-complete',
);
this.logger.info({ planId, branch: assignedBranch, releaseTarget }, 'Plan completed in local main mode without main merge');
return;
}
await cleanAutomationWorktree(env.PLAN_GIT_REPO_PATH);
await mergeReleaseToMain(
{
repoPath: env.PLAN_GIT_REPO_PATH,
releaseBranch: env.PLAN_RELEASE_BRANCH,
mainBranch: env.PLAN_MAIN_BRANCH,
},
releaseTarget,
);
await pushBranch(env.PLAN_GIT_REPO_PATH, env.PLAN_MAIN_BRANCH);
await pullMainProjectBranch(env.PLAN_MAIN_PROJECT_REPO_PATH, env.PLAN_MAIN_BRANCH);
const mergeResult = await markPlanMerged(planId, this.workerId);
if (!mergeResult?.mergedRow) {
this.logger.info({ planId }, 'Plan main merge result ignored because ownership changed');
return;
}
const notificationRows = mergeResult.notificationRows.length > 0 ? mergeResult.notificationRows : [mergeResult.mergedRow];
for (const notificationRow of notificationRows) {
const notificationItem = mapPlanRow(notificationRow);
const notificationPlanId = Number(notificationItem.id);
const notificationWorkId = String(notificationItem.workId);
const notificationPlanLabel = formatPlanNotificationLabel(notificationWorkId, notificationPlanId);
await this.notifyAutomationFinalStage(
notificationPlanId,
notificationWorkId,
notificationPlanLabel,
`${env.PLAN_MAIN_BRANCH} 브랜치 반영 후 메인 프로젝트 pull까지 완료되었습니다.`,
'main-merged',
);
}
this.logger.info(
{
planId,
branch: assignedBranch,
releaseTarget,
mainBranch: env.PLAN_MAIN_BRANCH,
mainProjectRepoPath: env.PLAN_MAIN_PROJECT_REPO_PATH,
notifiedPlanIds: notificationRows.map((notificationRow) => Number(notificationRow.id)).filter(Number.isFinite),
},
'Plan release branch merged to main and main project synced',
);
} catch (error) {
const message = error instanceof Error ? error.message : 'main 브랜치 반영에 실패했습니다.';
await this.persistAutomationErrorLog({
planId,
workId,
workerStatus: 'main반영실패',
error,
detail: {
stage: 'main-merge',
workerId: this.workerId,
assignedBranch,
releaseTarget,
mainBranch: env.PLAN_MAIN_BRANCH,
mainProjectRepoPath: env.PLAN_MAIN_PROJECT_REPO_PATH,
},
});
const failureRows = await markPlanMainMergeFailure(releaseTarget, this.workerId, message);
if (!failureRows.length) {
this.logger.info({ planId }, 'Plan main merge failure ignored because ownership changed');
return;
}
await this.notifyAutomationFinalStage(
planId,
workId,
planLabel,
`main 일괄 반영 실패\n${message}`,
'main-failed',
);
this.logger.error({ planId, err: error }, 'Plan main merge failed');
}
}
private async processExecutablePlans(appConfig: Awaited<ReturnType<typeof getAppConfigSnapshot>> | undefined) {
const env = getEnv();
if (!env.PLAN_CODEX_ENABLED) {
return;
}
const schedule = this.evaluateAutoReceiveSchedule(appConfig?.automation, new Date());
if (!schedule.due) {
return;
}
const row = await claimNextPlanForExecution(this.workerId);
if (!row) {
return;
}
this.lastAutoReceiveAt = new Date();
const item = mapPlanRow(row);
const planId = Number(item.id);
const workId = String(item.workId);
const planLabel = formatPlanNotificationLabel(workId, planId);
const autoDeployToMain = Boolean(item.autoDeployToMain ?? true);
try {
await this.notifyPlan(
planId,
workId,
`[${planLabel}] 자동화 시작`,
this.buildExecutionStartedBody(item.note),
'work-started',
);
const output = await this.runCodexCommandWithProgressNotifications(planId, workId, item.note);
if (output.includes('처리할 Plan 항목이 없습니다.')) {
throw new Error('자동 작업 대상 Plan 항목을 찾지 못했습니다. 상태 전환 로직을 확인해 주세요.');
}
if (output.includes('"outcome": "noop-complete"')) {
await this.notifyAutomationFinalStage(
planId,
workId,
planLabel,
'변경 사항 없이 자동 작업이 완료되었습니다.',
'work-noop-complete',
);
return;
}
if (output.includes('"outcome": "board-post-complete"')) {
await this.notifyAutomationFinalStage(
planId,
workId,
planLabel,
'Plan 게시판 미접수 글 작성이 완료되었습니다.',
'work-board-post-complete',
);
return;
}
const finalCompletedRow = this.isLocalMainMode()
? await markPlanAsCompleted(planId, '로컬 main 직접 작업으로 자동 작업을 완료했습니다.')
: await markPlanWorkCompleted(planId, this.workerId, '자동 작업을 완료했습니다.');
if (!finalCompletedRow) {
this.logger.info({ planId }, 'Plan Codex completion ignored because ownership changed');
return;
}
await this.notifyAutomationFinalStage(
planId,
workId,
planLabel,
this.isLocalMainMode() ? '자동 작업이 로컬 main 작업본에 직접 반영되어 완료되었습니다.' : this.buildExecutionCompletedBody(autoDeployToMain),
'work-completed',
);
this.logger.info({ planId }, 'Plan Codex execution completed');
} catch (error) {
const rawMessage = error instanceof Error ? error.message : '자동 작업 실행에 실패했습니다.';
const message = appendCodexHomeHint(rawMessage, env);
await this.persistAutomationErrorLog({
planId,
workId,
workerStatus: '자동작업실패',
error,
detail: {
stage: 'codex-execution',
workerId: this.workerId,
assignedBranch: item.assignedBranch ?? null,
autoDeployToMain,
releaseTarget: item.releaseTarget ?? env.PLAN_RELEASE_BRANCH,
},
});
const failureRow = await markPlanAutomationFailure(planId, this.workerId, '자동작업실패', message);
if (!failureRow) {
this.logger.info({ planId }, 'Plan Codex failure ignored because ownership changed');
return;
}
await this.notifyAutomationFinalStage(
planId,
workId,
planLabel,
`자동 작업 실패\n${message}`,
'work-failed',
);
this.logger.error({ planId, err: error }, 'Plan Codex execution failed');
}
}
private async processWorklogAutomation(appConfig: Awaited<ReturnType<typeof getAppConfigSnapshot>> | undefined) {
const env = getEnv();
const worklogAutomation = appConfig?.worklogAutomation;
if (!worklogAutomation?.autoCreateDailyWorklog) {
return;
}
const now = new Date();
const { dateKey } = getKstNowParts(now);
const dailyCreateTime = normalizeDailyCreateTime(worklogAutomation.dailyCreateTime);
const alreadyExecutedToday = await hasWorklogAutomationRunForDate(dateKey);
const repeatRequestEnabled = false;
const repeatIntervalMinutes = normalizeRepeatIntervalMinutes(worklogAutomation.repeatIntervalMinutes);
if (!isWorklogCreationDue(now, dailyCreateTime, false)) {
return;
}
if (alreadyExecutedToday && !repeatRequestEnabled) {
return;
}
if (alreadyExecutedToday && this.lastWorklogAutomationRequestAt) {
const nextRepeatAt = new Date(
this.lastWorklogAutomationRequestAt.getTime() + repeatIntervalMinutes * 60 * 1000,
);
if (now.getTime() < nextRepeatAt.getTime()) {
return;
}
}
try {
const worklogPath = await ensureDailyWorklogFile(env.PLAN_GIT_REPO_PATH, dateKey);
const autoWorklogPlanWorkId = buildAutoWorklogPlanWorkId(dateKey);
const upsertResult = await upsertAutoPlanItem({
workId: autoWorklogPlanWorkId,
note: this.buildAutoWorklogPlanNote(dateKey, worklogAutomation),
releaseTarget: env.PLAN_RELEASE_BRANCH,
jangsingProcessingRequired: false,
autoDeployToMain: true,
requeue: !alreadyExecutedToday,
});
this.lastWorklogAutomationRequestAt = now;
if (upsertResult.action === 'created' || upsertResult.action === 'requeued') {
this.logger.info(
{ dateKey, workId: autoWorklogPlanWorkId, worklogPath, repeatRequestEnabled, repeatIntervalMinutes },
'Daily worklog automation plan registered',
);
}
if (!alreadyExecutedToday) {
await markWorklogAutomationRun({
runDate: dateKey,
scheduledTime: dailyCreateTime,
worklogPath,
});
}
this.logger.info(
{ dateKey, worklogPath, dailyCreateTime, repeatRequestEnabled, repeatIntervalMinutes },
'Daily worklog automation completed',
);
} catch (error) {
const handledError = error instanceof Error ? error : new Error(String(error));
this.logger.error({ err: handledError, dateKey, dailyCreateTime }, 'Daily worklog automation failed');
await createErrorLog({
source: 'automation',
sourceLabel: '업무일지 자동화',
errorType: 'WorklogAutomationFailed',
errorName: handledError.name,
errorMessage: handledError.message,
stackTrace: handledError.stack ?? null,
context: {
dateKey,
dailyCreateTime,
repeatRequestEnabled,
repeatIntervalMinutes,
repoPath: env.PLAN_GIT_REPO_PATH,
},
});
}
}
}