453 lines
20 KiB
Bash
Executable File
453 lines
20 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
set -eu
|
|
|
|
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
|
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd)
|
|
COMPOSE_FILE="$REPO_ROOT/etc/servers/work-server/docker-compose.yml"
|
|
PROXY_SERVICE="${WORK_SERVER_PROXY_SERVICE:-work-server}"
|
|
PROXY_CONTAINER="${WORK_SERVER_PROXY_CONTAINER:-work-server}"
|
|
BLUE_SERVICE="${WORK_SERVER_BLUE_SERVICE:-work-server-blue}"
|
|
GREEN_SERVICE="${WORK_SERVER_GREEN_SERVICE:-work-server-green}"
|
|
BLUE_CONTAINER="${WORK_SERVER_BLUE_CONTAINER:-work-server-blue}"
|
|
GREEN_CONTAINER="${WORK_SERVER_GREEN_CONTAINER:-work-server-green}"
|
|
ACTIVE_SLOT_FILE="${WORK_SERVER_ACTIVE_SLOT_FILE:-$REPO_ROOT/etc/servers/work-server/.docker/runtime/active-slot}"
|
|
PROXY_CONFIG_FILE="${WORK_SERVER_PROXY_CONFIG_FILE:-$REPO_ROOT/etc/servers/work-server/.docker/proxy/default.conf}"
|
|
HEALTH_ENDPOINT="${WORK_SERVER_HEALTH_ENDPOINT:-http://127.0.0.1:3100/health}"
|
|
RUNTIME_ENDPOINT="${WORK_SERVER_RUNTIME_ENDPOINT:-http://127.0.0.1:3100/api/runtime}"
|
|
RECOVERY_ENDPOINT="${WORK_SERVER_RECOVERY_ENDPOINT:-http://127.0.0.1:3100/api/runtime/recover-interrupted-chat}"
|
|
PREVIOUS_SLOT_DRAIN_TIMEOUT_SECONDS="${WORK_SERVER_PREVIOUS_SLOT_DRAIN_TIMEOUT_SECONDS:-900}"
|
|
LOCK_FILE="${WORK_SERVER_RESTART_LOCK_FILE:-$REPO_ROOT/etc/servers/work-server/.docker/runtime/restart-in-progress.json}"
|
|
DEPLOY_STATE_FILE="${WORK_SERVER_DEPLOY_STATE_FILE:-$REPO_ROOT/etc/servers/work-server/.docker/runtime/deployment-state.json}"
|
|
DEPLOY_FINISHED="false"
|
|
LAST_DEPLOY_ERROR=""
|
|
LAST_DEPLOY_LOG=""
|
|
PREVIOUS_ACTIVE_COUNT=""
|
|
PREVIOUS_QUEUED_COUNT=""
|
|
RECOVERED_SESSION_COUNT=""
|
|
RECOVERED_RESTARTED_COUNT=""
|
|
RECOVERED_REQUEUED_COUNT=""
|
|
|
|
cd "$REPO_ROOT"
|
|
|
|
mkdir -p "$(dirname "$ACTIVE_SLOT_FILE")" "$(dirname "$PROXY_CONFIG_FILE")" "$(dirname "$LOCK_FILE")" "$(dirname "$DEPLOY_STATE_FILE")"
|
|
write_deploy_state() {
|
|
DEPLOY_STATUS="$1"
|
|
DEPLOY_PHASE="$2"
|
|
DEPLOY_SUMMARY="$3"
|
|
DEPLOY_STEP_KEY="${4:-}"
|
|
DEPLOY_STEP_STATUS="${5:-}"
|
|
DEPLOY_STEP_DETAIL="${6:-}"
|
|
DEPLOY_LAST_ERROR="${7:-}"
|
|
DEPLOY_LOG_EXCERPT="${8:-}"
|
|
DEPLOY_ACTIVE_SLOT_VALUE="${ACTIVE_SLOT:-}"
|
|
DEPLOY_TARGET_SLOT_VALUE="${TARGET_SLOT:-}"
|
|
DEPLOY_PREVIOUS_SLOT_VALUE="${PREVIOUS_SLOT:-}"
|
|
DEPLOY_TARGET_CONTAINER_VALUE="${TARGET_CONTAINER:-}"
|
|
DEPLOY_PREVIOUS_CONTAINER_VALUE="${PREVIOUS_CONTAINER:-}"
|
|
DEPLOY_PREVIOUS_ACTIVE_COUNT_VALUE="${PREVIOUS_ACTIVE_COUNT:-}"
|
|
DEPLOY_PREVIOUS_QUEUED_COUNT_VALUE="${PREVIOUS_QUEUED_COUNT:-}"
|
|
DEPLOY_RECOVERED_SESSION_COUNT_VALUE="${RECOVERED_SESSION_COUNT:-}"
|
|
DEPLOY_RECOVERED_RESTARTED_COUNT_VALUE="${RECOVERED_RESTARTED_COUNT:-}"
|
|
DEPLOY_RECOVERED_REQUEUED_COUNT_VALUE="${RECOVERED_REQUEUED_COUNT:-}"
|
|
export \
|
|
DEPLOY_STATUS \
|
|
DEPLOY_PHASE \
|
|
DEPLOY_SUMMARY \
|
|
DEPLOY_STEP_KEY \
|
|
DEPLOY_STEP_STATUS \
|
|
DEPLOY_STEP_DETAIL \
|
|
DEPLOY_LAST_ERROR \
|
|
DEPLOY_LOG_EXCERPT \
|
|
DEPLOY_ACTIVE_SLOT_VALUE \
|
|
DEPLOY_TARGET_SLOT_VALUE \
|
|
DEPLOY_PREVIOUS_SLOT_VALUE \
|
|
DEPLOY_TARGET_CONTAINER_VALUE \
|
|
DEPLOY_PREVIOUS_CONTAINER_VALUE \
|
|
DEPLOY_PREVIOUS_ACTIVE_COUNT_VALUE \
|
|
DEPLOY_PREVIOUS_QUEUED_COUNT_VALUE \
|
|
DEPLOY_RECOVERED_SESSION_COUNT_VALUE \
|
|
DEPLOY_RECOVERED_RESTARTED_COUNT_VALUE \
|
|
DEPLOY_RECOVERED_REQUEUED_COUNT_VALUE
|
|
node - "$DEPLOY_STATE_FILE" <<'NODE'
|
|
const fs = require('fs');
|
|
const filePath = process.argv[2];
|
|
const env = process.env;
|
|
const stepKeys = [
|
|
'build-target-slot',
|
|
'verify-target-health',
|
|
'switch-proxy',
|
|
'drain-previous-slot',
|
|
'rebuild-previous-slot',
|
|
'recover-interrupted-chat',
|
|
];
|
|
const readJson = () => {
|
|
try {
|
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
const parseIso = (value) => {
|
|
if (!value || typeof value !== 'string') {
|
|
return null;
|
|
}
|
|
const date = new Date(value);
|
|
return Number.isNaN(date.getTime()) ? null : date.toISOString();
|
|
};
|
|
const parseSlot = (value) => (value === 'blue' || value === 'green' ? value : null);
|
|
const parseCount = (value) => {
|
|
if (value == null || value === '') {
|
|
return null;
|
|
}
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
};
|
|
const current = readJson() || {};
|
|
const shouldResetForNewRun =
|
|
env.DEPLOY_STATUS === 'running'
|
|
&& env.DEPLOY_PHASE === 'build-target-slot'
|
|
&& !env.DEPLOY_STEP_KEY;
|
|
const now = new Date().toISOString();
|
|
const stepsByKey = new Map();
|
|
for (const stepKey of stepKeys) {
|
|
const existing = !shouldResetForNewRun && Array.isArray(current.steps)
|
|
? current.steps.find((item) => item && item.key === stepKey)
|
|
: null;
|
|
stepsByKey.set(stepKey, {
|
|
key: stepKey,
|
|
status:
|
|
existing?.status === 'running' || existing?.status === 'completed' || existing?.status === 'failed'
|
|
? existing.status
|
|
: 'pending',
|
|
detail: typeof existing?.detail === 'string' ? existing.detail : null,
|
|
updatedAt: parseIso(existing?.updatedAt) || null,
|
|
});
|
|
}
|
|
if (env.DEPLOY_STEP_KEY && stepsByKey.has(env.DEPLOY_STEP_KEY)) {
|
|
const target = stepsByKey.get(env.DEPLOY_STEP_KEY);
|
|
target.status =
|
|
env.DEPLOY_STEP_STATUS === 'running'
|
|
|| env.DEPLOY_STEP_STATUS === 'completed'
|
|
|| env.DEPLOY_STEP_STATUS === 'failed'
|
|
? env.DEPLOY_STEP_STATUS
|
|
: 'pending';
|
|
target.detail = env.DEPLOY_STEP_DETAIL || target.detail || null;
|
|
target.updatedAt = now;
|
|
}
|
|
const payload = {
|
|
status:
|
|
env.DEPLOY_STATUS === 'running' || env.DEPLOY_STATUS === 'completed' || env.DEPLOY_STATUS === 'failed'
|
|
? env.DEPLOY_STATUS
|
|
: 'idle',
|
|
phase:
|
|
env.DEPLOY_PHASE === 'build-target-slot'
|
|
|| env.DEPLOY_PHASE === 'verify-target-health'
|
|
|| env.DEPLOY_PHASE === 'switch-proxy'
|
|
|| env.DEPLOY_PHASE === 'drain-previous-slot'
|
|
|| env.DEPLOY_PHASE === 'rebuild-previous-slot'
|
|
|| env.DEPLOY_PHASE === 'recover-interrupted-chat'
|
|
|| env.DEPLOY_PHASE === 'completed'
|
|
|| env.DEPLOY_PHASE === 'failed'
|
|
? env.DEPLOY_PHASE
|
|
: 'idle',
|
|
summary: env.DEPLOY_SUMMARY || (!shouldResetForNewRun ? current.summary : null) || null,
|
|
startedAt: shouldResetForNewRun ? now : parseIso(current.startedAt) || now,
|
|
updatedAt: now,
|
|
completedAt:
|
|
shouldResetForNewRun
|
|
? null
|
|
: env.DEPLOY_STATUS === 'completed' || env.DEPLOY_STATUS === 'failed'
|
|
? now
|
|
: parseIso(current.completedAt),
|
|
activeSlot: parseSlot(env.DEPLOY_ACTIVE_SLOT_VALUE) || (!shouldResetForNewRun ? parseSlot(current.activeSlot) : null),
|
|
targetSlot: parseSlot(env.DEPLOY_TARGET_SLOT_VALUE) || (!shouldResetForNewRun ? parseSlot(current.targetSlot) : null),
|
|
previousSlot: parseSlot(env.DEPLOY_PREVIOUS_SLOT_VALUE) || (!shouldResetForNewRun ? parseSlot(current.previousSlot) : null),
|
|
targetContainer: env.DEPLOY_TARGET_CONTAINER_VALUE || (!shouldResetForNewRun ? current.targetContainer : null) || null,
|
|
previousContainer: env.DEPLOY_PREVIOUS_CONTAINER_VALUE || (!shouldResetForNewRun ? current.previousContainer : null) || null,
|
|
previousSlotActiveChatRequestCount:
|
|
parseCount(env.DEPLOY_PREVIOUS_ACTIVE_COUNT_VALUE)
|
|
?? (!shouldResetForNewRun ? parseCount(current.previousSlotActiveChatRequestCount) : null),
|
|
previousSlotQueuedChatRequestCount:
|
|
parseCount(env.DEPLOY_PREVIOUS_QUEUED_COUNT_VALUE)
|
|
?? (!shouldResetForNewRun ? parseCount(current.previousSlotQueuedChatRequestCount) : null),
|
|
recoveredSessionCount:
|
|
parseCount(env.DEPLOY_RECOVERED_SESSION_COUNT_VALUE)
|
|
?? (!shouldResetForNewRun ? parseCount(current.recoveredSessionCount) : null),
|
|
recoveredRestartedCount:
|
|
parseCount(env.DEPLOY_RECOVERED_RESTARTED_COUNT_VALUE)
|
|
?? (!shouldResetForNewRun ? parseCount(current.recoveredRestartedCount) : null),
|
|
recoveredRequeuedCount:
|
|
parseCount(env.DEPLOY_RECOVERED_REQUEUED_COUNT_VALUE)
|
|
?? (!shouldResetForNewRun ? parseCount(current.recoveredRequeuedCount) : null),
|
|
lastError: env.DEPLOY_LAST_ERROR || (!shouldResetForNewRun ? current.lastError : null) || null,
|
|
logExcerpt: env.DEPLOY_LOG_EXCERPT || (!shouldResetForNewRun ? current.logExcerpt : null) || null,
|
|
steps: stepKeys.map((stepKey) => stepsByKey.get(stepKey)),
|
|
};
|
|
fs.writeFileSync(filePath, JSON.stringify(payload) + '\n', 'utf8');
|
|
NODE
|
|
}
|
|
cleanup_restart_lock() {
|
|
EXIT_CODE="$1"
|
|
if [ "$DEPLOY_FINISHED" != "true" ]; then
|
|
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
SUMMARY="WORK-SERVER 배포가 중단되었습니다."
|
|
DETAIL="${LAST_DEPLOY_LOG:-알 수 없는 오류로 배포가 중단되었습니다.}"
|
|
ERROR_TEXT="${LAST_DEPLOY_ERROR:-$SUMMARY}"
|
|
else
|
|
SUMMARY="WORK-SERVER 배포 완료 표기 전에 스크립트가 종료되었습니다."
|
|
DETAIL="${LAST_DEPLOY_LOG:-completed 상태를 기록하기 전에 스크립트가 종료되었습니다.}"
|
|
ERROR_TEXT="${LAST_DEPLOY_ERROR:-$SUMMARY}"
|
|
fi
|
|
write_deploy_state failed failed "$SUMMARY" "" "" "" "$ERROR_TEXT" "$DETAIL"
|
|
fi
|
|
rm -f "$LOCK_FILE"
|
|
}
|
|
trap 'cleanup_restart_lock "$?"' EXIT INT TERM
|
|
|
|
read_active_slot() {
|
|
if [ -f "$ACTIVE_SLOT_FILE" ]; then
|
|
SLOT=$(tr -d '[:space:]' <"$ACTIVE_SLOT_FILE")
|
|
if [ "$SLOT" = "blue" ] || [ "$SLOT" = "green" ]; then
|
|
printf '%s' "$SLOT"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
printf 'blue'
|
|
}
|
|
|
|
container_is_running() {
|
|
CONTAINER_NAME="$1"
|
|
STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME" 2>/dev/null || true)
|
|
[ "$STATUS" = "running" ]
|
|
}
|
|
|
|
resolve_active_slot() {
|
|
SLOT=$(read_active_slot)
|
|
|
|
if [ "$SLOT" = "blue" ] && ! container_is_running "$BLUE_CONTAINER" && container_is_running "$GREEN_CONTAINER"; then
|
|
printf 'green'
|
|
return 0
|
|
fi
|
|
|
|
if [ "$SLOT" = "green" ] && ! container_is_running "$GREEN_CONTAINER" && container_is_running "$BLUE_CONTAINER"; then
|
|
printf 'blue'
|
|
return 0
|
|
fi
|
|
|
|
printf '%s' "$SLOT"
|
|
}
|
|
|
|
write_proxy_config() {
|
|
SLOT="$1"
|
|
TARGET_CONTAINER="$BLUE_CONTAINER"
|
|
|
|
if [ "$SLOT" = "green" ]; then
|
|
TARGET_CONTAINER="$GREEN_CONTAINER"
|
|
fi
|
|
|
|
cat >"$PROXY_CONFIG_FILE" <<EOF2
|
|
server {
|
|
listen 3100;
|
|
server_name _;
|
|
|
|
location /ws/chat {
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Host \$host;
|
|
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
|
proxy_set_header X-Forwarded-Proto \$scheme;
|
|
proxy_set_header X-Forwarded-Host \$host;
|
|
proxy_set_header X-Forwarded-Port \$server_port;
|
|
proxy_set_header Upgrade \$http_upgrade;
|
|
proxy_set_header Connection "upgrade";
|
|
proxy_pass http://$TARGET_CONTAINER:3100;
|
|
}
|
|
|
|
location / {
|
|
proxy_http_version 1.1;
|
|
proxy_set_header Host \$host;
|
|
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
|
proxy_set_header X-Forwarded-Proto \$scheme;
|
|
proxy_set_header X-Forwarded-Host \$host;
|
|
proxy_set_header X-Forwarded-Port \$server_port;
|
|
proxy_set_header Connection "";
|
|
proxy_pass http://$TARGET_CONTAINER:3100;
|
|
}
|
|
}
|
|
EOF2
|
|
}
|
|
|
|
wait_for_container_health() {
|
|
TARGET_CONTAINER="$1"
|
|
ATTEMPT=0
|
|
|
|
while [ "$ATTEMPT" -lt 60 ]; do
|
|
if docker exec "$TARGET_CONTAINER" node -e "fetch(process.argv[1]).then(async (response) => { if (!response.ok) process.exit(1); process.stdout.write(await response.text()); }).catch(() => process.exit(1));" "$HEALTH_ENDPOINT" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
ATTEMPT=$((ATTEMPT + 1))
|
|
sleep 2
|
|
done
|
|
|
|
echo "health check failed for $TARGET_CONTAINER" >&2
|
|
return 1
|
|
}
|
|
|
|
read_runtime_value() {
|
|
TARGET_CONTAINER="$1"
|
|
FIELD_NAME="$2"
|
|
|
|
docker exec "$TARGET_CONTAINER" node -e "fetch(process.argv[1]).then((response) => response.json()).then((payload) => { const value = payload?.[process.argv[2]]; if (typeof value === 'boolean') { process.stdout.write(value ? 'true' : 'false'); return; } if (value == null) { process.stdout.write(''); return; } process.stdout.write(String(value)); }).catch(() => process.exit(1));" "$RUNTIME_ENDPOINT" "$FIELD_NAME"
|
|
}
|
|
|
|
set_container_draining() {
|
|
TARGET_CONTAINER="$1"
|
|
DRAINING_VALUE="$2"
|
|
|
|
docker exec "$TARGET_CONTAINER" node -e "fetch(process.argv[1], { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ draining: process.argv[2] === 'true' }) }).then((response) => { if (!response.ok) process.exit(1); }).catch(() => process.exit(1));" "${RUNTIME_ENDPOINT}/drain" "$DRAINING_VALUE"
|
|
}
|
|
|
|
recover_interrupted_chat_requests() {
|
|
TARGET_CONTAINER="$1"
|
|
|
|
docker exec "$TARGET_CONTAINER" node -e "fetch(process.argv[1], { method: 'POST' }).then(async (response) => { if (!response.ok) { process.stderr.write(await response.text()); process.exit(1); } process.stdout.write(await response.text()); }).catch((error) => { process.stderr.write(String(error)); process.exit(1); });" "$RECOVERY_ENDPOINT"
|
|
}
|
|
|
|
wait_for_previous_slot_drain() {
|
|
TARGET_CONTAINER="$1"
|
|
ELAPSED=0
|
|
|
|
while [ "$ELAPSED" -lt "$PREVIOUS_SLOT_DRAIN_TIMEOUT_SECONDS" ]; do
|
|
ACTIVE_COUNT=$(read_runtime_value "$TARGET_CONTAINER" activeChatRequestCount 2>/dev/null || printf '0')
|
|
QUEUED_COUNT=$(read_runtime_value "$TARGET_CONTAINER" queuedChatRequestCount 2>/dev/null || printf '0')
|
|
PREVIOUS_ACTIVE_COUNT="${ACTIVE_COUNT:-0}"
|
|
PREVIOUS_QUEUED_COUNT="${QUEUED_COUNT:-0}"
|
|
write_deploy_state running drain-previous-slot "이전 슬롯 요청을 새 슬롯으로 이관하는 중입니다." "drain-previous-slot" running "active ${PREVIOUS_ACTIVE_COUNT} · queued ${PREVIOUS_QUEUED_COUNT}"
|
|
|
|
if [ "${ACTIVE_COUNT:-0}" = "0" ] && [ "${QUEUED_COUNT:-0}" = "0" ]; then
|
|
return 0
|
|
fi
|
|
|
|
sleep 2
|
|
ELAPSED=$((ELAPSED + 2))
|
|
done
|
|
|
|
echo "drain timeout reached for $TARGET_CONTAINER" >&2
|
|
return 1
|
|
}
|
|
|
|
ensure_proxy_running() {
|
|
docker compose -f "$COMPOSE_FILE" up -d --no-deps "$PROXY_SERVICE" >/dev/null
|
|
docker exec "$PROXY_CONTAINER" nginx -s reload >/dev/null
|
|
}
|
|
|
|
ACTIVE_SLOT=$(resolve_active_slot)
|
|
TARGET_SLOT="green"
|
|
TARGET_SERVICE="$GREEN_SERVICE"
|
|
TARGET_CONTAINER="$GREEN_CONTAINER"
|
|
PREVIOUS_SERVICE="$BLUE_SERVICE"
|
|
PREVIOUS_CONTAINER="$BLUE_CONTAINER"
|
|
PREVIOUS_SLOT="blue"
|
|
|
|
if [ "$ACTIVE_SLOT" = "green" ]; then
|
|
TARGET_SLOT="blue"
|
|
TARGET_SERVICE="$BLUE_SERVICE"
|
|
TARGET_CONTAINER="$BLUE_CONTAINER"
|
|
PREVIOUS_SERVICE="$GREEN_SERVICE"
|
|
PREVIOUS_CONTAINER="$GREEN_CONTAINER"
|
|
PREVIOUS_SLOT="green"
|
|
fi
|
|
|
|
write_deploy_state running build-target-slot "비활성 슬롯 빌드와 기동을 시작했습니다."
|
|
|
|
if BUILD_OUTPUT=$(docker compose -f "$COMPOSE_FILE" up -d --build --force-recreate --no-deps "$TARGET_SERVICE" 2>&1); then
|
|
[ -n "$BUILD_OUTPUT" ] && printf '%s\n' "$BUILD_OUTPUT"
|
|
write_deploy_state running build-target-slot "비활성 슬롯 빌드와 기동을 완료했습니다." "build-target-slot" completed "대상 슬롯 ${TARGET_SLOT} 준비 완료"
|
|
else
|
|
BUILD_STATUS=$?
|
|
LAST_DEPLOY_ERROR="대기 슬롯 빌드에 실패했습니다."
|
|
LAST_DEPLOY_LOG="${BUILD_OUTPUT:-docker compose build failed}"
|
|
[ -n "$BUILD_OUTPUT" ] && printf '%s\n' "$BUILD_OUTPUT" >&2
|
|
write_deploy_state failed failed "대기 슬롯 빌드에 실패했습니다." "build-target-slot" failed "대상 슬롯 ${TARGET_SLOT} 빌드 실패" "$LAST_DEPLOY_ERROR" "$LAST_DEPLOY_LOG"
|
|
exit "$BUILD_STATUS"
|
|
fi
|
|
|
|
write_deploy_state running verify-target-health "새 슬롯 health 확인을 진행합니다." "verify-target-health" running "대상 컨테이너 ${TARGET_CONTAINER}"
|
|
if wait_for_container_health "$TARGET_CONTAINER"; then
|
|
write_deploy_state running verify-target-health "새 슬롯 health 확인이 완료되었습니다." "verify-target-health" completed "대상 슬롯 ${TARGET_SLOT} 정상 응답"
|
|
else
|
|
LAST_DEPLOY_ERROR="새 슬롯 health 확인에 실패했습니다."
|
|
LAST_DEPLOY_LOG="health check failed for ${TARGET_CONTAINER}"
|
|
write_deploy_state failed failed "새 슬롯 health 확인에 실패했습니다." "verify-target-health" failed "대상 슬롯 ${TARGET_SLOT} health 실패" "$LAST_DEPLOY_ERROR" "$LAST_DEPLOY_LOG"
|
|
exit 1
|
|
fi
|
|
|
|
write_deploy_state running switch-proxy "프록시를 새 슬롯으로 전환합니다." "switch-proxy" running "활성 ${ACTIVE_SLOT} -> 대상 ${TARGET_SLOT}"
|
|
write_proxy_config "$TARGET_SLOT"
|
|
if ensure_proxy_running; then
|
|
write_deploy_state running switch-proxy "프록시 전환을 완료했습니다." "switch-proxy" completed "활성 ${ACTIVE_SLOT} -> 대상 ${TARGET_SLOT}"
|
|
else
|
|
LAST_DEPLOY_ERROR="프록시 전환에 실패했습니다."
|
|
LAST_DEPLOY_LOG="nginx reload failed"
|
|
write_deploy_state failed failed "프록시 전환에 실패했습니다." "switch-proxy" failed "활성 ${ACTIVE_SLOT} -> 대상 ${TARGET_SLOT}" "$LAST_DEPLOY_ERROR" "$LAST_DEPLOY_LOG"
|
|
exit 1
|
|
fi
|
|
printf '%s\n' "$TARGET_SLOT" >"$ACTIVE_SLOT_FILE"
|
|
ACTIVE_SLOT="$TARGET_SLOT"
|
|
|
|
if [ "$PREVIOUS_SERVICE" != "$TARGET_SERVICE" ]; then
|
|
set_container_draining "$PREVIOUS_CONTAINER" true
|
|
if wait_for_previous_slot_drain "$PREVIOUS_CONTAINER"; then
|
|
write_deploy_state running drain-previous-slot "이전 슬롯 요청 이관이 완료되었습니다." "drain-previous-slot" completed "active ${PREVIOUS_ACTIVE_COUNT:-0} · queued ${PREVIOUS_QUEUED_COUNT:-0}"
|
|
else
|
|
LAST_DEPLOY_ERROR="이전 슬롯 드레인 대기 시간이 초과되었습니다."
|
|
LAST_DEPLOY_LOG="drain timeout reached for ${PREVIOUS_CONTAINER}"
|
|
write_deploy_state failed failed "이전 슬롯 요청 이관이 시간 안에 끝나지 않았습니다." "drain-previous-slot" failed "active ${PREVIOUS_ACTIVE_COUNT:-0} · queued ${PREVIOUS_QUEUED_COUNT:-0}" "$LAST_DEPLOY_ERROR" "$LAST_DEPLOY_LOG"
|
|
exit 1
|
|
fi
|
|
|
|
write_deploy_state running rebuild-previous-slot "이전 슬롯을 대기 슬롯으로 복구합니다." "rebuild-previous-slot" running "대상 컨테이너 ${PREVIOUS_CONTAINER}"
|
|
if PREVIOUS_BUILD_OUTPUT=$(docker compose -f "$COMPOSE_FILE" up -d --build --force-recreate --no-deps "$PREVIOUS_SERVICE" 2>&1); then
|
|
[ -n "$PREVIOUS_BUILD_OUTPUT" ] && printf '%s\n' "$PREVIOUS_BUILD_OUTPUT"
|
|
else
|
|
PREVIOUS_BUILD_STATUS=$?
|
|
LAST_DEPLOY_ERROR="이전 슬롯 대기 복구 빌드에 실패했습니다."
|
|
LAST_DEPLOY_LOG="${PREVIOUS_BUILD_OUTPUT:-docker compose rebuild failed}"
|
|
[ -n "$PREVIOUS_BUILD_OUTPUT" ] && printf '%s\n' "$PREVIOUS_BUILD_OUTPUT" >&2
|
|
write_deploy_state failed failed "이전 슬롯 대기 복구 빌드에 실패했습니다." "rebuild-previous-slot" failed "대기 슬롯 ${PREVIOUS_SLOT} 복구 실패" "$LAST_DEPLOY_ERROR" "$LAST_DEPLOY_LOG"
|
|
exit "$PREVIOUS_BUILD_STATUS"
|
|
fi
|
|
|
|
if wait_for_container_health "$PREVIOUS_CONTAINER"; then
|
|
write_deploy_state running rebuild-previous-slot "이전 슬롯을 대기 슬롯으로 복구했습니다." "rebuild-previous-slot" completed "대기 슬롯 ${PREVIOUS_SLOT} 정상 응답"
|
|
else
|
|
LAST_DEPLOY_ERROR="이전 슬롯 복구 health 확인에 실패했습니다."
|
|
LAST_DEPLOY_LOG="health check failed for ${PREVIOUS_CONTAINER}"
|
|
write_deploy_state failed failed "이전 슬롯 복구 health 확인에 실패했습니다." "rebuild-previous-slot" failed "대기 슬롯 ${PREVIOUS_SLOT} health 실패" "$LAST_DEPLOY_ERROR" "$LAST_DEPLOY_LOG"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
write_deploy_state running recover-interrupted-chat "중단된 채팅 요청 복구를 확인합니다." "recover-interrupted-chat" running "대상 슬롯 ${TARGET_SLOT}"
|
|
if RECOVERY_JSON=$(recover_interrupted_chat_requests "$TARGET_CONTAINER" 2>&1); then
|
|
[ -n "$RECOVERY_JSON" ] && printf '%s\n' "$RECOVERY_JSON"
|
|
RECOVERY_COUNTS=$(printf '%s' "$RECOVERY_JSON" | node -e "let raw=''; process.stdin.on('data', (chunk) => raw += chunk); process.stdin.on('end', () => { try { const parsed = JSON.parse(raw); const recovered = parsed?.recovered ?? {}; process.stdout.write([recovered.sessionCount ?? '', recovered.restartedCount ?? '', recovered.requeuedCount ?? ''].join('\t')); } catch { process.stdout.write('\t\t'); } });")
|
|
RECOVERED_SESSION_COUNT=$(printf '%s' "$RECOVERY_COUNTS" | awk -F '\t' '{print $1}')
|
|
RECOVERED_RESTARTED_COUNT=$(printf '%s' "$RECOVERY_COUNTS" | awk -F '\t' '{print $2}')
|
|
RECOVERED_REQUEUED_COUNT=$(printf '%s' "$RECOVERY_COUNTS" | awk -F '\t' '{print $3}')
|
|
write_deploy_state running recover-interrupted-chat "중단된 채팅 요청 복구 확인이 완료되었습니다." "recover-interrupted-chat" completed "session ${RECOVERED_SESSION_COUNT:-0} · restarted ${RECOVERED_RESTARTED_COUNT:-0} · requeued ${RECOVERED_REQUEUED_COUNT:-0}"
|
|
else
|
|
RECOVERY_STATUS=$?
|
|
LAST_DEPLOY_ERROR="중단된 채팅 요청 복구 확인에 실패했습니다."
|
|
LAST_DEPLOY_LOG="${RECOVERY_JSON:-recover interrupted chat failed}"
|
|
[ -n "$RECOVERY_JSON" ] && printf '%s\n' "$RECOVERY_JSON" >&2
|
|
write_deploy_state failed failed "중단된 채팅 요청 복구 확인에 실패했습니다." "recover-interrupted-chat" failed "대상 슬롯 ${TARGET_SLOT}" "$LAST_DEPLOY_ERROR" "$LAST_DEPLOY_LOG"
|
|
exit "$RECOVERY_STATUS"
|
|
fi
|
|
|
|
DEPLOY_FINISHED="true"
|
|
write_deploy_state completed completed "WORK-SERVER 무중단 배포를 완료했습니다."
|
|
printf 'work-server zero-downtime switch completed: %s -> %s\n' "$PREVIOUS_SLOT" "$TARGET_SLOT"
|